diff --git a/makefile b/makefile new file mode 100644 index 0000000..20a802f --- /dev/null +++ b/makefile @@ -0,0 +1,21 @@ +FILES := $(shell git diff --name-only --diff-filter=AM $$(git merge-base origin/main HEAD) -- \*.py) + + +.PHONY: test +test: + pytest + +.PHONY: lint +lint: + flake8 --filename ./$(FILES) --max-complexity=10 --ignore=E501,W503 + +.PHONY: format +format: +ifneq ("$(FILES)"," ") + black $(FILES) + isort $(FILES) +endif + +.PHONY: tidy +tidy: format lint + diff --git a/parse.sh b/parse.sh index ca52f5e..9355963 100755 --- a/parse.sh +++ b/parse.sh @@ -13,13 +13,11 @@ function print_usage() { } results_dir="$HOME/results" -args=() while getopts l:r:h name; do case $name in l) event_log="$OPTARG" - args=("${args[@]}" -l "$OPTARG") ;; r) results_dir="$OPTARG" @@ -41,6 +39,4 @@ if [[ -z $event_log ]]; then exit 1 fi -args=("${args[@]}" -r "$results_dir") - -exec python3 -m spark_log_parser "${args[@]}" +exec python3 -m spark_log_parser -l "$event_log" -r "$results_dir" diff --git a/pyproject.toml b/pyproject.toml index 80f4331..d05b7f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "flit.buildapi" name = "spark_log_parser" authors = [{"name" = "Sync Computing"}] readme = "README.md" -requires-python = ">=3.6.1" +requires-python = ">=3.10.3" classifiers = [ "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", @@ -16,10 +16,6 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", ] dependencies = [ @@ -27,9 +23,19 @@ dependencies = [ "numpy==1.21.4", "pandas==1.3.5", "ujson==5.3.0", + "pydantic==1.9.0", + "requests==2.26.0", ] dynamic = ["version", "description"] +[project.optional-dependencies] +test = [ + "pytest==7.1.2", + "black==22.6.0", + "isort==5.10.1", + "flake8==4.0.1", +] + [project.urls] Home = "https://github.com/synccomputingcode/spark_log_parser" @@ -38,3 +44,12 @@ line-length = 100 [tool.isort] profile = "black" +line_length = 100 + +[tool.pytest.ini_options] +pythonpath = [ + "." +] +filterwarnings = [ + "ignore::UserWarning" +] diff --git a/spark_log_parser/__main__.py b/spark_log_parser/__main__.py index 159e4ad..1f6d748 100644 --- a/spark_log_parser/__main__.py +++ b/spark_log_parser/__main__.py @@ -1,37 +1,42 @@ +import argparse import logging +import os +import sys +import tempfile +from pathlib import Path logging.captureWarnings(True) -from spark_log_parser.parsing_models.application_model_v2 import sparkApplication - -import os -import argparse -import sys +from spark_log_parser.eventlog import EventLogBuilder # noqa: E402 +from spark_log_parser.parsing_models.application_model_v2 import sparkApplication # noqa: E402 logger = logging.getLogger("spark_log_parser") - -parser = argparse.ArgumentParser("spark_log_parser") -parser.add_argument("-l", "--log-file", required=True, help="path to event log") -parser.add_argument("-r", "--result-dir", required=True, help="path to directory in which to save parsed logs") -args = parser.parse_args() - -print("\n" + "*" * 12 + " Running the Log Parser for Spark Predictor " + "*" * 12 + "\n") -log_path = os.path.abspath(args.log_file) +parser = argparse.ArgumentParser("spark_log_parser") +parser.add_argument("-l", "--log-file", required=True, type=Path, help="path to event log") +parser.add_argument( + "-r", "--result-dir", required=True, help="path to directory in which to save parsed logs" +) +args = parser.parse_args() if not os.path.isdir(args.result_dir): logger.error("%s is not a directory", args.result_dir) sys.exit(1) -print("\n--Processing log file: " + log_path) +print("\n" + "*" * 12 + " Running the Log Parser for Spark Predictor " + "*" * 12 + "\n") +print("--Processing log file: " + str(args.log_file)) -log_name = os.path.basename(log_path) -result_path = os.path.join(args.result_dir, "parsed-" + log_name) +with tempfile.TemporaryDirectory() as work_dir: + event_log = EventLogBuilder(args.log_file.resolve().as_uri(), work_dir).build() + app = sparkApplication(eventlog=str(event_log)) -if os.path.exists(result_path): - os.remove(result_path) +if args.log_file.suffixes: + result_path = os.path.join( + args.result_dir, "parsed-" + args.log_file.name[: -len("".join(args.log_file.suffixes))] + ) +else: + result_path = os.path.join(args.result_dir, "parsed-" + args.log_file.name) -appobj = sparkApplication(eventlog=log_path) -appobj.save(result_path) +app.save(result_path) -print(f"--Log directory saved to: {result_path}") +print(f"--Result saved to: {result_path}.json") diff --git a/spark_log_parser/eventlog.py b/spark_log_parser/eventlog.py new file mode 100644 index 0000000..926fe28 --- /dev/null +++ b/spark_log_parser/eventlog.py @@ -0,0 +1,73 @@ +import json +import tempfile +from pathlib import Path +from urllib.parse import ParseResult + +import pandas as pd + +from spark_log_parser.extractor import Extractor + + +class EventLogBuilder: + def __init__(self, source_url: ParseResult | str, work_dir: Path | str, s3_client=None): + self.source_url = source_url + self.work_dir = self._validate_work_dir(work_dir) + self.s3_client = s3_client + self.extractor = Extractor(self.source_url, self.work_dir, self.s3_client) + + def _validate_work_dir(self, work_dir: Path | str) -> Path: + work_dir_path = work_dir if isinstance(work_dir, Path) else Path(work_dir) + if not work_dir_path.is_dir(): + raise ValueError("Path is not a directory") + + return work_dir_path + + def build(self) -> Path: + event_logs = self.extractor.extract() + + self.event_log = self._concat(event_logs) + + return self.event_log + + def _concat(self, event_logs: list[Path]) -> Path: + if len(event_logs) == 1: + return event_logs[0] + + dat = [] + for log in event_logs: + with open(log) as log_file: + try: + line = json.loads(log_file.readline()) + except ValueError: + continue # Maybe a Databricks pricing file + if line["Event"] == "DBCEventLoggingListenerMetadata": + dat.append((line["Rollover Number"], line["SparkContext Id"], log)) + else: + raise ValueError("Expected DBC event not found") + + df = pd.DataFrame(dat, columns=["rollover_index", "context_id", "path"]).sort_values( + "rollover_index" + ) + + self._validate_rollover_logs(df) + + event_log = Path(tempfile.mkstemp(suffix="-concatenated.json", dir=str(self.work_dir))[1]) + with open(event_log, "w") as fobj: + for path in df.path: + with open(path) as part_fobj: + for line in part_fobj: + fobj.write(line) + + return event_log + + def _validate_rollover_logs(self, df: pd.DataFrame): + if not len(df.context_id.unique()) == 1: + raise ValueError("Not all rollover files have the same Spark context ID") + + diffs = df.rollover_index.diff()[1:] + + if any(diffs > 1) or df.rollover_index[0] > 0: + raise ValueError("Rollover file appears to be missing") + + if any(diffs < 1): + raise ValueError("Duplicate rollover file detected") diff --git a/spark_log_parser/extractor.py b/spark_log_parser/extractor.py new file mode 100644 index 0000000..afce1a7 --- /dev/null +++ b/spark_log_parser/extractor.py @@ -0,0 +1,250 @@ +import gzip +import os +import shutil +import struct +import tarfile +import zipfile +from pathlib import Path +from urllib.parse import ParseResult, urlparse + +import requests + + +class Extractor: + """ + Expands archive to a "work" directory and provides a list of extracted files + """ + + ALLOWED_SCHEMES = {"https", "s3", "file"} + FILE_SKIP_PATTERNS = [".DS_Store".lower(), "__MACOSX".lower(), "/."] + + THRESHOLD_ENTRIES = 100 + THRESHOLD_SIZE = 5000000000 + THRESHOLD_RATIO = 100 + + def __init__(self, source_url: ParseResult | str, work_dir: Path | str, s3_client=None): + self.source_url = self._validate_url(source_url) + self.work_dir = self._validate_work_dir(work_dir) + self.s3_client = self._validate_s3_client(s3_client) + self.file_total = 0 + self.size_total = 0 + + def _validate_url(self, url: ParseResult | str) -> ParseResult: + parsed_url = url if isinstance(url, ParseResult) else urlparse(url) + if parsed_url.scheme not in self.ALLOWED_SCHEMES: + raise ValueError( + "URL scheme '%s' is not one of {'%s'}" + % (parsed_url.scheme, "', '".join(self.ALLOWED_SCHEMES)) + ) + + return parsed_url + + def _validate_work_dir(self, work_dir: Path | str) -> Path: + work_dir_path = work_dir if isinstance(work_dir, Path) else Path(work_dir) + if not work_dir_path.is_dir(): + raise ValueError("Path is not a directory") + + return work_dir_path + + def _validate_s3_client(self, s3_client): + if self.source_url.scheme == "s3" and s3_client is None: + raise ValueError("An S3 client is needed for S3 URLs") + + return s3_client + + def extract(self) -> list[Path]: + if self.source_url.scheme in {"https", "s3"}: + self._download() + local_path = self.work_dir + else: + local_path = Path(self.source_url.path) + + result_files = self._extract(local_path, self.work_dir) + + return result_files + + def _extract(self, event_log: Path, extract_dir: Path | None = None) -> list[Path]: + """returns list of uncompressed files""" + if not extract_dir: + extract_dir = event_log.parent + + if event_log.is_dir(): + return self._extract_dir(event_log, extract_dir) + + extension = "".join(event_log.suffixes) + if extension.endswith(".tar.gz"): + return self._extract_tgz( + event_log, extract_dir.joinpath(event_log.name[: -len(".tar.gz")]) + ) + if extension.endswith(".tgz"): + return self._extract_tgz( + event_log, extract_dir.joinpath(event_log.name[: -len(".tgz")]) + ) + if extension.endswith(".zip"): + return self._extract_zip( + event_log, extract_dir.joinpath(event_log.name[: -len(".zip")]) + ) + if extension.endswith(".gz"): + return self._extract_gz(event_log, extract_dir.joinpath(event_log.name[: -len(".gz")])) + if extension in {".json", ".log", ""}: + return [event_log] + + raise ValueError("Unsupported extension found in the archive") + + def _extract_dir(self, log_dir: Path, extract_dir: Path) -> list[Path]: + paths = [] + + for root, _, files in os.walk(log_dir): + for file in files: + path = Path(root, file) + if self._should_skip_file(file): + if path.is_relative_to(self.work_dir): + path.unlink() + continue + + self._add_to_stats_and_verify(path.stat().st_size) + + sub_paths = self._extract(path, Path(extract_dir, path.parent.relative_to(log_dir))) + if path not in sub_paths: + self._remove_from_stats(path.stat().st_size) + # Remove the archive if it was extracted to free up space + if path.is_relative_to(self.work_dir): + path.unlink() + + paths += sub_paths + + return paths + + def _extract_tgz(self, archive_path: Path, extract_dir: Path) -> list[Path]: + paths = [] + + with tarfile.open(archive_path) as tar_file: + for tf in tar_file: + if self._should_skip_file(tf.name): + continue + if tf.isdir(): + continue + + self._add_to_stats_and_verify(tf.size) + + extract_path = extract_dir.joinpath(tf.name) + extract_path.parent.mkdir(parents=True, exist_ok=True) + with tar_file.extractfile(tf.name) as source, open(extract_path, "wb") as target: + shutil.copyfileobj(source, target) + + # Remove the archive if it was extracted to free up space + sub_paths = self._extract(extract_path) + if extract_path not in sub_paths: + self._remove_from_stats(tf.size) + extract_path.unlink() + + paths += sub_paths + + return paths + + def _extract_zip(self, archive_path: Path, extract_dir: Path) -> list[Path]: + paths = [] + + with zipfile.ZipFile(archive_path) as zfile: + for zinfo in zfile.infolist(): + if self._should_skip_file(zinfo.filename): + continue + if zinfo.is_dir(): + continue + + self._add_to_stats_and_verify(zinfo.file_size) + + extract_path = extract_dir.joinpath(zinfo.filename) + extract_path.parent.mkdir(parents=True, exist_ok=True) + with zfile.open(zinfo.filename) as source, open(extract_path, "wb") as target: + shutil.copyfileobj(source, target) + + # Remove the archive if it was extracted to free up space + sub_paths = self._extract(extract_path) + if extract_path not in sub_paths: + self._remove_from_stats(zinfo.file_size) + extract_path.unlink() + + paths += sub_paths + + return paths + + def _extract_gz(self, archive_path: Path, extract_path: Path) -> list[Path]: + with open(archive_path, "rb") as fobj: + fobj.seek(-4, 2) + size = struct.unpack("I", fobj.read(4))[0] + + self._add_to_stats_and_verify(size) + + extract_path.parent.mkdir(parents=True, exist_ok=True) + with gzip.open(archive_path, "rb") as source, open(extract_path, "wb") as target: + shutil.copyfileobj(source, target) + + # Remove the archive if it was extracted to free up space + sub_paths = self._extract(extract_path) + if extract_path not in sub_paths: + self._remove_from_stats(size) + extract_path.unlink() + + return sub_paths + + def _should_skip_file(self, filename: str) -> bool: + if filename.startswith("."): + return True + + filename = filename.lower() + for name in self.FILE_SKIP_PATTERNS: + if name in filename: + return True + + return False + + def _add_to_stats_and_verify(self, size, count=1): + self.size_total += size + self.file_total += count + + ratio = size / self.size_total + if ratio > self.THRESHOLD_RATIO: + raise AssertionError("Encountered suspicious compression ratio in the archive") + + if self.size_total > self.THRESHOLD_SIZE: + raise AssertionError("The archive is too big") + + if self.file_total > self.THRESHOLD_ENTRIES: + raise AssertionError("Too many files in the archive") + + def _remove_from_stats(self, size, count=1): + self.size_total -= size + self.file_total -= count + + def _download(self): + if self.source_url.scheme == "https": + response = requests.get(self.source_url, stream=True) + target_path = self.work_dir.joinpath(self.source_url.path.split("/")[-1]) + with open(target_path, "wb") as fobj: + for chunk in response.iter_content(): + fobj.write(chunk) + elif self.source_url.scheme == "s3": + source_key = self.source_url.path.lstrip("/") + source_bucket = self.source_url.netloc + + s3_content_count = 0 + s3_content_size = 0 + result = self.s3_client.list_objects_v2(Bucket=source_bucket, Prefix=source_key) + if "Contents" in result: + relative_index = source_key.rfind("/") + 1 + for content in result["Contents"]: + s3_content_count += 1 + s3_content_size += content["Size"] + if s3_content_count > self.THRESHOLD_ENTRIES: + raise AssertionError("Too many objects at %s" % self.source_url) + if s3_content_size > self.THRESHOLD_SIZE: + raise AssertionError( + "Size limit exceeded while downloading from %s" % self.source_url + ) + + target_path = self.work_dir.joinpath( + *content["Key"][relative_index:].split("/") + ) + target_path.parent.mkdir(parents=True, exist_ok=True) + self.s3_client.download_file(source_bucket, content["Key"], str(target_path)) diff --git a/tests/logs/bad/duplicate-part.tgz b/tests/logs/bad/duplicate-part.tgz new file mode 100644 index 0000000..01df914 Binary files /dev/null and b/tests/logs/bad/duplicate-part.tgz differ diff --git a/tests/logs/bad/missing-dbc-event.zip b/tests/logs/bad/missing-dbc-event.zip new file mode 100644 index 0000000..806123b Binary files /dev/null and b/tests/logs/bad/missing-dbc-event.zip differ diff --git a/tests/logs/bad/missing-part.zip b/tests/logs/bad/missing-part.zip new file mode 100644 index 0000000..7cddc3e Binary files /dev/null and b/tests/logs/bad/missing-part.zip differ diff --git a/tests/logs/bad/non-unique-context-id.zip b/tests/logs/bad/non-unique-context-id.zip new file mode 100644 index 0000000..0839590 Binary files /dev/null and b/tests/logs/bad/non-unique-context-id.zip differ diff --git a/tests/logs/databricks-rollover-messy.zip b/tests/logs/databricks-rollover-messy.zip new file mode 100644 index 0000000..83df212 Binary files /dev/null and b/tests/logs/databricks-rollover-messy.zip differ diff --git a/tests/logs/databricks-rollover.zip b/tests/logs/databricks-rollover.zip new file mode 100644 index 0000000..bc6d5a0 Binary files /dev/null and b/tests/logs/databricks-rollover.zip differ diff --git a/tests/logs/databricks.json b/tests/logs/databricks.json new file mode 100644 index 0000000..0f43487 --- /dev/null +++ b/tests/logs/databricks.json @@ -0,0 +1,1162 @@ +{"Event":"DBCEventLoggingListenerMetadata","Spark Version":"3.1.2","Timestamp":1646432444550,"Rollover Number":0,"SparkContext Id":5706921391408253054} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":47016,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"10.77.243.103","Port":39191},"Maximum Memory":428762726,"Timestamp":1646432444382,"Maximum Onheap Memory":428762726,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/zulu8-ca-amd64/jre","Java Version":"1.8.0_302 (Azul Systems, Inc.)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","eventLog.rolloverIntervalSeconds":"3600","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.executor.extraJavaOptions":"-Djava.io.tmpdir=/local_disk0/tmp -XX:ReservedCodeCacheSize=512m -XX:+UseCodeCacheFlushing -Djava.security.properties=/databricks/spark/dbconf/java/extra.security -XX:-UseContainerSupport -XX:+PrintFlagsFinal -XX:+PrintGCDateStamps -XX:+PrintGCDetails -verbose:gc -Xss4m -Djava.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni -Djavax.xml.datatype.DatatypeFactory=com.sun.org.apache.xerces.internal.jaxp.datatype.DatatypeFactoryImpl -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -Djavax.xml.validation.SchemaFactory:http://www.w3.org/2001/XMLSchema=com.sun.org.apache.xerces.internal.jaxp.validation.XMLSchemaFactory -Dorg.xml.sax.driver=com.sun.org.apache.xerces.internal.parsers.SAXParser -Dorg.w3c.dom.DOMImplementationSourceList=com.sun.org.apache.xerces.internal.dom.DOMXSImplementationSourceImpl -Djavax.net.ssl.sessionCacheSize=10000 -Dscala.reflect.runtime.disable.typetag.cache=true -Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true -Dlog4j2.formatMsgNoLookups=true -Ddatabricks.serviceName=spark-executor-1","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","libraryDownload.timeoutSeconds":"180","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"Databricks Shell","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.databricks.driverNodeTypeId":"m5.xlarge","libraryDownload.sleepIntervalSeconds":"5","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.executor.extraClassPath":"/databricks/spark/dbconf/log4j/executor:/databricks/spark/dbconf/jets3t/:/databricks/spark/dbconf/hadoop:/databricks/hive/conf:/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-client-common_deploy.jar:/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-hive2-client_deploy.jar:/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-shim-common_deploy.jar:/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-shim-hive1_deploy.jar:/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-shim-hive2_deploy.jar:/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-shim-loader_deploy.jar:/databricks/jars/----jackson_annotations_shaded--libjackson-annotations.jar:/databricks/jars/----jackson_core_shaded--libjackson-core.jar:/databricks/jars/----jackson_databind_shaded--libjackson-databind.jar:/databricks/jars/----jackson_datatype_joda_shaded--libjackson-datatype-joda.jar:/databricks/jars/----scalapb_090--com.lihaoyi__fastparse_2.12__2.1.3_shaded.jar:/databricks/jars/----scalapb_090--com.lihaoyi__sourcecode_2.12__0.1.7_shaded.jar:/databricks/jars/----scalapb_090--runtime-unshaded-jetty9-hadoop1_2.12_deploy_shaded.jar:/databricks/jars/----workspace_spark_3_1--common--kvstore--kvstore-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--common--network-common--network-common-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--common--network-shuffle--network-shuffle-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--common--sketch--sketch-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--common--tags--tags-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--common--unsafe--unsafe-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--core--core-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--core--libcore_generated_resources.jar:/databricks/jars/----workspace_spark_3_1--core--libcore_resources.jar:/databricks/jars/----workspace_spark_3_1--core--proto-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--graphx--graphx-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--launcher--launcher-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--antlr--antlr--antlr__antlr__2.7.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--amazon-kinesis-client--com.amazonaws__amazon-kinesis-client__1.12.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-autoscaling--com.amazonaws__aws-java-sdk-autoscaling__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudformation--com.amazonaws__aws-java-sdk-cloudformation__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudfront--com.amazonaws__aws-java-sdk-cloudfront__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudhsm--com.amazonaws__aws-java-sdk-cloudhsm__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudsearch--com.amazonaws__aws-java-sdk-cloudsearch__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudtrail--com.amazonaws__aws-java-sdk-cloudtrail__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudwatch--com.amazonaws__aws-java-sdk-cloudwatch__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudwatchmetrics--com.amazonaws__aws-java-sdk-cloudwatchmetrics__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-codedeploy--com.amazonaws__aws-java-sdk-codedeploy__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cognitoidentity--com.amazonaws__aws-java-sdk-cognitoidentity__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cognitosync--com.amazonaws__aws-java-sdk-cognitosync__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-config--com.amazonaws__aws-java-sdk-config__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-core--com.amazonaws__aws-java-sdk-core__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-datapipeline--com.amazonaws__aws-java-sdk-datapipeline__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-directconnect--com.amazonaws__aws-java-sdk-directconnect__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-directory--com.amazonaws__aws-java-sdk-directory__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-dynamodb--com.amazonaws__aws-java-sdk-dynamodb__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-ec2--com.amazonaws__aws-java-sdk-ec2__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-ecs--com.amazonaws__aws-java-sdk-ecs__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-efs--com.amazonaws__aws-java-sdk-efs__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-elasticache--com.amazonaws__aws-java-sdk-elasticache__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-elasticbeanstalk--com.amazonaws__aws-java-sdk-elasticbeanstalk__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-elasticloadbalancing--com.amazonaws__aws-java-sdk-elasticloadbalancing__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-elastictranscoder--com.amazonaws__aws-java-sdk-elastictranscoder__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-emr--com.amazonaws__aws-java-sdk-emr__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-glacier--com.amazonaws__aws-java-sdk-glacier__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-glue--com.amazonaws__aws-java-sdk-glue__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-iam--com.amazonaws__aws-java-sdk-iam__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-importexport--com.amazonaws__aws-java-sdk-importexport__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-kinesis--com.amazonaws__aws-java-sdk-kinesis__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-kms--com.amazonaws__aws-java-sdk-kms__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-lambda--com.amazonaws__aws-java-sdk-lambda__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-logs--com.amazonaws__aws-java-sdk-logs__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-machinelearning--com.amazonaws__aws-java-sdk-machinelearning__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-marketplacecommerceanalytics--com.amazonaws__aws-java-sdk-marketplacecommerceanalytics__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-marketplacemeteringservice--com.amazonaws__aws-java-sdk-marketplacemeteringservice__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-opsworks--com.amazonaws__aws-java-sdk-opsworks__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-rds--com.amazonaws__aws-java-sdk-rds__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-redshift--com.amazonaws__aws-java-sdk-redshift__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-route53--com.amazonaws__aws-java-sdk-route53__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-s3--com.amazonaws__aws-java-sdk-s3__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-ses--com.amazonaws__aws-java-sdk-ses__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-simpledb--com.amazonaws__aws-java-sdk-simpledb__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-simpleworkflow--com.amazonaws__aws-java-sdk-simpleworkflow__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-sns--com.amazonaws__aws-java-sdk-sns__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-sqs--com.amazonaws__aws-java-sdk-sqs__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-ssm--com.amazonaws__aws-java-sdk-ssm__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-storagegateway--com.amazonaws__aws-java-sdk-storagegateway__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-sts--com.amazonaws__aws-java-sdk-sts__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-support--com.amazonaws__aws-java-sdk-support__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-swf-libraries--com.amazonaws__aws-java-sdk-swf-libraries__1.11.22.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-workspaces--com.amazonaws__aws-java-sdk-workspaces__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--jmespath-java--com.amazonaws__jmespath-java__1.11.655.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.chuusai--shapeless_2.12--com.chuusai__shapeless_2.12__2.3.3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.clearspring.analytics--stream--com.clearspring.analytics__stream__2.9.6.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.databricks--Rserve--com.databricks__Rserve__1.8-3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.databricks.scalapb--compilerplugin_2.12--com.databricks.scalapb__compilerplugin_2.12__0.4.15-10.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.databricks.scalapb--scalapb-runtime_2.12--com.databricks.scalapb__scalapb-runtime_2.12__0.4.15-10.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.esotericsoftware--kryo-shaded--com.esotericsoftware__kryo-shaded__4.0.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.esotericsoftware--minlog--com.esotericsoftware__minlog__1.3.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml--classmate--com.fasterxml__classmate__1.3.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.core--jackson-annotations--com.fasterxml.jackson.core__jackson-annotations__2.10.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.core--jackson-core--com.fasterxml.jackson.core__jackson-core__2.10.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.core--jackson-databind--com.fasterxml.jackson.core__jackson-databind__2.10.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.dataformat--jackson-dataformat-cbor--com.fasterxml.jackson.dataformat__jackson-dataformat-cbor__2.10.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.datatype--jackson-datatype-joda--com.fasterxml.jackson.datatype__jackson-datatype-joda__2.10.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.module--jackson-module-paranamer--com.fasterxml.jackson.module__jackson-module-paranamer__2.10.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.module--jackson-module-scala_2.12--com.fasterxml.jackson.module__jackson-module-scala_2.12__2.10.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.ben-manes.caffeine--caffeine--com.github.ben-manes.caffeine__caffeine__2.3.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil--jniloader--com.github.fommil__jniloader__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--core--com.github.fommil.netlib__core__1.1.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--native_ref-java--com.github.fommil.netlib__native_ref-java__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--native_ref-java-natives--com.github.fommil.netlib__native_ref-java-natives__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--native_system-java--com.github.fommil.netlib__native_system-java__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--native_system-java-natives--com.github.fommil.netlib__native_system-java-natives__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--netlib-native_ref-linux-x86_64-natives--com.github.fommil.netlib__netlib-native_ref-linux-x86_64-natives__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--netlib-native_system-linux-x86_64-natives--com.github.fommil.netlib__netlib-native_system-linux-x86_64-natives__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.joshelser--dropwizard-metrics-hadoop-metrics2-reporter--com.github.joshelser__dropwizard-metrics-hadoop-metrics2-reporter__0.1.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.luben--zstd-jni--com.github.luben__zstd-jni__1.4.8-1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.wendykierp--JTransforms--com.github.wendykierp__JTransforms__3.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.code.findbugs--jsr305--com.google.code.findbugs__jsr305__3.0.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.code.gson--gson--com.google.code.gson__gson__2.2.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.crypto.tink--tink--com.google.crypto.tink__tink__1.6.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.flatbuffers--flatbuffers-java--com.google.flatbuffers__flatbuffers-java__1.9.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.guava--guava--com.google.guava__guava__15.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.protobuf--protobuf-java--com.google.protobuf__protobuf-java__2.6.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.h2database--h2--com.h2database__h2__1.4.195.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.helger--profiler--com.helger__profiler__1.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.jcraft--jsch--com.jcraft__jsch__0.1.50.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.jolbox--bonecp--com.jolbox__bonecp__0.8.0.RELEASE.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.lihaoyi--sourcecode_2.12--com.lihaoyi__sourcecode_2.12__0.1.9.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.microsoft.azure--azure-data-lake-store-sdk--com.microsoft.azure__azure-data-lake-store-sdk__2.3.9.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.ning--compress-lzf--com.ning__compress-lzf__1.0.3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.sun.mail--javax.mail--com.sun.mail__javax.mail__1.5.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.tdunning--json--com.tdunning__json__1.8.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.thoughtworks.paranamer--paranamer--com.thoughtworks.paranamer__paranamer__2.8.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.trueaccord.lenses--lenses_2.12--com.trueaccord.lenses__lenses_2.12__0.4.12.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--chill-java--com.twitter__chill-java__0.9.5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--chill_2.12--com.twitter__chill_2.12__0.9.5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-app_2.12--com.twitter__util-app_2.12__7.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-core_2.12--com.twitter__util-core_2.12__7.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-function_2.12--com.twitter__util-function_2.12__7.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-jvm_2.12--com.twitter__util-jvm_2.12__7.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-lint_2.12--com.twitter__util-lint_2.12__7.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-registry_2.12--com.twitter__util-registry_2.12__7.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-stats_2.12--com.twitter__util-stats_2.12__7.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.typesafe--config--com.typesafe__config__1.2.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.typesafe.scala-logging--scala-logging_2.12--com.typesafe.scala-logging__scala-logging_2.12__3.7.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.univocity--univocity-parsers--com.univocity__univocity-parsers__2.9.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.zaxxer--HikariCP--com.zaxxer__HikariCP__3.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-beanutils--commons-beanutils--commons-beanutils__commons-beanutils__1.9.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-cli--commons-cli--commons-cli__commons-cli__1.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-codec--commons-codec--commons-codec__commons-codec__1.10.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-collections--commons-collections--commons-collections__commons-collections__3.2.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-configuration--commons-configuration--commons-configuration__commons-configuration__1.6.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-dbcp--commons-dbcp--commons-dbcp__commons-dbcp__1.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-digester--commons-digester--commons-digester__commons-digester__1.8.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-fileupload--commons-fileupload--commons-fileupload__commons-fileupload__1.3.3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-httpclient--commons-httpclient--commons-httpclient__commons-httpclient__3.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-io--commons-io--commons-io__commons-io__2.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-lang--commons-lang--commons-lang__commons-lang__2.6.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-logging--commons-logging--commons-logging__commons-logging__1.1.3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-net--commons-net--commons-net__commons-net__3.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-pool--commons-pool--commons-pool__commons-pool__1.5.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--info.ganglia.gmetric4j--gmetric4j--info.ganglia.gmetric4j__gmetric4j__1.0.10.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.airlift--aircompressor--io.airlift__aircompressor__0.10.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.delta--delta-sharing-spark_2.12--io.delta__delta-sharing-spark_2.12__0.2.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-core--io.dropwizard.metrics__metrics-core__4.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-graphite--io.dropwizard.metrics__metrics-graphite__4.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-healthchecks--io.dropwizard.metrics__metrics-healthchecks__4.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-jetty9--io.dropwizard.metrics__metrics-jetty9__4.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-jmx--io.dropwizard.metrics__metrics-jmx__4.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-json--io.dropwizard.metrics__metrics-json__4.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-jvm--io.dropwizard.metrics__metrics-jvm__4.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-servlets--io.dropwizard.metrics__metrics-servlets__4.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.netty--netty-all--io.netty__netty-all__4.1.51.Final.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient--io.prometheus__simpleclient__0.7.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient_common--io.prometheus__simpleclient_common__0.7.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient_dropwizard--io.prometheus__simpleclient_dropwizard__0.7.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient_pushgateway--io.prometheus__simpleclient_pushgateway__0.7.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient_servlet--io.prometheus__simpleclient_servlet__0.7.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus.jmx--collector--io.prometheus.jmx__collector__0.12.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jakarta.annotation--jakarta.annotation-api--jakarta.annotation__jakarta.annotation-api__1.3.5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jakarta.validation--jakarta.validation-api--jakarta.validation__jakarta.validation-api__2.0.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jakarta.ws.rs--jakarta.ws.rs-api--jakarta.ws.rs__jakarta.ws.rs-api__2.1.6.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.activation--activation--javax.activation__activation__1.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.el--javax.el-api--javax.el__javax.el-api__2.2.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.jdo--jdo-api--javax.jdo__jdo-api__3.0.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.servlet--javax.servlet-api--javax.servlet__javax.servlet-api__3.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.servlet.jsp--jsp-api--javax.servlet.jsp__jsp-api__2.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.transaction--jta--javax.transaction__jta__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.transaction--transaction-api--javax.transaction__transaction-api__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.xml.bind--jaxb-api--javax.xml.bind__jaxb-api__2.2.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.xml.stream--stax-api--javax.xml.stream__stax-api__1.0-2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javolution--javolution--javolution__javolution__5.5.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jets3t-0.7--com.databricks--jets3t--com.databricks__jets3t__0.7.1-0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jets3t-0.7--liball_deps_2.12.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jline--jline--jline__jline__2.14.6.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--joda-time--joda-time--joda-time__joda-time__2.10.5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--liball_deps_2.12.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--log4j--apache-log4j-extras--log4j__apache-log4j-extras__1.2.17.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--log4j--log4j--log4j__log4j__1.2.17.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.java.dev.jna--jna--net.java.dev.jna__jna__5.8.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.razorvine--pyrolite--net.razorvine__pyrolite__4.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.sf.jpam--jpam--net.sf.jpam__jpam__1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.sf.opencsv--opencsv--net.sf.opencsv__opencsv__2.3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.sf.supercsv--super-csv--net.sf.supercsv__super-csv__2.2.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.snowflake--snowflake-ingest-sdk--net.snowflake__snowflake-ingest-sdk__0.9.6.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.snowflake--snowflake-jdbc--net.snowflake__snowflake-jdbc__3.13.3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.snowflake--spark-snowflake_2.12--net.snowflake__spark-snowflake_2.12__2.9.0-spark_3.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.sourceforge.f2j--arpack_combined_all--net.sourceforge.f2j__arpack_combined_all__0.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.acplt.remotetea--remotetea-oncrpc--org.acplt.remotetea__remotetea-oncrpc__1.1.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.antlr--ST4--org.antlr__ST4__4.0.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.antlr--antlr-runtime--org.antlr__antlr-runtime__3.5.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.antlr--antlr4-runtime--org.antlr__antlr4-runtime__4.8-1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.antlr--stringtemplate--org.antlr__stringtemplate__3.2.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.ant--ant--org.apache.ant__ant__1.9.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.ant--ant-jsch--org.apache.ant__ant-jsch__1.9.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.ant--ant-launcher--org.apache.ant__ant-launcher__1.9.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-format--org.apache.arrow__arrow-format__2.0.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-memory-core--org.apache.arrow__arrow-memory-core__2.0.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-memory-netty--org.apache.arrow__arrow-memory-netty__2.0.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-vector--org.apache.arrow__arrow-vector__2.0.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.avro--avro--org.apache.avro__avro__1.8.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.avro--avro-ipc--org.apache.avro__avro-ipc__1.8.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.avro--avro-mapred-hadoop2--org.apache.avro__avro-mapred-hadoop2__1.8.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-compress--org.apache.commons__commons-compress__1.20.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-crypto--org.apache.commons__commons-crypto__1.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-lang3--org.apache.commons__commons-lang3__3.10.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-math3--org.apache.commons__commons-math3__3.4.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-text--org.apache.commons__commons-text__1.6.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.curator--curator-client--org.apache.curator__curator-client__2.7.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.curator--curator-framework--org.apache.curator__curator-framework__2.7.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.curator--curator-recipes--org.apache.curator__curator-recipes__2.7.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.derby--derby--org.apache.derby__derby__10.12.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.directory.api--api-asn1-api--org.apache.directory.api__api-asn1-api__1.0.0-M20.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.directory.api--api-util--org.apache.directory.api__api-util__1.0.0-M20.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.directory.server--apacheds-i18n--org.apache.directory.server__apacheds-i18n__2.0.0-M15.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.directory.server--apacheds-kerberos-codec--org.apache.directory.server__apacheds-kerberos-codec__2.0.0-M15.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-annotations--org.apache.hadoop__hadoop-annotations__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-auth--org.apache.hadoop__hadoop-auth__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-client--org.apache.hadoop__hadoop-client__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-common--org.apache.hadoop__hadoop-common__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-hdfs--org.apache.hadoop__hadoop-hdfs__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-app--org.apache.hadoop__hadoop-mapreduce-client-app__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-common--org.apache.hadoop__hadoop-mapreduce-client-common__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-core--org.apache.hadoop__hadoop-mapreduce-client-core__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-jobclient--org.apache.hadoop__hadoop-mapreduce-client-jobclient__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-shuffle--org.apache.hadoop__hadoop-mapreduce-client-shuffle__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-yarn-api--org.apache.hadoop__hadoop-yarn-api__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-yarn-client--org.apache.hadoop__hadoop-yarn-client__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-yarn-common--org.apache.hadoop__hadoop-yarn-common__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-yarn-server-common--org.apache.hadoop__hadoop-yarn-server-common__2.7.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-beeline--org.apache.hive__hive-beeline__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-cli--org.apache.hive__hive-cli__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-jdbc--org.apache.hive__hive-jdbc__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-llap-client--org.apache.hive__hive-llap-client__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-llap-common--org.apache.hive__hive-llap-common__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-serde--org.apache.hive__hive-serde__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-shims--org.apache.hive__hive-shims__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-storage-api--org.apache.hive__hive-storage-api__2.7.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive.shims--hive-shims-0.23--org.apache.hive.shims__hive-shims-0.23__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive.shims--hive-shims-common--org.apache.hive.shims__hive-shims-common__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive.shims--hive-shims-scheduler--org.apache.hive.shims__hive-shims-scheduler__2.3.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.htrace--htrace-core--org.apache.htrace__htrace-core__3.1.0-incubating.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.httpcomponents--httpclient--org.apache.httpcomponents__httpclient__4.5.6.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.httpcomponents--httpcore--org.apache.httpcomponents__httpcore__4.4.12.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.ivy--ivy--org.apache.ivy__ivy__2.4.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.mesos--mesos-shaded-protobuf--org.apache.mesos__mesos-shaded-protobuf__1.4.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.orc--orc-core--org.apache.orc__orc-core__1.5.12.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.orc--orc-mapreduce--org.apache.orc__orc-mapreduce__1.5.12.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.orc--orc-shims--org.apache.orc__orc-shims__1.5.12.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-column--org.apache.parquet__parquet-column__1.10.1-databricks9.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-common--org.apache.parquet__parquet-common__1.10.1-databricks9.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-encoding--org.apache.parquet__parquet-encoding__1.10.1-databricks9.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-format--org.apache.parquet__parquet-format__2.4.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-hadoop--org.apache.parquet__parquet-hadoop__1.10.1-databricks9.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-jackson--org.apache.parquet__parquet-jackson__1.10.1-databricks9.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.thrift--libfb303--org.apache.thrift__libfb303__0.9.3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.thrift--libthrift--org.apache.thrift__libthrift__0.12.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.xbean--xbean-asm7-shaded--org.apache.xbean__xbean-asm7-shaded__4.15.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.jackson--jackson-core-asl--org.codehaus.jackson__jackson-core-asl__1.9.13.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.jackson--jackson-jaxrs--org.codehaus.jackson__jackson-jaxrs__1.9.13.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.jackson--jackson-mapper-asl--org.codehaus.jackson__jackson-mapper-asl__1.9.13.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.jackson--jackson-xc--org.codehaus.jackson__jackson-xc__1.9.13.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.janino--commons-compiler--org.codehaus.janino__commons-compiler__3.0.16.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.janino--janino--org.codehaus.janino__janino__3.0.16.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.datanucleus--datanucleus-api-jdo--org.datanucleus__datanucleus-api-jdo__4.2.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.datanucleus--datanucleus-core--org.datanucleus__datanucleus-core__4.1.17.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.datanucleus--datanucleus-rdbms--org.datanucleus__datanucleus-rdbms__4.1.19.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.datanucleus--javax.jdo--org.datanucleus__javax.jdo__3.2.0-m3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-client--org.eclipse.jetty__jetty-client__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-continuation--org.eclipse.jetty__jetty-continuation__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-http--org.eclipse.jetty__jetty-http__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-io--org.eclipse.jetty__jetty-io__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-jndi--org.eclipse.jetty__jetty-jndi__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-plus--org.eclipse.jetty__jetty-plus__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-proxy--org.eclipse.jetty__jetty-proxy__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-security--org.eclipse.jetty__jetty-security__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-server--org.eclipse.jetty__jetty-server__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-servlet--org.eclipse.jetty__jetty-servlet__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-servlets--org.eclipse.jetty__jetty-servlets__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-util--org.eclipse.jetty__jetty-util__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-util-ajax--org.eclipse.jetty__jetty-util-ajax__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-webapp--org.eclipse.jetty__jetty-webapp__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-xml--org.eclipse.jetty__jetty-xml__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-api--org.eclipse.jetty.websocket__websocket-api__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-client--org.eclipse.jetty.websocket__websocket-client__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-common--org.eclipse.jetty.websocket__websocket-common__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-server--org.eclipse.jetty.websocket__websocket-server__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-servlet--org.eclipse.jetty.websocket__websocket-servlet__9.4.42.v20210604.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.fusesource.leveldbjni--leveldbjni-all--org.fusesource.leveldbjni__leveldbjni-all__1.8.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2--hk2-api--org.glassfish.hk2__hk2-api__2.6.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2--hk2-locator--org.glassfish.hk2__hk2-locator__2.6.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2--hk2-utils--org.glassfish.hk2__hk2-utils__2.6.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2--osgi-resource-locator--org.glassfish.hk2__osgi-resource-locator__1.0.3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2.external--aopalliance-repackaged--org.glassfish.hk2.external__aopalliance-repackaged__2.6.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2.external--jakarta.inject--org.glassfish.hk2.external__jakarta.inject__2.6.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.containers--jersey-container-servlet--org.glassfish.jersey.containers__jersey-container-servlet__2.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.containers--jersey-container-servlet-core--org.glassfish.jersey.containers__jersey-container-servlet-core__2.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.core--jersey-client--org.glassfish.jersey.core__jersey-client__2.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.core--jersey-common--org.glassfish.jersey.core__jersey-common__2.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.core--jersey-server--org.glassfish.jersey.core__jersey-server__2.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.inject--jersey-hk2--org.glassfish.jersey.inject__jersey-hk2__2.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.media--jersey-media-jaxb--org.glassfish.jersey.media__jersey-media-jaxb__2.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.hibernate.validator--hibernate-validator--org.hibernate.validator__hibernate-validator__6.1.0.Final.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.javassist--javassist--org.javassist__javassist__3.25.0-GA.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.jboss.logging--jboss-logging--org.jboss.logging__jboss-logging__3.3.2.Final.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.jdbi--jdbi--org.jdbi__jdbi__2.63.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.joda--joda-convert--org.joda__joda-convert__1.7.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.jodd--jodd-core--org.jodd__jodd-core__3.5.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-ast_2.12--org.json4s__json4s-ast_2.12__3.7.0-M5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-core_2.12--org.json4s__json4s-core_2.12__3.7.0-M5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-jackson_2.12--org.json4s__json4s-jackson_2.12__3.7.0-M5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-scalap_2.12--org.json4s__json4s-scalap_2.12__3.7.0-M5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.lz4--lz4-java--org.lz4__lz4-java__1.7.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.mariadb.jdbc--mariadb-java-client--org.mariadb.jdbc__mariadb-java-client__2.2.5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.objenesis--objenesis--org.objenesis__objenesis__2.5.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.postgresql--postgresql--org.postgresql__postgresql__42.1.4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.roaringbitmap--RoaringBitmap--org.roaringbitmap__RoaringBitmap__0.9.14.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.roaringbitmap--shims--org.roaringbitmap__shims__0.9.14.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.rocksdb--rocksdbjni--org.rocksdb__rocksdbjni__6.20.3.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.rosuda.REngine--REngine--org.rosuda.REngine__REngine__2.1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang--scala-compiler_2.12--org.scala-lang__scala-compiler__2.12.10.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang--scala-library_2.12--org.scala-lang__scala-library__2.12.10.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang--scala-reflect_2.12--org.scala-lang__scala-reflect__2.12.10.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang.modules--scala-collection-compat_2.12--org.scala-lang.modules__scala-collection-compat_2.12__2.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang.modules--scala-parser-combinators_2.12--org.scala-lang.modules__scala-parser-combinators_2.12__1.1.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang.modules--scala-xml_2.12--org.scala-lang.modules__scala-xml_2.12__1.2.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-sbt--test-interface--org.scala-sbt__test-interface__1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalacheck--scalacheck_2.12--org.scalacheck__scalacheck_2.12__1.14.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalactic--scalactic_2.12--org.scalactic__scalactic_2.12__3.0.8.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalanlp--breeze-macros_2.12--org.scalanlp__breeze-macros_2.12__1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalanlp--breeze_2.12--org.scalanlp__breeze_2.12__1.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalatest--scalatest_2.12--org.scalatest__scalatest_2.12__3.0.8.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.slf4j--jcl-over-slf4j--org.slf4j__jcl-over-slf4j__1.7.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.slf4j--jul-to-slf4j--org.slf4j__jul-to-slf4j__1.7.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.slf4j--slf4j-api--org.slf4j__slf4j-api__1.7.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.slf4j--slf4j-log4j12--org.slf4j__slf4j-log4j12__1.7.30.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.spark-project.spark--unused--org.spark-project.spark__unused__1.0.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.springframework--spring-core--org.springframework__spring-core__4.1.4.RELEASE.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.springframework--spring-test--org.springframework__spring-test__4.1.4.RELEASE.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.threeten--threeten-extra--org.threeten__threeten-extra__1.5.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.tukaani--xz--org.tukaani__xz__1.5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--algebra_2.12--org.typelevel__algebra_2.12__2.0.0-M2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--cats-kernel_2.12--org.typelevel__cats-kernel_2.12__2.0.0-M4.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--machinist_2.12--org.typelevel__machinist_2.12__0.6.8.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--macro-compat_2.12--org.typelevel__macro-compat_2.12__1.1.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--spire-macros_2.12--org.typelevel__spire-macros_2.12__0.17.0-M1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--spire-platform_2.12--org.typelevel__spire-platform_2.12__0.17.0-M1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--spire-util_2.12--org.typelevel__spire-util_2.12__0.17.0-M1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--spire_2.12--org.typelevel__spire_2.12__0.17.0-M1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.wildfly.openssl--wildfly-openssl--org.wildfly.openssl__wildfly-openssl__1.0.7.Final.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.xerial--sqlite-jdbc--org.xerial__sqlite-jdbc__3.8.11.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.xerial.snappy--snappy-java--org.xerial.snappy__snappy-java__1.1.8.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.yaml--snakeyaml--org.yaml__snakeyaml__1.24.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--oro--oro--oro__oro__2.0.8.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--pl.edu.icm--JLargeArrays--pl.edu.icm__JLargeArrays__1.5.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--software.amazon.ion--ion-java--software.amazon.ion__ion-java__1.0.2.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--stax--stax-api--stax__stax-api__1.0.1.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--xmlenc--xmlenc--xmlenc__xmlenc__0.52.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--zookeeper-3.4--liball_deps_2.12.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--zookeeper-3.4--org.apache.yetus--audience-annotations--org.apache.yetus__audience-annotations__0.5.0.jar:/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--zookeeper-3.4--org.apache.zookeeper--zookeeper--org.apache.zookeeper__zookeeper__3.4.14.jar:/databricks/jars/----workspace_spark_3_1--mllib--libmllib_resources.jar:/databricks/jars/----workspace_spark_3_1--mllib--mllib_2.12_deploy_shaded.jar:/databricks/jars/----workspace_spark_3_1--mllib--org.jpmml__pmml-model__1.4.8_shaded-for-mllib.jar:/databricks/jars/----workspace_spark_3_1--mllib-local--mllib-local-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--patched-hive-with-glue--hive-12679-patch_deploy.jar:/databricks/jars/----workspace_spark_3_1--patched-hive-with-glue--hive-common_shaded.jar:/databricks/jars/----workspace_spark_3_1--patched-hive-with-glue--hive-exec-core_shaded.jar:/databricks/jars/----workspace_spark_3_1--patched-hive-with-glue--hive-metastore_shaded.jar:/databricks/jars/----workspace_spark_3_1--py4j--py4j-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--repl--repl-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--sql--catalyst--catalyst-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--sql--catalyst--libcatalyst_resources.jar:/databricks/jars/----workspace_spark_3_1--sql--catalyst--libspark-sql-parser-compiled.jar:/databricks/jars/----workspace_spark_3_1--sql--catalyst--proto-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--sql--core--core-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--sql--core--libcore_resources.jar:/databricks/jars/----workspace_spark_3_1--sql--core--proto_deploy.jar:/databricks/jars/----workspace_spark_3_1--sql--core--spark-sql-databricks-command-parser_java_out.srcjar:/databricks/jars/----workspace_spark_3_1--sql--hive--hive_2.12_deploy_shaded.jar:/databricks/jars/----workspace_spark_3_1--sql--hive--libhive_resources.jar:/databricks/jars/----workspace_spark_3_1--sql--hive--org.apache.commons__commons-pool2__2.6.2_shaded-for-hive.jar:/databricks/jars/----workspace_spark_3_1--sql--hive-thriftserver--hive-thriftserver-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--sql--hive-thriftserver--hive-thriftserver-protocol-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--streaming--libstreaming_resources.jar:/databricks/jars/----workspace_spark_3_1--streaming--streaming-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--third_party--bigquery-connector--fatJar-assembly-0.22.2-SNAPSHOT.jar_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--bigquery-connector--gcs-connector-hadoop2-2.0.0-shaded.jar_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.azure__azure-core-http-netty__1.6.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.azure__azure-core__1.8.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.azure__azure-identity__1.1.3_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.azure__azure-security-keyvault-keys__4.2.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml.jackson.dataformat__jackson-dataformat-xml__2.11.2_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml.jackson.datatype__jackson-datatype-jsr310__2.11.2_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml.jackson.module__jackson-module-jaxb-annotations__2.11.2_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml.woodstox__woodstox-core__6.2.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml__aalto-xml__1.0.0_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.github.stephenc.jcip__jcip-annotations__1.0-1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.madgag.spongycastle__core__1.54.0.0_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.microsoft.azure__msal4j-persistence-extension__1.0.0_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.microsoft.azure__msal4j__1.7.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.microsoft.sqlserver__mssql-jdbc__9.2.1.jre8_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.nimbusds__content-type__2.0_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.nimbusds__lang-tag__1.4.4_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.nimbusds__nimbus-jose-jwt__8.8_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.nimbusds__oauth2-oidc-sdk__7.1.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-buffer__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-codec-http2__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-codec-http__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-codec-socks__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-codec__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-common__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-handler-proxy__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-handler__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-resolver__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-tcnative-boringssl-static__2.0.31.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-transport-native-epoll-linux-x86_64__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-transport-native-kqueue-osx-x86_64__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-transport-native-unix-common__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-transport__4.1.51.Final_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.projectreactor.netty__reactor-netty__0.9.11.RELEASE_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.projectreactor__reactor-core__3.3.9.RELEASE_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--jakarta.activation__jakarta.activation-api__1.2.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--jakarta.xml.bind__jakarta.xml.bind-api__2.3.2_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--libmssql-shaded-meta-services_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--net.java.dev.jna__jna-platform__5.6.0_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--net.minidev__accessors-smart__1.2_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--net.minidev__json-smart__2.3_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.bouncycastle__bcprov-jdk15on__1.69_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.codehaus.woodstox__stax2-api__4.2.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.jetbrains__annotations__15.0_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-dom__2.1.4_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-jaxb__2.1.4_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-kdb__2.1.4_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-kdbx__2.1.4_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-simple__2.1.4_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2__2.1.4_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__database__2.1.4_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.nanohttpd__nanohttpd__2.3.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.ow2.asm__asm__5.0.4_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.reactivestreams__reactive-streams__1.0.3_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.simpleframework__simple-xml__2.7.1_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--stax__stax__1.2.0_shaded.jar:/databricks/jars/----workspace_spark_3_1--third_party--mssql--xpp3__xpp3__1.1.3.3_shaded.jar:/databricks/jars/----workspace_spark_3_1--vendor--avro--avro_2.12_deploy_shaded.jar:/databricks/jars/----workspace_spark_3_1--vendor--avro--io.confluent__common-utils__4.0.0_shaded-for-avro.jar:/databricks/jars/----workspace_spark_3_1--vendor--avro--io.confluent__kafka-schema-registry-client__4.0.0_shaded-for-avro.jar:/databricks/jars/----workspace_spark_3_1--vendor--avro--libavro_resources_shaded-for-avro.jar:/databricks/jars/----workspace_spark_3_1--vendor--file-notification-common--file-notification-common-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--vendor--file-notification-common--libfile-notification-common_resources.jar:/databricks/jars/----workspace_spark_3_1--vendor--iceberg--iceberg-converter_2.12_deploy_shaded.jar:/databricks/jars/----workspace_spark_3_1--vendor--iceberg--iceberg-spark3-runtime_shaded_shaded-for-iceberg-converter.jar:/databricks/jars/----workspace_spark_3_1--vendor--kafka-0-10-token-provider--libkafka-0-10-token-provider_resources.jar:/databricks/jars/----workspace_spark_3_1--vendor--kafka-0-10-token-provider-unshaded_2.12_deploy_shaded-for-kafka-0-10.jar:/databricks/jars/----workspace_spark_3_1--vendor--kafka-0-10_2.12_deploy_shaded.jar:/databricks/jars/----workspace_spark_3_1--vendor--libkafka-0-10-resources_shaded-for-kafka-0-10.jar:/databricks/jars/----workspace_spark_3_1--vendor--org.apache.kafka__kafka-clients__2.6.0_shaded-for-kafka-0-10.jar:/databricks/jars/----workspace_spark_3_1--vendor--redshift--redshift-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--vendor--spark-ganglia-lgpl--libmetrics-ganglia.jar:/databricks/jars/----workspace_spark_3_1--vendor--spark-ganglia-lgpl--spark-ganglia-lgpl-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--vendor--sql-aws-connectors--libsql-aws-connectors_resources.jar:/databricks/jars/----workspace_spark_3_1--vendor--sql-aws-connectors--sql-aws-connectors-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--vendor--sql-azure-connectors--libsql-azure-connectors_resources.jar:/databricks/jars/----workspace_spark_3_1--vendor--sql-azure-connectors--sql-azure-connectors-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--vendor--sql-dw--sql-dw-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/----workspace_spark_3_1--vendor--sql-gcp-connectors--libsql-gcp-connectors_resources.jar:/databricks/jars/----workspace_spark_3_1--vendor--sql-gcp-connectors--sql-gcp-connectors-hive-2.3__hadoop-2.7_2.12_deploy.jar:/databricks/jars/acl--api--helpers--helpers-spark_3.1_2.12_deploy.jar:/databricks/jars/acl--auth--api--api-spark_3.1_2.12_deploy.jar:/databricks/jars/api--common--workspace-spark_3.1_2.12_deploy.jar:/databricks/jars/api--rpc--rpc_parser-spark_3.1_2.12_deploy.jar:/databricks/jars/api-base--api-base-spark_3.1_2.12_deploy.jar:/databricks/jars/api-base--api-base_java-spark_3.1_2.12_deploy.jar:/databricks/jars/central--api--api-spark_3.1_2.12_deploy.jar:/databricks/jars/chauffeur-api--api--endpoints--endpoints-spark_3.1_2.12_deploy.jar:/databricks/jars/chauffeur-api--chauffeur-api-spark_3.1_2.12_deploy.jar:/databricks/jars/cmdexec--api--api-experimental-proto-spark_3.1_2.12_deploy.jar:/databricks/jars/cmdexec--api--api-experimental-spark_3.1_2.12_deploy.jar:/databricks/jars/common--client--client-spark_3.1_2.12_deploy.jar:/databricks/jars/common--cloudstorage--presigned-url-spark_3.1_2.12_deploy.jar:/databricks/jars/common--common-spark_3.1_2.12_deploy.jar:/databricks/jars/common--credentials--credentials-spark_3.1_2.12_deploy.jar:/databricks/jars/common--crypto-providers--amazon-corretto-crypto-provider--libamazon-corretto-crypto-provider.jar:/databricks/jars/common--hadoop--hadoop-spark_3.1_2.12_deploy.jar:/databricks/jars/common--java-flight-recorder--java-flight-recorder-spark_3.1_2.12_deploy.jar:/databricks/jars/common--jetty--client--client-spark_3.1_2.12_deploy.jar:/databricks/jars/common--jupyter-utils--jupyter_utils-spark_3.1_2.12_deploy.jar:/databricks/jars/common--lazy--lazy-spark_3.1_2.12_deploy.jar:/databricks/jars/common--libcommon_resources.jar:/databricks/jars/common--network--network-spark_3.1_2.12_deploy.jar:/databricks/jars/common--node-types--node-types-spark_3.1_2.12_deploy.jar:/databricks/jars/common--path--path-spark_3.1_2.12_deploy.jar:/databricks/jars/common--pricing--pricing-spark_3.1_2.12_deploy.jar:/databricks/jars/common--rate-limiter--rate-limiter-spark_3.1_2.12_deploy.jar:/databricks/jars/common--reflection--reflection-spark_3.1_2.12_deploy.jar:/databricks/jars/common--storage--storage-spark_3.1_2.12_deploy.jar:/databricks/jars/common--storage-driver-utils--storage_driver_utils-spark_3.1_2.12_deploy.jar:/databricks/jars/common--tracing--tracing-spark_3.1_2.12_deploy.jar:/databricks/jars/common--util--locks-spark_3.1_2.12_deploy.jar:/databricks/jars/credentials-manager--api--api-spark_3.1_2.12_deploy.jar:/databricks/jars/daemon--data--client--client-spark_3.1_2.12_deploy.jar:/databricks/jars/daemon--data--client--conf--conf-spark_3.1_2.12_deploy.jar:/databricks/jars/daemon--data--client--utils-spark_3.1_2.12_deploy.jar:/databricks/jars/daemon--data--data-common--data-common-spark_3.1_2.12_deploy.jar:/databricks/jars/dbfs--exceptions--exceptions-spark_3.1_2.12_deploy.jar:/databricks/jars/dbfs--utils--dbfs-utils-spark_3.1_2.12_deploy.jar:/databricks/jars/extern--acl--auth--auth-spark_3.1_2.12_deploy.jar:/databricks/jars/extern--extern-spark_3.1_2.12_deploy.jar:/databricks/jars/extern--libaws-regions.jar:/databricks/jars/jsonutil--jsonutil-spark_3.1_2.12_deploy.jar:/databricks/jars/libraries--api--managedLibraries--managedLibraries-spark_3.1_2.12_deploy.jar:/databricks/jars/libraries--api--typemappers--typemappers-spark_3.1_2.12_deploy.jar:/databricks/jars/libraries--libraries-spark_3.1_2.12_deploy.jar:/databricks/jars/logging--log4j-mod--log4j-mod-spark_3.1_2.12_deploy.jar:/databricks/jars/logging--utils--logging-utils-spark_3.1_2.12_deploy.jar:/databricks/jars/macros--ratelimitedlogger--ratelimitedlogger-spark_3.1_2.12_deploy.jar:/databricks/jars/macros--sourcecode--sourcecode-spark_3.1_2.12_deploy.jar:/databricks/jars/managed-catalog--api--api-spark_3.1_2.12_deploy.jar:/databricks/jars/managed-catalog--api--api_common-spark_3.1_2.12_deploy.jar:/databricks/jars/maven-trees--amazon-corretto-crypto-provider--software.amazon.cryptools--AmazonCorrettoCryptoProvider-linux-x86_64--software.amazon.cryptools__AmazonCorrettoCryptoProvider-linux-x86_64__1.4.0.jar:/databricks/jars/s3--s3-spark_3.1_2.12_deploy.jar:/databricks/jars/s3commit--api--api-spark_3.1_2.12_deploy.jar:/databricks/jars/s3commit--client--client-spark_3.1_2.12_deploy.jar:/databricks/jars/s3commit--common--common-spark_3.1_2.12_deploy.jar:/databricks/jars/secret-manager--api--api-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--command--command-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--common--spark-common-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--common-utils--utils-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--conf-reader--conf-reader_lib-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--dbutils--dbutils-api-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--driver--antlr--parser-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--driver--common--driver-common-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--driver--dbutils_impl--data--rendering--rendering-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--driver--display--display-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--driver--driver-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--driver--events-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--driver--ml--ml-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--driver--secret-redaction-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--driver--spark--resources-resources.jar:/databricks/jars/spark--sql-extension--sql-extension-spark_3.1_2.12_deploy.jar:/databricks/jars/spark--versions--3.1--shim_2.12_deploy.jar:/databricks/jars/spark--versions--3.1--spark_2.12_deploy.jar:/databricks/jars/sqlgateway--common--endpoint_id-spark_3.1_2.12_deploy.jar:/databricks/jars/sqlgateway--history--api--api-spark_3.1_2.12_deploy.jar:/databricks/jars/sqlgateway--history--api--client-spark_3.1_2.12_deploy.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.fasterxml.jackson.core_jackson-annotations_com.fasterxml.jackson.core__jackson-annotations__2.12.3_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.fasterxml.jackson.core_jackson-core_com.fasterxml.jackson.core__jackson-core__2.12.3_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.fasterxml.jackson.core_jackson-databind_com.fasterxml.jackson.core__jackson-databind__2.12.3_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.android_annotations_com.google.android__annotations__4.1.1.4_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.api.grpc_proto-google-common-protos_com.google.api.grpc__proto-google-common-protos__2.0.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.code.findbugs_jsr305_com.google.code.findbugs__jsr305__3.0.2_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.code.gson_gson_com.google.code.gson__gson__2.8.6_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.errorprone_error_prone_annotations_com.google.errorprone__error_prone_annotations__2.4.0_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.guava_failureaccess_com.google.guava__failureaccess__1.0.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.guava_guava_com.google.guava__guava__30.1-android_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.guava_listenablefuture_com.google.guava__listenablefuture__9999.0-empty-to-avoid-conflict-with-guava_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.j2objc_j2objc-annotations_com.google.j2objc__j2objc-annotations__1.3_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.protobuf_protobuf-java-util_com.google.protobuf__protobuf-java-util__3.12.0_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.protobuf_protobuf-java_com.google.protobuf__protobuf-java__3.14.0_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.linecorp.armeria_armeria-grpc-protocol_com.linecorp.armeria__armeria-grpc-protocol__1.9.2_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.linecorp.armeria_armeria-grpc_com.linecorp.armeria__armeria-grpc__1.9.2_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_com.linecorp.armeria_armeria_com.linecorp.armeria__armeria__1.9.2_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-api_io.grpc__grpc-api__1.38.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-context_io.grpc__grpc-context__1.38.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-core_io.grpc__grpc-core__1.38.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-protobuf-lite_io.grpc__grpc-protobuf-lite__1.38.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-protobuf_io.grpc__grpc-protobuf__1.38.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-services_io.grpc__grpc-services__1.38.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-stub_io.grpc__grpc-stub__1.38.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.micrometer_micrometer-core_io.micrometer__micrometer-core__1.7.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-buffer_io.netty__netty-buffer__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-dns_io.netty__netty-codec-dns__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-haproxy_io.netty__netty-codec-haproxy__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-http2_io.netty__netty-codec-http2__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-http_io.netty__netty-codec-http__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-socks_io.netty__netty-codec-socks__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec_io.netty__netty-codec__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-common_io.netty__netty-common__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-handler-proxy_io.netty__netty-handler-proxy__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-handler_io.netty__netty-handler__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-resolver-dns_io.netty__netty-resolver-dns__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-resolver_io.netty__netty-resolver__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-transport-native-unix-common-linux-x86_64_io.netty__netty-transport-native-unix-common-linux-x86_64__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-transport_io.netty__netty-transport__4.1.65.Final_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_io.perfmark_perfmark-api_io.perfmark__perfmark-api__0.23.0_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_liball_deps_2.12_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_net.bytebuddy_byte-buddy_net.bytebuddy__byte-buddy__1.10.19_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_org.checkerframework_checker-compat-qual_org.checkerframework__checker-compat-qual__2.5.5_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_org.codehaus.mojo_animal-sniffer-annotations_org.codehaus.mojo__animal-sniffer-annotations__1.19_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_org.curioswitch.curiostack_protobuf-jackson_org.curioswitch.curiostack__protobuf-jackson__1.2.0_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_org.hdrhistogram_HdrHistogram_org.hdrhistogram__HdrHistogram__2.1.12_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_org.joda_joda-convert_org.joda__joda-convert__2.2.1_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_org.latencyutils_LatencyUtils_org.latencyutils__LatencyUtils__2.0.3_shaded.jar:/databricks/jars/third_party--armeria--maven-trees_armeria_org.reactivestreams_reactive-streams_org.reactivestreams__reactive-streams__1.0.3_shaded.jar:/databricks/jars/third_party--armeria--service_discovery-resources.jar:/databricks/jars/third_party--azure--com.fasterxml.jackson.core__jackson-core__2.7.2_shaded.jar:/databricks/jars/third_party--azure--com.microsoft.azure__azure-client-runtime__1.7.12_container_shaded.jar:/databricks/jars/third_party--azure--com.microsoft.azure__azure-keyvault-core__1.0.0_shaded.jar:/databricks/jars/third_party--azure--com.microsoft.azure__azure-storage__8.6.4_shaded.jar:/databricks/jars/third_party--azure--com.microsoft.rest__client-runtime__1.7.12_container_shaded.jar:/databricks/jars/third_party--azure--org.apache.commons__commons-lang3__3.4_shaded.jar:/databricks/jars/third_party--datalake--datalake-spark_3.1_2.12_deploy.jar:/databricks/jars/third_party--dropwizard-metrics-log4j-v3.2.6--metrics-log4j-spark_3.1_2.12_deploy.jar:/databricks/jars/third_party--gcp-java--animal-sniffer-annotations_shaded.jar:/databricks/jars/third_party--gcp-java--api-common_shaded.jar:/databricks/jars/third_party--gcp-java--auto-value-annotations_shaded.jar:/databricks/jars/third_party--gcp-java--checker-compat-qual_shaded.jar:/databricks/jars/third_party--gcp-java--commons-codec_shaded.jar:/databricks/jars/third_party--gcp-java--commons-lang3_shaded.jar:/databricks/jars/third_party--gcp-java--commons-logging_shaded.jar:/databricks/jars/third_party--gcp-java--error_prone_annotations_shaded.jar:/databricks/jars/third_party--gcp-java--failureaccess_shaded.jar:/databricks/jars/third_party--gcp-java--gax-grpc_shaded.jar:/databricks/jars/third_party--gcp-java--gax-httpjson_shaded.jar:/databricks/jars/third_party--gcp-java--gax_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-client_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-cloudkms_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-cloudresourcemanager_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-compute_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-container_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-deploymentmanager_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-dns_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-iam_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-networkmanagement_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-servicecontrol_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-serviceusage_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-storage_shaded.jar:/databricks/jars/third_party--gcp-java--google-api-services-sts_shaded.jar:/databricks/jars/third_party--gcp-java--google-auth-library-credentials_shaded.jar:/databricks/jars/third_party--gcp-java--google-auth-library-oauth2-http_shaded.jar:/databricks/jars/third_party--gcp-java--google-cloud-core-http_shaded.jar:/databricks/jars/third_party--gcp-java--google-cloud-core_shaded.jar:/databricks/jars/third_party--gcp-java--google-cloud-notification_shaded.jar:/databricks/jars/third_party--gcp-java--google-cloud-pubsub_shaded.jar:/databricks/jars/third_party--gcp-java--google-cloud-storage_shaded.jar:/databricks/jars/third_party--gcp-java--google-http-client-apache-v2_shaded.jar:/databricks/jars/third_party--gcp-java--google-http-client-appengine_shaded.jar:/databricks/jars/third_party--gcp-java--google-http-client-jackson2_shaded.jar:/databricks/jars/third_party--gcp-java--google-http-client_shaded.jar:/databricks/jars/third_party--gcp-java--google-oauth-client_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-alts_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-api_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-auth_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-context_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-core_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-grpclb_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-netty-shaded_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-protobuf-lite_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-protobuf_shaded.jar:/databricks/jars/third_party--gcp-java--grpc-stub_shaded.jar:/databricks/jars/third_party--gcp-java--gson_shaded.jar:/databricks/jars/third_party--gcp-java--guava_shaded.jar:/databricks/jars/third_party--gcp-java--httpclient_shaded.jar:/databricks/jars/third_party--gcp-java--httpcore_shaded.jar:/databricks/jars/third_party--gcp-java--j2objc-annotations_shaded.jar:/databricks/jars/third_party--gcp-java--jackson-core_shaded.jar:/databricks/jars/third_party--gcp-java--javax.annotation-api_shaded.jar:/databricks/jars/third_party--gcp-java--jsr305_shaded.jar:/databricks/jars/third_party--gcp-java--listenablefuture_shaded.jar:/databricks/jars/third_party--gcp-java--opencensus-api_shaded.jar:/databricks/jars/third_party--gcp-java--opencensus-contrib-http-util_shaded.jar:/databricks/jars/third_party--gcp-java--perfmark-api_shaded.jar:/databricks/jars/third_party--gcp-java--proto-google-cloud-pubsub-v1_shaded.jar:/databricks/jars/third_party--gcp-java--proto-google-common-protos_shaded.jar:/databricks/jars/third_party--gcp-java--proto-google-iam-v1_shaded.jar:/databricks/jars/third_party--gcp-java--protobuf-java-util_shaded.jar:/databricks/jars/third_party--gcp-java--protobuf-java_shaded.jar:/databricks/jars/third_party--gcp-java--threetenbp_shaded.jar:/databricks/jars/third_party--gcs-private--animal-sniffer-annotations_shaded.jar:/databricks/jars/third_party--gcs-private--annotations_shaded.jar:/databricks/jars/third_party--gcs-private--auto-value-annotations_shaded.jar:/databricks/jars/third_party--gcs-private--checker-compat-qual_shaded.jar:/databricks/jars/third_party--gcs-private--checker-qual_shaded.jar:/databricks/jars/third_party--gcs-private--commons-codec_shaded.jar:/databricks/jars/third_party--gcs-private--commons-logging_shaded.jar:/databricks/jars/third_party--gcs-private--conscrypt-openjdk-uber_shaded.jar:/databricks/jars/third_party--gcs-private--error_prone_annotations_shaded.jar:/databricks/jars/third_party--gcs-private--failureaccess_shaded.jar:/databricks/jars/third_party--gcs-private--flogger-system-backend_shaded.jar:/databricks/jars/third_party--gcs-private--flogger_shaded.jar:/databricks/jars/third_party--gcs-private--gcs-connector-hadoop2_shaded.jar:/databricks/jars/third_party--gcs-private--gcs-shaded-jetty9-hadoop1_2.12_deploy.jar:/databricks/jars/third_party--gcs-private--gcs-shaded-spark_3.1_2.12_deploy.jar:/databricks/jars/third_party--gcs-private--gcsio-hadoop2_shaded.jar:/databricks/jars/third_party--gcs-private--google-api-client-jackson2_shaded.jar:/databricks/jars/third_party--gcs-private--google-api-client_shaded.jar:/databricks/jars/third_party--gcs-private--google-api-services-iamcredentials_shaded.jar:/databricks/jars/third_party--gcs-private--google-api-services-storage_shaded.jar:/databricks/jars/third_party--gcs-private--google-auth-library-credentials_shaded.jar:/databricks/jars/third_party--gcs-private--google-auth-library-oauth2-http_shaded.jar:/databricks/jars/third_party--gcs-private--google-extensions_shaded.jar:/databricks/jars/third_party--gcs-private--google-http-client-apache-v2_shaded.jar:/databricks/jars/third_party--gcs-private--google-http-client-gson_shaded.jar:/databricks/jars/third_party--gcs-private--google-http-client-jackson2_shaded.jar:/databricks/jars/third_party--gcs-private--google-http-client_shaded.jar:/databricks/jars/third_party--gcs-private--google-oauth-client_shaded.jar:/databricks/jars/third_party--gcs-private--grpc-alts_shaded.jar:/databricks/jars/third_party--gcs-private--grpc-api_shaded.jar:/databricks/jars/third_party--gcs-private--grpc-auth_shaded.jar:/databricks/jars/third_party--gcs-private--grpc-context_shaded.jar:/databricks/jars/third_party--gcs-private--grpc-core_shaded.jar:/databricks/jars/third_party--gcs-private--grpc-netty-shaded_shaded.jar:/databricks/jars/third_party--gcs-private--grpc-protobuf-lite_shaded.jar:/databricks/jars/third_party--gcs-private--grpc-protobuf_shaded.jar:/databricks/jars/third_party--gcs-private--grpc-stub_shaded.jar:/databricks/jars/third_party--gcs-private--gson_shaded.jar:/databricks/jars/third_party--gcs-private--guava_shaded.jar:/databricks/jars/third_party--gcs-private--httpclient_shaded.jar:/databricks/jars/third_party--gcs-private--httpcore_shaded.jar:/databricks/jars/third_party--gcs-private--j2objc-annotations_shaded.jar:/databricks/jars/third_party--gcs-private--jackson-core_shaded.jar:/databricks/jars/third_party--gcs-private--libmeta-services.jar:/databricks/jars/third_party--gcs-private--listenablefuture_shaded.jar:/databricks/jars/third_party--gcs-private--opencensus-api_shaded.jar:/databricks/jars/third_party--gcs-private--opencensus-contrib-http-util_shaded.jar:/databricks/jars/third_party--gcs-private--perfmark-api_shaded.jar:/databricks/jars/third_party--gcs-private--proto-google-common-protos_shaded.jar:/databricks/jars/third_party--gcs-private--proto-google-iam-v1_shaded.jar:/databricks/jars/third_party--gcs-private--protobuf-java-util_shaded.jar:/databricks/jars/third_party--gcs-private--protobuf-java_shaded.jar:/databricks/jars/third_party--gcs-private--util-hadoop-hadoop2_shaded.jar:/databricks/jars/third_party--gcs-private--util-hadoop2_shaded.jar:/databricks/jars/third_party--hadoop--hadoop-tools--hadoop-aws--lib-spark_3.1_2.12_deploy_shaded.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--aopalliance__aopalliance__1.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.core__jackson-annotations__2.7.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.core__jackson-core__2.7.2_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.core__jackson-core__2.7.2_2.12_shaded_20180920_b33d810_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.core__jackson-databind__2.7.2_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.datatype__jackson-datatype-joda__2.7.2_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.google.code.findbugs__jsr305__1.3.9_2.12_shaded_20180920_b33d810_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.google.guava__guava__11.0.2_2.12_shaded_20180920_b33d810_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.google.guava__guava__16.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.google.inject__guice__3.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-annotations__1.2.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-keyvault-core__1.0.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-keyvault-core__1.0.0_2.12_shaded_20180920_b33d810_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-storage__7.0.0_2.12_shaded_20180920_b33d810_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-storage__8.6.4_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.rest__client-runtime__1.1.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.okhttp3__logging-interceptor__3.3.1_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.okhttp3__okhttp-urlconnection__3.3.1_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.okhttp3__okhttp__3.3.1_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.okio__okio__1.8.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.retrofit2__adapter-rxjava__2.1.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.retrofit2__converter-jackson__2.1.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.retrofit2__retrofit__2.1.0_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.sun.xml.bind__jaxb-impl__2.2.3-1_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--commons-codec__commons-codec__1.9_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--commons-codec__commons-codec__1.9_2.12_shaded_20180920_b33d810_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--commons-logging__commons-logging__1.2_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--commons-logging__commons-logging__1.2_2.12_shaded_20180920_b33d810_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--hadoop-azure-2.7.3-abfs-external-20180625_3682417-spark_3.1_2.12_deploy_shaded.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--hadoop-azure-2.7.3-abfs-external-20180920_b33d810-spark_3.1_2.12_deploy_shaded.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--io.netty__netty-all__4.0.52.Final_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--io.reactivex__rxjava__1.2.4_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--javax.activation__activation__1.1_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--javax.inject__javax.inject__1_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--javax.xml.bind__jaxb-api__2.2.2_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--javax.xml.stream__stax-api__1.0-2_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--joda-time__joda-time__2.4_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.htrace__htrace-core__3.1.0-incubating_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.httpcomponents__httpclient__4.5.2_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.httpcomponents__httpclient__4.5.2_2.12_shaded_20180920_b33d810_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.httpcomponents__httpcore__4.4.4_2.12_shaded_20180625_3682417_spark_3.1.jar:/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.httpcomponents__httpcore__4.4.4_2.12_shaded_20180920_b33d810_spark_3.1.jar:/databricks/jars/third_party--hadoop_azure_abfs--hadoop-tools--hadoop-azure--lib-spark_3.1_2.12_deploy.jar_shaded.jar:/databricks/jars/third_party--jackson--guava_only_shaded.jar:/databricks/jars/third_party--jackson--jackson-module-scala-shaded_2.12_deploy.jar:/databricks/jars/third_party--jackson--jsr305_only_shaded.jar:/databricks/jars/third_party--jackson--paranamer_only_shaded.jar:/databricks/jars/third_party--jetty-client--jetty-client_shaded.jar:/databricks/jars/third_party--jetty-client--jetty-http_shaded.jar:/databricks/jars/third_party--jetty-client--jetty-io_shaded.jar:/databricks/jars/third_party--jetty-client--jetty-util_shaded.jar:/databricks/jars/third_party--jsonwebtoken--jackson-annotations_shaded.jar:/databricks/jars/third_party--jsonwebtoken--jackson-core_shaded.jar:/databricks/jars/third_party--jsonwebtoken--jackson-databind_shaded.jar:/databricks/jars/third_party--jsonwebtoken--jjwt-api_shaded.jar:/databricks/jars/third_party--jsonwebtoken--jjwt-impl_shaded.jar:/databricks/jars/third_party--jsonwebtoken--jjwt-jackson_shaded.jar:/databricks/jars/third_party--opencensus-shaded--com.google.code.findbugs__jsr305__3.0.2_shaded.jar:/databricks/jars/third_party--opencensus-shaded--com.google.code.gson__gson__2.8.2_shaded.jar:/databricks/jars/third_party--opencensus-shaded--com.google.errorprone__error_prone_annotations__2.1.3_shaded.jar:/databricks/jars/third_party--opencensus-shaded--com.google.guava__guava__26.0-android_shaded.jar:/databricks/jars/third_party--opencensus-shaded--com.google.j2objc__j2objc-annotations__1.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--com.lmax__disruptor__3.4.2_shaded.jar:/databricks/jars/third_party--opencensus-shaded--com.squareup.okhttp3__okhttp__3.9.0_shaded.jar:/databricks/jars/third_party--opencensus-shaded--com.squareup.okio__okio__1.13.0_shaded.jar:/databricks/jars/third_party--opencensus-shaded--commons-codec__commons-codec__1.9_shaded.jar:/databricks/jars/third_party--opencensus-shaded--commons-logging__commons-logging__1.2_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.grpc__grpc-context__1.19.0_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.jaegertracing__jaeger-client__0.33.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.jaegertracing__jaeger-core__0.33.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.jaegertracing__jaeger-thrift__0.33.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.jaegertracing__jaeger-tracerresolver__0.33.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-api__0.22.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-exporter-trace-jaeger__0.22.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-exporter-trace-util__0.22.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-impl-core__0.22.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-impl__0.22.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.opentracing.contrib__opentracing-tracerresolver__0.1.5_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.opentracing__opentracing-api__0.31.0_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.opentracing__opentracing-noop__0.31.0_shaded.jar:/databricks/jars/third_party--opencensus-shaded--io.opentracing__opentracing-util__0.31.0_shaded.jar:/databricks/jars/third_party--opencensus-shaded--org.apache.httpcomponents__httpclient__4.4.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--org.apache.httpcomponents__httpcore__4.4.1_shaded.jar:/databricks/jars/third_party--opencensus-shaded--org.apache.thrift__libthrift__0.11.0_shaded.jar:/databricks/jars/third_party--opencensus-shaded--org.checkerframework__checker-compat-qual__2.5.2_shaded.jar:/databricks/jars/third_party--opencensus-shaded--org.codehaus.mojo__animal-sniffer-annotations__1.14_shaded.jar:/databricks/jars/third_party--zeromq--jeromq_shaded.jar:/databricks/jars/third_party--zeromq--jnacl_shaded.jar:/databricks/jars/utils--process_utils-spark_3.1_2.12_deploy.jar:/databricks/jars/workflow--workflow-spark_3.1_2.12_deploy.jar","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"},"Hadoop Properties":{"yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"200","mapreduce.jobtracker.jobhistory.task.numberprogresssplits":"12","mapreduce.tasktracker.healthchecker.script.timeout":"600000","fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","fs.tachyon.impl":"tachyon.hadoop.TFS","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"*********(redacted)","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","mapreduce.tasktracker.local.dir.minspacekill":"0","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.s3.block.size":"67108864","hadoop.registry.secure":"false","fs.s3a.fast.buffer.size":"1048576","fs.s3.buffer.dir":"${hadoop.tmp.dir}/s3","mapreduce.job.acl-view-job":" ","mapreduce.jobhistory.loadedjobs.cache.size":"5","mapreduce.jobtracker.persist.jobstatus.hours":"1","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","mapreduce.tasktracker.http.threads":"40","fs.azure.skip.metrics":"true","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","mapreduce.jobtracker.jobhistory.block.size":"3145728","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","spark.driverproxy.customHeadersToProperties":"*********(redacted)","fs.cpfs-s3.impl":"*********(redacted)","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","mapreduce.jobtracker.retiredjobs.cache.size":"1000","mapreduce.am.max-attempts":"2","fs.trash.checkpoint.interval":"0","yarn.nodemanager.container-monitor.interval-ms":"3000","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","fs.gs.impl.disable.cache":"true","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"10000","mapreduce.map.skip.proc.count.autoincr":"true","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","ipc.client.fallback-to-simple-auth-allowed":"false","mapreduce.tasktracker.taskcontroller":"org.apache.hadoop.mapred.DefaultTaskController","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","mapreduce.jobtracker.tasktracker.maxblacklists":"4","yarn.nodemanager.docker-container-executor.exec-name":"/usr/bin/docker","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","mapred.child.java.opts":"-Xmx200m","hadoop.common.configuration.version":"0.23.0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","hive.hmshandler.retry.interval":"2000","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.task.profile.reduces":"0-2","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"dbfs:///","fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","fs.har.impl.disable.cache":"true","io.compression.codec.bzip2.library":"system-native","hive.warehouse.subdir.inherit.perms":"false","fs.fcfs-s3a.impl":"*********(redacted)","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","databricks.dbfs.client.version":"v2","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","parquet.page.write-checksum.enabled":"true","ipc.server.log.slow.rpc":"false","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"8192","fs.s3a.fast.upload.active.blocks":"32","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","mapreduce.jobtracker.expire.trackers.interval":"600000","yarn.resourcemanager.nodemanager-connect-retries":"10","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","mapreduce.tasktracker.outofband.heartbeat":"false","ha.failover-controller.new-active.rpc-timeout.ms":"60000","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","ha.zookeeper.session-timeout.ms":"5000","tfile.io.chunk.size":"1048576","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","fs.cpfs-abfss.impl":"*********(redacted)","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","mapreduce.jobtracker.persist.jobstatus.dir":"/jobtracker/jobsInfo","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","io.skip.checksum.errors":"false","fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","fs.s3a.connection.timeout":"50000","fs.cpfs-abfss.impl.disable.cache":"true","mapreduce.job.max.split.locations":"10","hadoop.registry.zk.session.timeout.ms":"60000","mapreduce.shuffle.transfer.buffer.size":"131072","fs.dbs3n.impl":"com.databricks.backend.daemon.data.client.DBS3N","yarn.timeline-service.client.retry-interval-ms":"1000","mapreduce.jobtracker.taskcache.levels":"2","yarn.http.policy":"HTTP_ONLY","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","mapreduce.job.emit-timeline-data":"false","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","mapreduce.jobtracker.persist.jobstatus.active":"true","fs.abfss.impl.disable.cache":"true","yarn.dispatcher.drain-events.timeout":"300000","datanucleus.connectionPool.initialPoolSize":"0","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","mapreduce.shuffle.port":"13562","yarn.nodemanager.health-checker.interval-ms":"600000","mapreduce.tasktracker.report.address":"127.0.0.1:0","io.seqfile.lazydecompress":"true","ftp.blocksize":"67108864","mapreduce.jobtracker.instrumentation":"org.apache.hadoop.mapred.JobTrackerMetricsInst","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.delete.debug-delay-sec":"0","mapreduce.jobtracker.http.address":"0.0.0.0:50030","hive.server2.enable.doAs":"false","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","hadoop.security.groups.cache.secs":"300","yarn.resourcemanager.zk-retry-interval-ms":"1000","datanucleus.connectionPool.minPoolSize":"0","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","fs.cpfs-s3a.impl":"*********(redacted)","fs.mcfs-abfss.impl.disable.cache":"true","s3native.replication":"3","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","ha.zookeeper.parent-znode":"/hadoop-ha","spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","hive.server2.transport.mode":"http","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","hive.server2.idle.operation.timeout":"7200000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","fs.s3a.threads.core":"15","mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","fs.fcfs-s3a.impl.disable.cache":"true","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"136","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","mapreduce.cluster.temp.dir":"${hadoop.tmp.dir}/mapred/temp","s3.replication":"3","fs.adl.impl.disable.cache":"true","hadoop.tmp.dir":"/local_disk0/tmp","mapreduce.job.maps":"2","parquet.block.size.row.check.min":"10","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"104857600","fs.cpfs-adl.impl":"*********(redacted)","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","yarn.resourcemanager.zk-acl":"world:anyone:rwcda","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","fs.ftp.host.port":"21","ipc.ping.interval":"60000","fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","fs.dbfsartifacts.impl":"com.databricks.backend.daemon.data.client.DBFSV1","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","fs.dbfs.impl":"com.databricks.backend.daemon.data.client.DBFS","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","mapreduce.local.clientfactory.class.name":"org.apache.hadoop.mapred.LocalClientFactory","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"2000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","fs.df.interval":"60000","fs.s3.sleepTimeSeconds":"10","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","io.file.buffer.size":"65536","hadoop.work.around.non.threadsafe.getpwuid":"false","hadoop.security.group.mapping.ldap.search.attr.member":"member","fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","hadoop.security.random.device.file.path":"/dev/urandom","mapreduce.tasktracker.dns.interface":"default","hadoop.security.sensitive-config-keys":"*********(redacted)","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","mapreduce.reduce.log.level":"INFO","mapreduce.job.ubertask.enable":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","s3native.stream-buffer-size":"4096","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3n.multipart.uploads.enabled":"false","hadoop.security.crypto.buffer.size":"8192","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","s3.blocksize":"67108864","fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","javax.jdo.option.ConnectionUserName":"*********(redacted)","io.map.index.interval":"128","fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","mapreduce.tasktracker.instrumentation":"org.apache.hadoop.mapred.TaskTrackerMetricsInst","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","s3.stream-buffer-size":"4096","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","s3native.bytes-per-checksum":"512","hive.metastore.client.connect.retry.delay":"10","mapreduce.jobtracker.address":"local","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","parquet.memory.pool.ratio":"0.5","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","fs.dbfs2.impl":"com.databricks.backend.daemon.data.client.DBFSV2","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","hive.server2.session.check.interval":"60000","mapreduce.fileoutputcommitter.algorithm.version":"2","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","databricks.s3commit.client.sslTrustAll":"false","hive.server2.thrift.http.port":"10000","mapreduce.jobtracker.handler.count":"10","hadoop.http.authentication.type":"simple","mapreduce.jobtracker.taskscheduler":"org.apache.hadoop.mapred.JobQueueTaskScheduler","mapreduce.job.jvm.numtasks":"1","mapreduce.task.userlog.limit.kb":"*********(redacted)","yarn.resourcemanager.scheduler.monitor.enable":"false","fs.s3n.block.size":"67108864","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","mapreduce.jobtracker.staging.root.dir":"${hadoop.tmp.dir}/mapred/staging","mapreduce.jobtracker.jobhistory.lru.cache.size":"5","mapreduce.shuffle.listen.queue.size":"128","mapreduce.tasktracker.local.dir.minspacestart":"0","mapreduce.map.cpu.vcores":"1","hadoop.user.group.static.mapping.overrides":"*********(redacted)","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"*********(redacted)","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.app.mapreduce.client.job.max-retries":"0","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","fs.abfs.impl.disable.cache":"true","hadoop.registry.zk.retry.interval.ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","hadoop.ssl.server.conf":"ssl-server.xml","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.acl.enable":"false","mapreduce.tasktracker.map.tasks.maximum":"2","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"*********(redacted)","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"*********(redacted)","io.seqfile.sorter.recordlimit":"1000000","fs.automatic.close":"true","fs.s3n.multipart.copy.block.size":"5368709120","yarn.nodemanager.hostname":"0.0.0.0","yarn.resourcemanager.zk-timeout-ms":"10000","fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","hadoop.security.group.mapping.ldap.search.filter.user":"*********(redacted)","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","mapreduce.reduce.cpu.vcores":"1","fs.client.resolve.remote.symlinks":"true","mapreduce.jobtracker.restart.recover":"false","hadoop.http.cross-origin.allowed-origins":"*","yarn.timeline-service.generic-application-history.max-applications":"10000","mapreduce.tasktracker.reduce.tasks.maximum":"2","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","fs.s3a.fast.upload.default":"true","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"com.databricks.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","mapreduce.job.userlog.retain.hours":"*********(redacted)","fs.fcfs-s3n.impl":"*********(redacted)","yarn.scheduler.maximum-allocation-vcores":"32","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.jobtracker.heartbeats.in.second":"100","mapreduce.job.running.reduce.limit":"0","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","fs.wasbs.impl.disable.cache":"true","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","fs.s3a.max.total.tasks":"1000","mapreduce.tasktracker.healthchecker.interval":"60000","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.fast.upload":"true","fs.gs.outputstream.upload.chunk.size":"16777216","spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","fs.s3a.attempts.maximum":"10","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","hive.server2.thrift.http.cookie.auth.enabled":"false","mapreduce.map.log.level":"INFO","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","hadoop.registry.rm.enabled":"false","mapreduce.ifile.readahead.bytes":"4194304","mapreduce.tasktracker.tasks.sleeptimebeforesigkill":"5000","yarn.resourcemanager.fs.state-store.retry-policy-spec":"2000, 500","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"*********(redacted)","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","datanucleus.connectionPool.maxPoolSize":"1","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.resourcemanager.hostname":"0.0.0.0","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"1024","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","ipc.client.rpc-timeout.ms":"0","fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","fs.s3.maxRetries":"4","datanucleus.fixedDatastore":"true","mapreduce.input.lineinputformat.linespermap":"1","javax.jdo.option.ConnectionURL":"*********(redacted)","datanucleus.schema.autoCreateAll":"false","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3n.multipart.uploads.block.size":"67108864","mapreduce.tasktracker.http.address":"0.0.0.0:50060","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","parquet.page.verify-checksum.enabled":"true","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","fs.fcfs-s3n.impl.disable.cache":"true","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"*********(redacted)","mapreduce.reduce.memory.mb":"1024","s3native.client-write-packet-size":"65536","yarn.timeline-service.hostname":"0.0.0.0","javax.jdo.option.ConnectionDriverName":"org.mariadb.jdbc.Driver","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.log.retain-seconds":"10800","yarn.resourcemanager.keytab":"/etc/krb5.keytab","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","yarn.sharedcache.nm.uploader.thread-count":"20","mapreduce.shuffle.ssl.enabled":"false","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","javax.jdo.option.ConnectionPassword":"*********(redacted)","mapreduce.job.speculative.minimum-allowed-tasks":"10","parquet.page.size.check.estimate":"false","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","mapreduce.jobtracker.system.dir":"${hadoop.tmp.dir}/mapred/system","fs.fcfs-s3.impl.disable.cache":"true","mapreduce.tasktracker.taskmemorymanager.monitoringinterval":"5000","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","hive.hmshandler.retry.attempts":"10","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.nodemanager.resource.cpu-vcores":"8","mapreduce.job.reduces":"100","fs.s3a.multipart.size":"10485760","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","ha.health-monitor.sleep-after-disconnect.ms":"1000","s3.bytes-per-checksum":"512","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","fs.fcfs-s3.impl":"*********(redacted)","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"*********(redacted)","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","s3.client-write-packet-size":"65536","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","hive.server2.use.SSL":"true","hive.server2.keystore.password":"*********(redacted)","ipc.server.max.connections":"0","databricks.fs.perfMetrics.enable":"true","mapreduce.reduce.skip.proc.count.autoincr":"true","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","datanucleus.autoCreateSchema":"false","yarn.app.mapreduce.am.container.log.limit.kb":"0","s3native.blocksize":"67108864","ipc.client.connect.retry.interval":"1000","fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","hive.metastore.failure.retries":"30","mapreduce.jobhistory.cleaner.enable":"true","hive.server2.idle.session.timeout":"900000","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","mapreduce.tasktracker.dns.nameserver":"default","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","mapreduce.job.end-notification.retry.attempts":"0","yarn.resourcemanager.zk-num-retries":"1000","mapreduce.tasktracker.indexcache.mb":"10","hadoop.registry.zk.root":"/registry","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.cpfs-adl.impl.disable.cache":"true","ftp.client-write-packet-size":"65536","mapreduce.jobtracker.maxtasks.perjob":"-1","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","io.native.lib.available":"true","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","ipc.server.listen.queue.size":"128","hive.metastore.client.socket.timeout":"3600","map.sort.class":"org.apache.hadoop.util.QuickSort","hadoop.security.kms.client.authentication.retry-count":"1","fs.wasb.impl.disable.cache":"true","fs.permissions.umask-mode":"022","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","mapreduce.job.ubertask.maxreduces":"1","parquet.block.size.row.check.max":"10","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","fs.cpfs-s3n.impl":"*********(redacted)","spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME","yarn.sharedcache.nested-level":"3","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","hive.stats.autogather":"false","hadoop.http.authentication.kerberos.keytab":"*********(redacted)","yarn.resourcemanager.recovery.enabled":"false"},"System Properties":{"java.io.tmpdir":"/local_disk0/tmp","databricks.usageLogging.bufferedIO":"false","javax.xml.validation.SchemaFactory:http://www.w3.org/2001/XMLSchema":"com.sun.org.apache.xerces.internal.jaxp.validation.XMLSchemaFactory","java.security.properties":"/databricks/spark/dbconf/java/extra.security","databricks.accessJsonLogging.bufferSize":"8192","databricks.auditLogging.bufferSize":"8192","com.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible":"true","line.separator":"\n","path.separator":":","databricks.serviceName":"driver-1","jdk.tls.ephemeralDHKeySize":"2048","databricks.auditLogging.bufferedIO":"false","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Azul Systems, Inc.","javax.xml.parsers.DocumentBuilderFactory":"com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl","java.vm.specification.version":"1.8","databricks.accessJsonLogging.bufferedIO":"false","log4j2.formatMsgNoLookups":"true","user.home":"*********(redacted)","file.encoding.pkg":"sun.io","databricks.productLogging.bufferSize":"8192","sun.nio.ch.bugLevel":"","databricks.auditLogging.immediateFlush":"true","javax.net.ssl.sessionCacheSize":"10000","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/zulu8-ca-amd64/jre/lib/amd64","databricks.billingLogging.bufferedIO":"false","user.dir":"*********(redacted)","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni","sun.cpu.isalist":"","databricks.requestLogging.immediateFlush":"true","databricks.serviceLogging.bufferedIO":"false","os.arch":"amd64","java.vm.version":"25.302-b08","databricks.productLogging.immediateFlush":"true","jetty.git.hash":"5cd5e6d2375eeab146813b0de9f19eda6ab6e6cb","java.endorsed.dirs":"/usr/lib/jvm/zulu8-ca-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_302-b08","java.vm.info":"mixed mode","javax.xml.parsers.SAXParserFactory":"com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl","databricks.accessJsonLogging.immediateFlush":"true","java.ext.dirs":"/usr/lib/jvm/zulu8-ca-amd64/jre/lib/ext:/usr/java/packages/lib/ext","net.snowflake.client.jdbc.internal.io.netty.tryReflectionSetAccessible":"true","java.runtime.name":"OpenJDK Runtime Environment","jdk.vendor.version":"Zulu 8.56.0.21-CA-linux64","javax.xml.datatype.DatatypeFactory":"com.sun.org.apache.xerces.internal.jaxp.datatype.DatatypeFactoryImpl","file.separator":"/","org.w3c.dom.DOMImplementationSourceList":"com.sun.org.apache.xerces.internal.dom.DOMXSImplementationSourceImpl","databricks.requestLogging.bufferSize":"8192","org.xml.sax.driver":"com.sun.org.apache.xerces.internal.parsers.SAXParser","databricks.requestLogging.bufferedIO":"false","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/zulu8-ca-amd64/jre/lib/resources.jar:/usr/lib/jvm/zulu8-ca-amd64/jre/lib/rt.jar:/usr/lib/jvm/zulu8-ca-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/zulu8-ca-amd64/jre/lib/jsse.jar:/usr/lib/jvm/zulu8-ca-amd64/jre/lib/jce.jar:/usr/lib/jvm/zulu8-ca-amd64/jre/lib/charsets.jar:/usr/lib/jvm/zulu8-ca-amd64/jre/lib/jfr.jar:/usr/lib/jvm/zulu8-ca-amd64/jre/classes","file.encoding":"UTF-8","databricks.usageLogging.bufferSize":"8192","user.timezone":"*********(redacted)","jna.platform.library.path":"/usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib64:/lib64:/usr/lib:/lib:/usr/lib/x86_64-linux-gnu/libfakeroot","java.specification.vendor":"Oracle Corporation","jnidispatch.path":"/root/.cache/JNA/temp/jna5854738460128949549.tmp","sun.java.launcher":"SUN_STANDARD","os.version":"5.4.0-1063-aws","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","sun.jnu.encoding":"UTF-8","user.language":"*********(redacted)","databricks.billingLogging.bufferSize":"8192","scala.reflect.runtime.disable.typetag.cache":"true","jna.loaded":"true","databricks.usageLogging.immediateFlush":"true","java.vendor.url":"*********(redacted)","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","scala.repl.name.line":"dummyPrefix","java.vm.vendor":"Azul Systems, Inc.","java.vendor.url.bug":"*********(redacted)","databricks.serviceLogging.bufferSize":"8192","user.name":"*********(redacted)","databricks.billingLogging.immediateFlush":"true","databricks.serviceLogging.immediateFlush":"true","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"com.databricks.backend.daemon.driver.DriverDaemon","java.home":"/usr/lib/jvm/zulu8-ca-amd64/jre","java.version":"1.8.0_302","sun.io.unicode.encoding":"UnicodeLittle","databricks.productLogging.bufferedIO":"false"},"Classpath Entries":{"/databricks/jars/chauffeur-api--api--endpoints--endpoints-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--spire-macros_2.12--org.typelevel__spire-macros_2.12__0.17.0-M1.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-servicecontrol_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--org.codehaus.mojo__animal-sniffer-annotations__1.14_shaded.jar":"System Classpath","/databricks/jars/spark--command--command-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--httpcore_shaded.jar":"System Classpath","/databricks/jars/s3commit--api--api-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--sql-azure-connectors--libsql-azure-connectors_resources.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.springframework--spring-core--org.springframework__spring-core__4.1.4.RELEASE.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jakarta.validation--jakarta.validation-api--jakarta.validation__jakarta.validation-api__2.0.2.jar":"System Classpath","/databricks/jars/third_party--gcs-private--grpc-api_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--common--kvstore--kvstore-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-client--org.eclipse.jetty__jetty-client__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-compress--org.apache.commons__commons-compress__1.20.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.roaringbitmap--RoaringBitmap--org.roaringbitmap__RoaringBitmap__0.9.14.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-serviceusage_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.slf4j--slf4j-log4j12--org.slf4j__slf4j-log4j12__1.7.30.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--core--libcore_resources.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.helger--profiler--com.helger__profiler__1.1.1.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-container_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--httpclient_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-simpledb--com.amazonaws__aws-java-sdk-simpledb__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.avro--avro-mapred-hadoop2--org.apache.avro__avro-mapred-hadoop2__1.8.2.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--com.google.code.findbugs__jsr305__3.0.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2.external--jakarta.inject--org.glassfish.hk2.external__jakarta.inject__2.6.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.transaction--transaction-api--javax.transaction__transaction-api__1.1.jar":"System Classpath","/databricks/jars/----jackson_databind_shaded--libjackson-databind.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-handler-proxy_io.netty__netty-handler-proxy__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--jackson-core_shaded.jar":"System Classpath","/databricks/jars/third_party--azure--com.fasterxml.jackson.core__jackson-core__2.7.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-tcnative-boringssl-static__2.0.31.Final_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--grpc-protobuf_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.transaction--jta--javax.transaction__jta__1.1.jar":"System Classpath","/databricks/hive/conf":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-io--commons-io--commons-io__commons-io__2.4.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.httpcomponents__httpclient__4.5.2_2.12_shaded_20180920_b33d810_spark_3.1.jar":"System Classpath","/databricks/jars/secret-manager--api--api-spark_3.1_2.12_deploy.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-crypto--org.apache.commons__commons-crypto__1.1.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--sql-dw--sql-dw-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.sun.mail--javax.mail--com.sun.mail__javax.mail__1.5.2.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.perfmark_perfmark-api_io.perfmark__perfmark-api__0.23.0_shaded.jar":"System Classpath","/databricks/jars/common--storage--storage-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--hadoop-azure-2.7.3-abfs-external-20180920_b33d810-spark_3.1_2.12_deploy_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.jcraft--jsch--com.jcraft__jsch__0.1.50.jar":"System Classpath","/databricks/jars/----jackson_datatype_joda_shaded--libjackson-datatype-joda.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml.jackson.module__jackson-module-jaxb-annotations__2.11.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--chill_2.12--com.twitter__chill_2.12__0.9.5.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-kinesis--com.amazonaws__aws-java-sdk-kinesis__1.11.655.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-dns_io.netty__netty-codec-dns__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-protobuf-lite_io.grpc__grpc-protobuf-lite__1.38.1_shaded.jar":"System Classpath","/databricks/jars/----jackson_core_shaded--libjackson-core.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--algebra_2.12--org.typelevel__algebra_2.12__2.0.0-M2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-jvm_2.12--com.twitter__util-jvm_2.12__7.1.0.jar":"System Classpath","/databricks/jars/third_party--gcp-java--api-common_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.razorvine--pyrolite--net.razorvine__pyrolite__4.30.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--org.checkerframework__checker-compat-qual__2.5.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.mariadb.jdbc--mariadb-java-client--org.mariadb.jdbc__mariadb-java-client__2.2.5.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-ec2--com.amazonaws__aws-java-sdk-ec2__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-servlet--org.eclipse.jetty.websocket__websocket-servlet__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.clearspring.analytics--stream--com.clearspring.analytics__stream__2.9.6.jar":"System Classpath","/databricks/jars/spark--driver--common--driver-common-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/common--storage-driver-utils--storage_driver_utils-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-yarn-server-common--org.apache.hadoop__hadoop-yarn-server-common__2.7.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--catalyst--libcatalyst_resources.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.datatype--jackson-datatype-joda--com.fasterxml.jackson.datatype__jackson-datatype-joda__2.10.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--catalyst--catalyst-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/dbfs--utils--dbfs-utils-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.fusesource.leveldbjni--leveldbjni-all--org.fusesource.leveldbjni__leveldbjni-all__1.8.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-keyvault-core__1.0.0_2.12_shaded_20180920_b33d810_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--gcp-java--grpc-api_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.curator--curator-recipes--org.apache.curator__curator-recipes__2.7.1.jar":"System Classpath","/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-shim-common_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--stax--stax-api--stax__stax-api__1.0.1.jar":"System Classpath","/databricks/jars/third_party--azure--com.microsoft.azure__azure-client-runtime__1.7.12_container_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.httpcomponents__httpcore__4.4.4_2.12_shaded_20180920_b33d810_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.sf.jpam--jpam--net.sf.jpam__jpam__1.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-jvm--io.dropwizard.metrics__metrics-jvm__4.1.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml.jackson.datatype__jackson-datatype-jsr310__2.11.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-http--org.eclipse.jetty__jetty-http__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-simpleworkflow--com.amazonaws__aws-java-sdk-simpleworkflow__1.11.655.jar":"System Classpath","/databricks/jars/common--network--network-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudwatchmetrics--com.amazonaws__aws-java-sdk-cloudwatchmetrics__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-codec-socks__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.hibernate.validator--hibernate-validator--org.hibernate.validator__hibernate-validator__6.1.0.Final.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.google.guava__guava__16.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--amazon-kinesis-client--com.amazonaws__amazon-kinesis-client__1.12.0.jar":"System Classpath","/databricks/jars/jsonutil--jsonutil-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-llap-client--org.apache.hive__hive-llap-client__2.3.7.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-sbt--test-interface--org.scala-sbt__test-interface__1.0.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--com.google.code.gson__gson__2.8.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jakarta.ws.rs--jakarta.ws.rs-api--jakarta.ws.rs__jakarta.ws.rs-api__2.1.6.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.core--jackson-databind--com.fasterxml.jackson.core__jackson-databind__2.10.0.jar":"System Classpath","/databricks/jars/third_party--gcs-private--grpc-core_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--gax_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-app_2.12--com.twitter__util-app_2.12__7.1.0.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-oauth-client_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--jackson-core_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-plus--org.eclipse.jetty__jetty-plus__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/chauffeur-api--chauffeur-api-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-xml--org.eclipse.jetty__jetty-xml__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.wildfly.openssl--wildfly-openssl--org.wildfly.openssl__wildfly-openssl__1.0.7.Final.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.datanucleus--datanucleus-api-jdo--org.datanucleus__datanucleus-api-jdo__4.2.4.jar":"System Classpath","/databricks/jars/third_party--jackson--jackson-module-scala-shaded_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--dropwizard-metrics-log4j-v3.2.6--metrics-log4j-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-http-client-appengine_shaded.jar":"System Classpath","/databricks/jars/third_party--jetty-client--jetty-util_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-impl__0.22.1_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--grpc-stub_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang.modules--scala-collection-compat_2.12--org.scala-lang.modules__scala-collection-compat_2.12__2.1.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudformation--com.amazonaws__aws-java-sdk-cloudformation__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-elasticloadbalancing--com.amazonaws__aws-java-sdk-elasticloadbalancing__1.11.655.jar":"System Classpath","/databricks/jars/common--path--path-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-proxy--org.eclipse.jetty__jetty-proxy__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--native_ref-java--com.github.fommil.netlib__native_ref-java__1.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-stats_2.12--com.twitter__util-stats_2.12__7.1.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--zookeeper-3.4--liball_deps_2.12.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--libkafka-0-10-resources_shaded-for-kafka-0-10.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.yaml--snakeyaml--org.yaml__snakeyaml__1.24.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-jobclient--org.apache.hadoop__hadoop-mapreduce-client-jobclient__2.7.4.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--commons-codec__commons-codec__1.9_2.12_shaded_20180920_b33d810_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.dataformat--jackson-dataformat-cbor--com.fasterxml.jackson.dataformat__jackson-dataformat-cbor__2.10.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-server--org.eclipse.jetty__jetty-server__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalactic--scalactic_2.12--org.scalactic__scalactic_2.12__3.0.8.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-transport__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-support--com.amazonaws__aws-java-sdk-support__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-marketplacecommerceanalytics--com.amazonaws__aws-java-sdk-marketplacecommerceanalytics__1.11.655.jar":"System Classpath","/databricks/jars/third_party--gcs-private--proto-google-iam-v1_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.okhttp3__okhttp__3.3.1_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-core_io.grpc__grpc-core__1.38.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudhsm--com.amazonaws__aws-java-sdk-cloudhsm__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.antlr--ST4--org.antlr__ST4__4.0.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-elasticache--com.amazonaws__aws-java-sdk-elasticache__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.thrift--libthrift--org.apache.thrift__libthrift__0.12.0.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-handler_io.netty__netty-handler__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.core--jackson-core--com.fasterxml.jackson.core__jackson-core__2.10.0.jar":"System Classpath","/databricks/jars/third_party--gcp-java--httpclient_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-common_io.netty__netty-common__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--grpc-protobuf-lite_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-handler-proxy__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--hive-thriftserver--hive-thriftserver-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_org.curioswitch.curiostack_protobuf-jackson_org.curioswitch.curiostack__protobuf-jackson__1.2.0_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--grpc-stub_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--net.minidev__accessors-smart__1.2_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--j2objc-annotations_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-storage__8.6.4_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-vector--org.apache.arrow__arrow-vector__2.0.0.jar":"System Classpath","/databricks/jars/spark--versions--3.1--spark_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-util--org.eclipse.jetty__jetty-util__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--com.google.j2objc__j2objc-annotations__1.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.reactivestreams__reactive-streams__1.0.3_shaded.jar":"System Classpath","/databricks/jars/third_party--azure--org.apache.commons__commons-lang3__3.4_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-extensions_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.ant--ant--org.apache.ant__ant__1.9.2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-format--org.apache.parquet__parquet-format__2.4.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-client--org.eclipse.jetty.websocket__websocket-client__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--macro-compat_2.12--org.typelevel__macro-compat_2.12__1.1.1.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-transport-native-unix-common-linux-x86_64_io.netty__netty-transport-native-unix-common-linux-x86_64__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--com.lmax__disruptor__3.4.2_shaded.jar":"System Classpath","/databricks/jars/common--client--client-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.janino--janino--org.codehaus.janino__janino__3.0.16.jar":"System Classpath","/databricks/jars/common--rate-limiter--rate-limiter-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.okhttp3__logging-interceptor__3.3.1_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-client_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.spark-project.spark--unused--org.spark-project.spark__unused__1.0.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.htrace--htrace-core--org.apache.htrace__htrace-core__3.1.0-incubating.jar":"System Classpath","/databricks/jars/libraries--libraries-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--jsr305_shaded.jar":"System Classpath","/databricks/jars/common--common-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-lint_2.12--com.twitter__util-lint_2.12__7.1.0.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.opentracing__opentracing-api__0.31.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--avro--io.confluent__common-utils__4.0.0_shaded-for-avro.jar":"System Classpath","/databricks/spark/dbconf/hadoop":"System Classpath","/databricks/jars/third_party--gcp-java--gson_shaded.jar":"System Classpath","/databricks/jars/third_party--jetty-client--jetty-client_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--spire_2.12--org.typelevel__spire_2.12__0.17.0-M1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.github.stephenc.jcip__jcip-annotations__1.0-1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--avro--avro_2.12_deploy_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.httpcomponents--httpclient--org.apache.httpcomponents__httpclient__4.5.6.jar":"System Classpath","/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-client-common_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--spire-platform_2.12--org.typelevel__spire-platform_2.12__0.17.0-M1.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-api-client-jackson2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-jetty9--io.dropwizard.metrics__metrics-jetty9__4.1.1.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-http_io.netty__netty-codec-http__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.projectreactor.netty__reactor-netty__0.9.11.RELEASE_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.luben--zstd-jni--com.github.luben__zstd-jni__1.4.8-1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-ses--com.amazonaws__aws-java-sdk-ses__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-glue--com.amazonaws__aws-java-sdk-glue__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--redshift--redshift-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/spark--driver--display--display-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.code.gson--gson--com.google.code.gson__gson__2.2.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalanlp--breeze_2.12--org.scalanlp__breeze_2.12__1.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.tdunning--json--com.tdunning__json__1.8.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-hdfs--org.apache.hadoop__hadoop-hdfs__2.7.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient_pushgateway--io.prometheus__simpleclient_pushgateway__0.7.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--core--proto-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.datanucleus--javax.jdo--org.datanucleus__javax.jdo__3.2.0-m3.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-lang3--org.apache.commons__commons-lang3__3.10.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-core--org.apache.hadoop__hadoop-mapreduce-client-core__2.7.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--launcher--launcher-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.inject--jersey-hk2--org.glassfish.jersey.inject__jersey-hk2__2.30.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.derby--derby--org.apache.derby__derby__10.12.1.1.jar":"System Classpath","/databricks/jars/spark--driver--events-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--failureaccess_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--sql-azure-connectors--sql-azure-connectors-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/spark--driver--secret-redaction-spark_3.1_2.12_deploy.jar":"*********(redacted)","/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-shim-loader_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.javassist--javassist--org.javassist__javassist__3.25.0-GA.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudwatch--com.amazonaws__aws-java-sdk-cloudwatch__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.esotericsoftware--minlog--com.esotericsoftware__minlog__1.3.0.jar":"System Classpath","/databricks/jars/third_party--azure--com.microsoft.azure__azure-storage__8.6.4_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_net.bytebuddy_byte-buddy_net.bytebuddy__byte-buddy__1.10.19_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.google.code.findbugs__jsr305__1.3.9_2.12_shaded_20180920_b33d810_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-annotations__1.2.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.airlift--aircompressor--io.airlift__aircompressor__0.10.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.jackson--jackson-core-asl--org.codehaus.jackson__jackson-core-asl__1.9.13.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.linecorp.armeria_armeria-grpc_com.linecorp.armeria__armeria-grpc__1.9.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-encoding--org.apache.parquet__parquet-encoding__1.10.1-databricks9.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jets3t-0.7--liball_deps_2.12.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.avro--avro--org.apache.avro__avro__1.8.2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.media--jersey-media-jaxb--org.glassfish.jersey.media__jersey-media-jaxb__2.30.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-http-client_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--opencensus-contrib-http-util_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--xmlenc--xmlenc--xmlenc__xmlenc__0.52.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.code.findbugs--jsr305--com.google.code.findbugs__jsr305__3.0.0.jar":"System Classpath","/databricks/jars/third_party--gcp-java--grpc-netty-shaded_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.azure__azure-identity__1.1.3_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--mllib--mllib_2.12_deploy_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.grpc__grpc-context__1.19.0_shaded.jar":"System Classpath","/databricks/jars/workflow--workflow-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--spark-ganglia-lgpl--libmetrics-ganglia.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-servlets--io.dropwizard.metrics__metrics-servlets__4.1.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jets3t-0.7--com.databricks--jets3t--com.databricks__jets3t__0.7.1-0.jar":"System Classpath","/databricks/jars/third_party--azure--com.microsoft.rest__client-runtime__1.7.12_container_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--commons-codec_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-common--org.eclipse.jetty.websocket__websocket-common__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive.shims--hive-shims-0.23--org.apache.hive.shims__hive-shims-0.23__2.3.7.jar":"System Classpath","/databricks/jars/third_party--gcs-private--protobuf-java-util_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-http2_io.netty__netty-codec-http2__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--common--sketch--sketch-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--jsonwebtoken--jjwt-impl_shaded.jar":"*********(redacted)","/databricks/jars/third_party--gcp-java--google-api-services-storage_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-beanutils--commons-beanutils--commons-beanutils__commons-beanutils__1.9.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.threeten--threeten-extra--org.threeten__threeten-extra__1.5.0.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.okio__okio__1.8.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--gcs-private--util-hadoop-hadoop2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cognitoidentity--com.amazonaws__aws-java-sdk-cognitoidentity__1.11.655.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.rest__client-runtime__1.1.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--com.google.guava__guava__26.0-android_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--javax.activation__activation__1.1_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.xerial.snappy--snappy-java--org.xerial.snappy__snappy-java__1.1.8.2.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_org.latencyutils_LatencyUtils_org.latencyutils__LatencyUtils__2.0.3_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.trueaccord.lenses--lenses_2.12--com.trueaccord.lenses__lenses_2.12__0.4.12.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-elastictranscoder--com.amazonaws__aws-java-sdk-elastictranscoder__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-kms--com.amazonaws__aws-java-sdk-kms__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-importexport--com.amazonaws__aws-java-sdk-importexport__1.11.655.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-auth-library-oauth2-http_shaded.jar":"*********(redacted)","/databricks/jars/third_party--gcp-java--animal-sniffer-annotations_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--py4j--py4j-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----jackson_annotations_shaded--libjackson-annotations.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-pool--commons-pool--commons-pool__commons-pool__1.5.4.jar":"System Classpath","/databricks/spark/dbconf/jets3t/":"System Classpath","/databricks/jars/managed-catalog--api--api-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--datalake--datalake-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.databricks.scalapb--scalapb-runtime_2.12--com.databricks.scalapb__scalapb-runtime_2.12__0.4.15-10.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.avro--avro-ipc--org.apache.avro__avro-ipc__1.8.2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.databricks--Rserve--com.databricks__Rserve__1.8-3.jar":"System Classpath","/databricks/jars/api--common--workspace-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.microsoft.azure__msal4j__1.7.1_shaded.jar":"System Classpath","/databricks/jars/acl--auth--api--api-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2--hk2-api--org.glassfish.hk2__hk2-api__2.6.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.projectreactor__reactor-core__3.3.9.RELEASE_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-hadoop--org.apache.parquet__parquet-hadoop__1.10.1-databricks9.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-datapipeline--com.amazonaws__aws-java-sdk-datapipeline__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient_servlet--io.prometheus__simpleclient_servlet__0.7.0.jar":"System Classpath","/databricks/jars/third_party--gcp-java--gax-grpc_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-transport-native-unix-common__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/utils--process_utils-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--net.minidev__json-smart__2.3_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-http-client-jackson2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-net--commons-net--commons-net__commons-net__3.1.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-auth-library-credentials_shaded.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-dynamodb--com.amazonaws__aws-java-sdk-dynamodb__1.11.655.jar":"System Classpath","/databricks/jars/third_party--jetty-client--jetty-http_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-auth-library-oauth2-http_shaded.jar":"*********(redacted)","/databricks/jars/third_party--gcs-private--google-http-client-apache-v2_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.linecorp.armeria_armeria_com.linecorp.armeria__armeria__1.9.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.curator--curator-framework--org.apache.curator__curator-framework__2.7.1.jar":"System Classpath","/databricks/jars/----scalapb_090--com.lihaoyi__sourcecode_2.12__0.1.7_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.module--jackson-module-paranamer--com.fasterxml.jackson.module__jackson-module-paranamer__2.10.0.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--javax.inject__javax.inject__1_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.ow2.asm__asm__5.0.4_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.xbean--xbean-asm7-shaded--org.apache.xbean__xbean-asm7-shaded__4.15.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.code.findbugs_jsr305_com.google.code.findbugs__jsr305__3.0.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang.modules--scala-xml_2.12--org.scala-lang.modules__scala-xml_2.12__1.2.0.jar":"System Classpath","/databricks/jars/common--tracing--tracing-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-yarn-api--org.apache.hadoop__hadoop-yarn-api__2.7.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-math3--org.apache.commons__commons-math3__3.4.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-app--org.apache.hadoop__hadoop-mapreduce-client-app__2.7.4.jar":"System Classpath","/databricks/jars/third_party--gcs-private--gcs-shaded-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-format--org.apache.arrow__arrow-format__2.0.0.jar":"System Classpath","/databricks/jars/s3commit--client--client-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-memory-core--org.apache.arrow__arrow-memory-core__2.0.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--patched-hive-with-glue--hive-common_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-util-ajax--org.eclipse.jetty__jetty-util-ajax__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-iam_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--commons-codec_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-yarn-common--org.apache.hadoop__hadoop-yarn-common__2.7.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-security--org.eclipse.jetty__jetty-security__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient_common--io.prometheus__simpleclient_common__0.7.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--hive--hive_2.12_deploy_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.crypto.tink--tink--com.google.crypto.tink__tink__1.6.0.jar":"System Classpath","/databricks/jars/third_party--jetty-client--jetty-io_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-http-client_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.thoughtworks.paranamer--paranamer--com.thoughtworks.paranamer__paranamer__2.8.jar":"System Classpath","/databricks/jars/third_party--gcs-private--grpc-auth_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2--hk2-locator--org.glassfish.hk2__hk2-locator__2.6.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.java.dev.jna--jna--net.java.dev.jna__jna__5.8.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-jaxb__2.1.4_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.ant--ant-launcher--org.apache.ant__ant-launcher__1.9.2.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.jaegertracing__jaeger-client__0.33.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-codec__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--file-notification-common--file-notification-common-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.sf.opencsv--opencsv--net.sf.opencsv__opencsv__2.3.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--streaming--libstreaming_resources.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--com.squareup.okio__okio__1.13.0_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--io.reactivex__rxjava__1.2.4_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--common--unsafe--unsafe-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--sql-gcp-connectors--sql-gcp-connectors-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.httpcomponents__httpcore__4.4.4_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--zookeeper-3.4--org.apache.zookeeper--zookeeper--org.apache.zookeeper__zookeeper__3.4.14.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-core--io.dropwizard.metrics__metrics-core__4.1.1.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-impl-core__0.22.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.xerial--sqlite-jdbc--org.xerial__sqlite-jdbc__3.8.11.2.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-cloud-pubsub_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.opentracing__opentracing-noop__0.31.0_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-http-client-apache-v2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-sqs--com.amazonaws__aws-java-sdk-sqs__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--chill-java--com.twitter__chill-java__0.9.5.jar":"System Classpath","/databricks/jars/cmdexec--api--api-experimental-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-ssm--com.amazonaws__aws-java-sdk-ssm__1.11.655.jar":"System Classpath","/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-shim-hive2_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml.woodstox__woodstox-core__6.2.1_shaded.jar":"System Classpath","/databricks/jars/common--node-types--node-types-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.tukaani--xz--org.tukaani__xz__1.5.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-core_2.12--com.twitter__util-core_2.12__7.1.0.jar":"System Classpath","/databricks/jars/api-base--api-base_java-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--catalyst--proto-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_org.joda_joda-convert_org.joda__joda-convert__2.2.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.joda--joda-convert--org.joda__joda-convert__1.7.jar":"System Classpath","/databricks/spark/dbconf/log4j/driver":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--commons-codec__commons-codec__1.9_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalacheck--scalacheck_2.12--org.scalacheck__scalacheck_2.12__1.14.2.jar":"System Classpath","/databricks/jars/managed-catalog--api--api_common-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--threetenbp_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudsearch--com.amazonaws__aws-java-sdk-cloudsearch__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2--osgi-resource-locator--org.glassfish.hk2__osgi-resource-locator__1.0.3.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang.modules--scala-parser-combinators_2.12--org.scala-lang.modules__scala-parser-combinators_2.12__1.1.2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.directory.api--api-util--org.apache.directory.api__api-util__1.0.0-M20.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--commons-logging__commons-logging__1.2_2.12_shaded_20180920_b33d810_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--gcs-private--flogger-system-backend_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--patched-hive-with-glue--hive-exec-core_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--patched-hive-with-glue--hive-metastore_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--kafka-0-10-token-provider--libkafka-0-10-token-provider_resources.jar":"*********(redacted)","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-socks_io.netty__netty-codec-socks__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.datatype__jackson-datatype-joda__2.7.2_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--gcs-private--proto-google-common-protos_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-http-client-gson_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-api-services-iamcredentials_shaded.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--antlr--antlr--antlr__antlr__2.7.7.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-handler__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--mllib-local--mllib-local-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcs-private--opencensus-api_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--catalyst--libspark-sql-parser-compiled.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient_dropwizard--io.prometheus__simpleclient_dropwizard__0.7.0.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.fasterxml.jackson.core_jackson-annotations_com.fasterxml.jackson.core__jackson-annotations__2.12.3_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-cloud-core-http_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.ning--compress-lzf--com.ning__compress-lzf__1.0.3.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--log4j--log4j--log4j__log4j__1.2.17.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.xml.bind--jaxb-api--javax.xml.bind__jaxb-api__2.2.2.jar":"System Classpath","/databricks/jars/third_party--gcs-private--libmeta-services.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalatest--scalatest_2.12--org.scalatest__scalatest_2.12__3.0.8.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-sns--com.amazonaws__aws-java-sdk-sns__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-column--org.apache.parquet__parquet-column__1.10.1-databricks9.jar":"System Classpath","/databricks/jars/logging--utils--logging-utils-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--auto-value-annotations_shaded.jar":"System Classpath","/databricks/jars/sqlgateway--common--endpoint_id-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.guava_failureaccess_com.google.guava__failureaccess__1.0.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.antlr--antlr-runtime--org.antlr__antlr-runtime__3.5.2.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-stub_io.grpc__grpc-stub__1.38.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.netty--netty-all--io.netty__netty-all__4.1.51.Final.jar":"System Classpath","/databricks/jars/common--libcommon_resources.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.acplt.remotetea--remotetea-oncrpc--org.acplt.remotetea__remotetea-oncrpc__1.1.2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-core--com.amazonaws__aws-java-sdk-core__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.rosuda.REngine--REngine--org.rosuda.REngine__REngine__2.1.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2__2.1.4_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.commons--commons-text--org.apache.commons__commons-text__1.6.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.directory.api--api-asn1-api--org.apache.directory.api__api-asn1-api__1.0.0-M20.jar":"System Classpath","/databricks/jars/credentials-manager--api--api-spark_3.1_2.12_deploy.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.roaringbitmap--shims--org.roaringbitmap__shims__0.9.14.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.thrift--libfb303--org.apache.thrift__libfb303__0.9.3.jar":"System Classpath","/databricks/jars/common--reflection--reflection-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/common--crypto-providers--amazon-corretto-crypto-provider--libamazon-corretto-crypto-provider.jar":"System Classpath","/databricks/jars/third_party--gcp-java--grpc-context_shaded.jar":"System Classpath","/databricks/jars/dbfs--exceptions--exceptions-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----scalapb_090--com.lihaoyi__fastparse_2.12__2.1.3_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--kafka-0-10-token-provider-unshaded_2.12_deploy_shaded-for-kafka-0-10.jar":"*********(redacted)","/databricks/jars/third_party--gcp-java--grpc-protobuf_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.fasterxml.jackson.core_jackson-databind_com.fasterxml.jackson.core__jackson-databind__2.12.3_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--graphx--graphx-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--commons-logging_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang--scala-compiler_2.12--org.scala-lang__scala-compiler__2.12.10.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.api.grpc_proto-google-common-protos_com.google.api.grpc__proto-google-common-protos__2.0.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-shims--org.apache.hive__hive-shims__2.3.7.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javolution--javolution--javolution__javolution__5.5.1.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec-haproxy_io.netty__netty-codec-haproxy__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.slf4j--jul-to-slf4j--org.slf4j__jul-to-slf4j__1.7.30.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--org.apache.httpcomponents__httpcore__4.4.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-rds--com.amazonaws__aws-java-sdk-rds__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.snowflake--snowflake-ingest-sdk--net.snowflake__snowflake-ingest-sdk__0.9.6.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--machinist_2.12--org.typelevel__machinist_2.12__0.6.8.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--oro--oro--oro__oro__2.0.8.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-httpclient--commons-httpclient--commons-httpclient__commons-httpclient__3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.slf4j--slf4j-api--org.slf4j__slf4j-api__1.7.30.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--spire-util_2.12--org.typelevel__spire-util_2.12__0.17.0-M1.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--io.netty__netty-all__4.0.52.Final_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-sts--com.amazonaws__aws-java-sdk-sts__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.azure__azure-core-http-netty__1.6.1_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-auth-library-credentials_shaded.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--sql--core--spark-sql-databricks-command-parser_java_out.srcjar":"System Classpath","/databricks/jars/third_party--gcp-java--grpc-alts_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--pl.edu.icm--JLargeArrays--pl.edu.icm__JLargeArrays__1.5.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--netlib-native_ref-linux-x86_64-natives--com.github.fommil.netlib__netlib-native_ref-linux-x86_64-natives__1.1.jar":"System Classpath","/databricks/jars/sqlgateway--history--api--client-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.guava_listenablefuture_com.google.guava__listenablefuture__9999.0-empty-to-avoid-conflict-with-guava_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-exporter-trace-jaeger__0.22.1_shaded.jar":"System Classpath","/databricks/jars/common--java-flight-recorder--java-flight-recorder-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-logs--com.amazonaws__aws-java-sdk-logs__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jline--jline--jline__jline__2.14.6.jar":"System Classpath","/databricks/jars/third_party--gcs-private--gcsio-hadoop2_shaded.jar":"System Classpath","/databricks/jars/libraries--api--typemappers--typemappers-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-codec--commons-codec--commons-codec__commons-codec__1.10.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.microsoft.azure--azure-data-lake-store-sdk--com.microsoft.azure__azure-data-lake-store-sdk__2.3.9.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--patched-hive-with-glue--hive-12679-patch_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--grpc-core_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.bouncycastle__bcprov-jdk15on__1.69_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-networkmanagement_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-api_io.grpc__grpc-api__1.38.1_shaded.jar":"System Classpath","/databricks/jars/libraries--api--managedLibraries--managedLibraries-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-cloudresourcemanager_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--annotations_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.typesafe.scala-logging--scala-logging_2.12--com.typesafe.scala-logging__scala-logging_2.12__3.7.2.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-api-services-storage_shaded.jar":"System Classpath","/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-shim-hive1_deploy.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.opentracing__opentracing-util__0.31.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-directory--com.amazonaws__aws-java-sdk-directory__1.11.655.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_org.checkerframework_checker-compat-qual_org.checkerframework__checker-compat-qual__2.5.5_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive.shims--hive-shims-common--org.apache.hive.shims__hive-shims-common__2.3.7.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-annotations--org.apache.hadoop__hadoop-annotations__2.7.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-core_2.12--org.json4s__json4s-core_2.12__3.7.0-M5.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-resolver_io.netty__netty-resolver__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--hive--libhive_resources.jar":"System Classpath","/databricks/jars/third_party--gcs-private--protobuf-java_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-storage-api--org.apache.hive__hive-storage-api__2.7.2.jar":"System Classpath","/databricks/jars/third_party--gcp-java--javax.annotation-api_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--streaming--streaming-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/extern--acl--auth--auth-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--kafka-0-10_2.12_deploy_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--org.apache.thrift__libthrift__0.11.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.containers--jersey-container-servlet--org.glassfish.jersey.containers__jersey-container-servlet__2.30.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--org.apache.httpcomponents__httpclient__4.4.1_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-protobuf_io.grpc__grpc-protobuf__1.38.1_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.retrofit2__retrofit__2.1.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-servlets--org.eclipse.jetty__jetty-servlets__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.el--javax.el-api--javax.el__javax.el-api__2.2.4.jar":"System Classpath","/databricks/jars/third_party--gcp-java--grpc-grpclb_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.jackson--jackson-jaxrs--org.codehaus.jackson__jackson-jaxrs__1.9.13.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_org.reactivestreams_reactive-streams_org.reactivestreams__reactive-streams__1.0.3_shaded.jar":"System Classpath","/databricks/jars/daemon--data--client--utils-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--jsonwebtoken--jackson-core_shaded.jar":"*********(redacted)","/databricks/jars/third_party--gcs-private--error_prone_annotations_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--avro--io.confluent__kafka-schema-registry-client__4.0.0_shaded-for-avro.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.micrometer_micrometer-core_io.micrometer__micrometer-core__1.7.1_shaded.jar":"System Classpath","/databricks/jars/third_party--jsonwebtoken--jjwt-jackson_shaded.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-cli--commons-cli--commons-cli__commons-cli__1.2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.zaxxer--HikariCP--com.zaxxer__HikariCP__3.1.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--spark-ganglia-lgpl--spark-ganglia-lgpl-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.madgag.spongycastle__core__1.54.0.0_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--error_prone_annotations_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.sun.xml.bind__jaxb-impl__2.2.3-1_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.orc--orc-shims--org.apache.orc__orc-shims__1.5.12.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-common__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--httpcore_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-s3--com.amazonaws__aws-java-sdk-s3__1.11.655.jar":"System Classpath","/databricks/jars/third_party--gcs-private--grpc-alts_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--zookeeper-3.4--org.apache.yetus--audience-annotations--org.apache.yetus__audience-annotations__0.5.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-common--org.apache.hadoop__hadoop-common__2.7.4.jar":"System Classpath","/databricks/jars/daemon--data--data-common--data-common-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.opentracing.contrib__opentracing-tracerresolver__0.1.5_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.sf.supercsv--super-csv--net.sf.supercsv__super-csv__2.2.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.databricks.scalapb--compilerplugin_2.12--com.databricks.scalapb__compilerplugin_2.12__0.4.15-10.jar":"System Classpath","/databricks/jars/cmdexec--api--api-experimental-proto-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.wendykierp--JTransforms--com.github.wendykierp__JTransforms__3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.jackson--jackson-xc--org.codehaus.jackson__jackson-xc__1.9.13.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.sourceforge.f2j--arpack_combined_all--net.sourceforge.f2j__arpack_combined_all__0.1.jar":"System Classpath","/databricks/jars/common--util--locks-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-shuffle--org.apache.hadoop__hadoop-mapreduce-client-shuffle__2.7.4.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.errorprone_error_prone_annotations_com.google.errorprone__error_prone_annotations__2.4.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-json--io.dropwizard.metrics__metrics-json__4.1.1.jar":"System Classpath","/databricks/jars/third_party--zeromq--jnacl_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--joda-time__joda-time__2.4_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.chuusai--shapeless_2.12--com.chuusai__shapeless_2.12__2.3.3.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-iam--com.amazonaws__aws-java-sdk-iam__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.jodd--jodd-core--org.jodd__jodd-core__3.5.2.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-buffer_io.netty__netty-buffer__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-codec-http2__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-lambda--com.amazonaws__aws-java-sdk-lambda__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--mllib--libmllib_resources.jar":"System Classpath","/databricks/jars/third_party--gcs-private--checker-compat-qual_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-cloud-core_shaded.jar":"System Classpath","/databricks/jars/third_party--jackson--guava_only_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-autoscaling--com.amazonaws__aws-java-sdk-autoscaling__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-efs--com.amazonaws__aws-java-sdk-efs__1.11.655.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-cloudkms_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--core--libcore_generated_resources.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--native_system-java-natives--com.github.fommil.netlib__native_system-java-natives__1.1.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.retrofit2__adapter-rxjava__2.1.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--gcp-java--grpc-protobuf-lite_shaded.jar":"System Classpath","/databricks/jars/third_party--jackson--jsr305_only_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-ast_2.12--org.json4s__json4s-ast_2.12__3.7.0-M5.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-dns_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.core--jersey-client--org.glassfish.jersey.core__jersey-client__2.30.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.mesos--mesos-shaded-protobuf--org.apache.mesos__mesos-shaded-protobuf__1.4.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-storagegateway--com.amazonaws__aws-java-sdk-storagegateway__1.11.655.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-context_io.grpc__grpc-context__1.38.1_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-http-client-jackson2_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.core__jackson-annotations__2.7.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--jmespath-java--com.amazonaws__jmespath-java__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-dbcp--commons-dbcp--commons-dbcp__commons-dbcp__1.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.nimbusds__nimbus-jose-jwt__8.8_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-marketplacemeteringservice--com.amazonaws__aws-java-sdk-marketplacemeteringservice__1.11.655.jar":"System Classpath","/databricks/jars/third_party--jackson--paranamer_only_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--info.ganglia.gmetric4j--gmetric4j--info.ganglia.gmetric4j__gmetric4j__1.0.10.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-api__0.22.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil--jniloader--com.github.fommil__jniloader__1.1.jar":"System Classpath","/databricks/jars/third_party--gcp-java--opencensus-api_shaded.jar":"System Classpath","/databricks/jars/macros--sourcecode--sourcecode-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.retrofit2__converter-jackson__2.1.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-graphite--io.dropwizard.metrics__metrics-graphite__4.1.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.antlr--antlr4-runtime--org.antlr__antlr4-runtime__4.8-1.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--hadoop-azure-2.7.3-abfs-external-20180625_3682417-spark_3.1_2.12_deploy_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--common--tags--tags-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/spark--common--spark-common-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcs-private--conscrypt-openjdk-uber_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.squareup.okhttp3__okhttp-urlconnection__3.3.1_2.12_shaded_20180625_3682417_spark_3.1.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.joshelser--dropwizard-metrics-hadoop-metrics2-reporter--com.github.joshelser__dropwizard-metrics-hadoop-metrics2-reporter__0.1.2.jar":"System Classpath","/databricks/jars/maven-trees--amazon-corretto-crypto-provider--software.amazon.cryptools--AmazonCorrettoCryptoProvider-linux-x86_64--software.amazon.cryptools__AmazonCorrettoCryptoProvider-linux-x86_64__1.4.0.jar":"System Classpath","/databricks/jars/third_party--hadoop--hadoop-tools--hadoop-aws--lib-spark_3.1_2.12_deploy_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-dom__2.1.4_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.orc--orc-mapreduce--org.apache.orc__orc-mapreduce__1.5.12.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-jndi--org.eclipse.jetty__jetty-jndi__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-deploymentmanager_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-machinelearning--com.amazonaws__aws-java-sdk-machinelearning__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus.jmx--collector--io.prometheus.jmx__collector__0.12.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-transport-native-epoll-linux-x86_64__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-llap-common--org.apache.hive__hive-llap-common__2.3.7.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.core--jackson-annotations--com.fasterxml.jackson.core__jackson-annotations__2.10.0.jar":"System Classpath","/databricks/jars/macros--ratelimitedlogger--ratelimitedlogger-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/s3--s3-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml--classmate--com.fasterxml__classmate__1.3.4.jar":"System Classpath","/databricks/jars/acl--api--helpers--helpers-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-collections--commons-collections--commons-collections__commons-collections__3.2.2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2--hk2-utils--org.glassfish.hk2__hk2-utils__2.6.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.janino--commons-compiler--org.codehaus.janino__commons-compiler__3.0.16.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.orc--orc-core--org.apache.orc__orc-core__1.5.12.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.core__jackson-core__2.7.2_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.slf4j--jcl-over-slf4j--org.slf4j__jcl-over-slf4j__1.7.30.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--avro--libavro_resources_shaded-for-avro.jar":"System Classpath","/databricks/jars/third_party--gcs-private--grpc-context_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--perfmark-api_shaded.jar":"System Classpath","/databricks/jars/common--lazy--lazy-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-cloud-notification_shaded.jar":"System Classpath","/databricks/jars/spark--driver--ml--ml-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcs-private--gcs-connector-hadoop2_shaded.jar":"System Classpath","/databricks/jars/logging--log4j-mod--log4j-mod-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--software.amazon.ion--ion-java--software.amazon.ion__ion-java__1.0.2.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.grpc_grpc-services_io.grpc__grpc-services__1.38.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-continuation--org.eclipse.jetty__jetty-continuation__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudfront--com.amazonaws__aws-java-sdk-cloudfront__1.11.655.jar":"System Classpath","/databricks/jars/third_party--gcs-private--guava_shaded.jar":"System Classpath","/databricks/jars/sqlgateway--history--api--api-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_org.codehaus.mojo_animal-sniffer-annotations_org.codehaus.mojo__animal-sniffer-annotations__1.19_shaded.jar":"System Classpath","/databricks/jars/third_party--zeromq--jeromq_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.rocksdb--rocksdbjni--org.rocksdb__rocksdbjni__6.20.3.jar":"System Classpath","/databricks/jars/third_party--gcs-private--listenablefuture_shaded.jar":"System Classpath","/databricks/jars/common--hadoop--hadoop-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.containers--jersey-container-servlet-core--org.glassfish.jersey.containers__jersey-container-servlet-core__2.30.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-io--org.eclipse.jetty__jetty-io__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.google.inject__guice__3.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-cli--org.apache.hive__hive-cli__2.3.7.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.typelevel--cats-kernel_2.12--org.typelevel__cats-kernel_2.12__2.0.0-M4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-simple__2.1.4_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-fileupload--commons-fileupload--commons-fileupload__commons-fileupload__1.3.3.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--liball_deps_2.12.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.httpcomponents__httpclient__4.5.2_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-resolver-dns_io.netty__netty-resolver-dns__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.snowflake--spark-snowflake_2.12--net.snowflake__spark-snowflake_2.12__2.9.0-spark_3.1.jar":"System Classpath","/databricks/jars/common--jupyter-utils--jupyter_utils-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--libmssql-shaded-meta-services_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.servlet.jsp--jsp-api--javax.servlet.jsp__jsp-api__2.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-registry_2.12--com.twitter__util-registry_2.12__7.1.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-opsworks--com.amazonaws__aws-java-sdk-opsworks__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.microsoft.sqlserver__mssql-jdbc__9.2.1.jre8_shaded.jar":"System Classpath","/databricks/jars/spark--driver--dbutils_impl--data--rendering--rendering-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--proto-google-common-protos_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.code.gson_gson_com.google.code.gson__gson__2.8.6_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--commons-lang3_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--service_discovery-resources.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.antlr--stringtemplate--org.antlr__stringtemplate__3.2.1.jar":"System Classpath","/databricks/jars/spark--driver--driver-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--bigquery-connector--gcs-connector-hadoop2-2.0.0-shaded.jar_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.nimbusds__oauth2-oidc-sdk__7.1.1_shaded.jar":"*********(redacted)","/databricks/jars/third_party--gcp-java--google-cloud-storage_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.jaegertracing__jaeger-tracerresolver__0.33.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.jdo--jdo-api--javax.jdo__jdo-api__3.0.1.jar":"System Classpath","/databricks/jars/third_party--gcs-private--commons-logging_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-logging--commons-logging--commons-logging__commons-logging__1.1.3.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-redshift--com.amazonaws__aws-java-sdk-redshift__1.11.655.jar":"System Classpath","/databricks/jars/third_party--gcs-private--google-api-client_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--hive--org.apache.commons__commons-pool2__2.6.2_shaded-for-hive.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--native_system-java--com.github.fommil.netlib__native_system-java__1.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.lihaoyi--sourcecode_2.12--com.lihaoyi__sourcecode_2.12__0.1.9.jar":"System Classpath","/databricks/jars/common--pricing--pricing-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.azure__azure-security-keyvault-keys__4.2.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--jakarta.annotation--jakarta.annotation-api--jakarta.annotation__jakarta.annotation-api__1.3.5.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.azure__azure-core__1.8.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--sql-aws-connectors--libsql-aws-connectors_resources.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-ecs--com.amazonaws__aws-java-sdk-ecs__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.ivy--ivy--org.apache.ivy__ivy__2.4.0.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.fasterxml.jackson.core_jackson-core_com.fasterxml.jackson.core__jackson-core__2.12.3_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--bigquery-connector--fatJar-assembly-0.22.2-SNAPSHOT.jar_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--aopalliance__aopalliance__1.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/spark--common-utils--utils-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang--scala-reflect_2.12--org.scala-lang__scala-reflect__2.12.10.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--javax.xml.stream__stax-api__1.0-2_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.datanucleus--datanucleus-core--org.datanucleus__datanucleus-core__4.1.17.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--joda-time--joda-time--joda-time__joda-time__2.10.5.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.nimbusds__content-type__2.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--net.java.dev.jna__jna-platform__5.6.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml__aalto-xml__1.0.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-kdbx__2.1.4_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--core--com.github.fommil.netlib__core__1.1.2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.core--jersey-server--org.glassfish.jersey.core__jersey-server__2.30.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-auth--org.apache.hadoop__hadoop-auth__2.7.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.nanohttpd__nanohttpd__2.3.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--net.snowflake--snowflake-jdbc--net.snowflake__snowflake-jdbc__3.13.3.jar":"System Classpath","/databricks/jars/----scalapb_090--runtime-unshaded-jetty9-hadoop1_2.12_deploy_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--file-notification-common--libfile-notification-common_resources.jar":"System Classpath","/databricks/jars/s3commit--common--common-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.springframework--spring-test--org.springframework__spring-test__4.1.4.RELEASE.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--xpp3__xpp3__1.1.3.3_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--gson_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.curator--curator-client--org.apache.curator__curator-client__2.7.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--org.apache.kafka__kafka-clients__2.6.0_shaded-for-kafka-0-10.jar":"System Classpath","/databricks/jars/common--jetty--client--client-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.fasterxml.jackson.dataformat__jackson-dataformat-xml__2.11.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--jakarta.activation__jakarta.activation-api__1.2.1_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--grpc-auth_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.guava_guava_com.google.guava__guava__30.1-android_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--commons-codec__commons-codec__1.9_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-serde--org.apache.hive__hive-serde__2.3.7.jar":"System Classpath","/databricks/jars/third_party--gcs-private--failureaccess_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--commons-logging__commons-logging__1.2_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-storage__7.0.0_2.12_shaded_20180920_b33d810_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__KeePassJava2-kdb__2.1.4_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-transport-native-kqueue-osx-x86_64__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-beeline--org.apache.hive__hive-beeline__2.3.7.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.jolbox--bonecp--com.jolbox__bonecp__0.8.0.RELEASE.jar":"System Classpath","/databricks/jars/third_party--gcp-java--checker-compat-qual_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-jackson_2.12--org.json4s__json4s-jackson_2.12__3.7.0-M5.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-codedeploy--com.amazonaws__aws-java-sdk-codedeploy__1.11.655.jar":"System Classpath","/databricks/jars/----glue-catalog-spark3.1-client--glue-catalog-hive2-client_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.httpcomponents--httpcore--org.apache.httpcomponents__httpcore__4.4.12.jar":"System Classpath","/databricks/jars/third_party--gcs-private--animal-sniffer-annotations_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.delta--delta-sharing-spark_2.12--io.delta__delta-sharing-spark_2.12__0.2.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.jdbi--jdbi--org.jdbi__jdbi__2.63.1.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.protobuf_protobuf-java-util_com.google.protobuf__protobuf-java-util__3.12.0_shaded.jar":"System Classpath","/databricks/jars/third_party--jsonwebtoken--jackson-annotations_shaded.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-workspaces--com.amazonaws__aws-java-sdk-workspaces__1.11.655.jar":"System Classpath","/databricks/jars/third_party--hadoop_azure_abfs--hadoop-tools--hadoop-azure--lib-spark_3.1_2.12_deploy.jar_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.jetbrains__annotations__15.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cognitosync--com.amazonaws__aws-java-sdk-cognitosync__1.11.655.jar":"System Classpath","/databricks/jars/third_party--gcp-java--opencensus-contrib-http-util_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.flatbuffers--flatbuffers-java--com.google.flatbuffers__flatbuffers-java__1.9.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-client--org.apache.hadoop__hadoop-client__2.7.4.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--com.squareup.okhttp3__okhttp__3.9.0_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.core__jackson-databind__2.7.2_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-codec-http__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/spark--driver--antlr--parser-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--perfmark-api_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.postgresql--postgresql--org.postgresql__postgresql__42.1.4.jar":"System Classpath","/databricks/jars/third_party--azure--com.microsoft.azure__azure-keyvault-core__1.0.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--jakarta.xml.bind__jakarta.xml.bind-api__2.3.2_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--checker-qual_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.protobuf--protobuf-java--com.google.protobuf__protobuf-java__2.6.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-buffer__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive--hive-jdbc--org.apache.hive__hive-jdbc__2.3.7.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.h2database--h2--com.h2database__h2__1.4.195.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-server--org.eclipse.jetty.websocket__websocket-server__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--iceberg--iceberg-converter_2.12_deploy_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.ben-manes.caffeine--caffeine--com.github.ben-manes.caffeine__caffeine__2.3.4.jar":"System Classpath","/databricks/jars/api--rpc--rpc_parser-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.typesafe--config--com.typesafe__config__1.2.1.jar":"System Classpath","/databricks/jars/extern--libaws-regions.jar":"System Classpath","/databricks/jars/third_party--gcp-java--gax-httpjson_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.linguafranca.pwdb__database__2.1.4_shaded.jar":"System Classpath","/databricks/jars/spark--versions--3.1--shim_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--proto-google-cloud-pubsub-v1_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.j2objc_j2objc-annotations_com.google.j2objc__j2objc-annotations__1.3_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--io.netty__netty-resolver__4.1.51.Final_shaded.jar":"System Classpath","/databricks/jars/third_party--jsonwebtoken--jackson-databind_shaded.jar":"*********(redacted)","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--javax.xml.bind__jaxb-api__2.2.2_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-yarn-client--org.apache.hadoop__hadoop-yarn-client__2.7.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.prometheus--simpleclient--io.prometheus__simpleclient__0.7.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--common--network-common--network-common-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcp-java--protobuf-java-util_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--repl--repl-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-elasticbeanstalk--com.amazonaws__aws-java-sdk-elasticbeanstalk__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scala-lang--scala-library_2.12--org.scala-lang__scala-library__2.12.10.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_org.hdrhistogram_HdrHistogram_org.hdrhistogram__HdrHistogram__2.1.12_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--mllib--org.jpmml__pmml-model__1.4.8_shaded-for-mllib.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-glacier--com.amazonaws__aws-java-sdk-glacier__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.hk2.external--aopalliance-repackaged--org.glassfish.hk2.external__aopalliance-repackaged__2.6.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.json4s--json4s-scalap_2.12--org.json4s__json4s-scalap_2.12__3.7.0-M5.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.jaegertracing__jaeger-core__0.33.1_shaded.jar":"System Classpath","/databricks/jars/third_party--jsonwebtoken--jjwt-api_shaded.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.datanucleus--datanucleus-rdbms--org.datanucleus__datanucleus-rdbms__4.1.19.jar":"System Classpath","/databricks/jars/third_party--gcs-private--flogger_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--grpc-netty-shaded_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-lang--commons-lang--commons-lang__commons-lang__2.6.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.google.guava--guava--com.google.guava__guava__15.0.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.microsoft.azure__azure-keyvault-core__1.0.0_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.activation--activation--javax.activation__activation__1.1.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--sql-aws-connectors--sql-aws-connectors-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.simpleframework__simple-xml__2.7.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-jackson--org.apache.parquet__parquet-jackson__1.10.1-databricks9.jar":"System Classpath","/databricks/jars/spark--sql-extension--sql-extension-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.xml.stream--stax-api--javax.xml.stream__stax-api__1.0-2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--iceberg--iceberg-spark3-runtime_shaded_shaded-for-iceberg-converter.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.ant--ant-jsch--org.apache.ant__ant-jsch__1.9.2.jar":"System Classpath","/databricks/jars/central--api--api-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/spark--conf-reader--conf-reader_lib-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.microsoft.azure__msal4j-persistence-extension__1.0.0_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--protobuf-java_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-swf-libraries--com.amazonaws__aws-java-sdk-swf-libraries__1.11.22.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-transport_io.netty__netty-transport__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--com.nimbusds__lang-tag__1.4.4_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.esotericsoftware--kryo-shaded--com.esotericsoftware__kryo-shaded__4.0.2.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.parquet--parquet-common--org.apache.parquet__parquet-common__1.10.1-databricks9.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.fasterxml.jackson.module--jackson-module-scala_2.12--com.fasterxml.jackson.module__jackson-module-scala_2.12__2.10.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.jboss.logging--jboss-logging--org.jboss.logging__jboss-logging__3.3.2.Final.jar":"System Classpath","/databricks/jars/daemon--data--client--conf--conf-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-healthchecks--io.dropwizard.metrics__metrics-healthchecks__4.1.1.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_liball_deps_2.12_shaded.jar":"System Classpath","/databricks/jars/common--credentials--credentials-spark_3.1_2.12_deploy.jar":"*********(redacted)","/databricks/jars/third_party--gcp-java--listenablefuture_shaded.jar":"System Classpath","/databricks/jars/third_party--gcp-java--guava_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.glassfish.jersey.core--jersey-common--org.glassfish.jersey.core__jersey-common__2.30.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--org.codehaus.woodstox__stax2-api__4.2.1_shaded.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_io.netty_netty-codec_io.netty__netty-codec__4.1.65.Final_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--log4j--apache-log4j-extras--log4j__apache-log4j-extras__1.2.17.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-configuration--commons-configuration--commons-configuration__commons-configuration__1.6.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.directory.server--apacheds-i18n--org.apache.directory.server__apacheds-i18n__2.0.0-M15.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.objenesis--objenesis--org.objenesis__objenesis__2.5.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.twitter--util-function_2.12--com.twitter__util-function_2.12__7.1.0.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-compute_shaded.jar":"System Classpath","/databricks/jars/daemon--data--client--client-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-cloudtrail--com.amazonaws__aws-java-sdk-cloudtrail__1.11.655.jar":"System Classpath","/databricks/jars/spark--driver--spark--resources-resources.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--core--proto_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.univocity--univocity-parsers--com.univocity__univocity-parsers__2.9.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.codehaus.jackson--jackson-mapper-asl--org.codehaus.jackson__jackson-mapper-asl__1.9.13.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.scalanlp--breeze-macros_2.12--org.scalanlp__breeze-macros_2.12__1.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.lz4--lz4-java--org.lz4__lz4-java__1.7.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-route53--com.amazonaws__aws-java-sdk-route53__1.11.655.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.android_annotations_com.google.android__annotations__4.1.1.4_shaded.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--com.google.errorprone__error_prone_annotations__2.1.3_shaded.jar":"System Classpath","/databricks/jars/extern--extern-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.opencensus__opencensus-exporter-trace-util__0.22.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--commons-digester--commons-digester--commons-digester__commons-digester__1.8.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--javax.servlet--javax.servlet-api--javax.servlet__javax.servlet-api__3.1.0.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--io.jaegertracing__jaeger-thrift__0.33.1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--vendor--sql-gcp-connectors--libsql-gcp-connectors_resources.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.fasterxml.jackson.core__jackson-core__2.7.2_2.12_shaded_20180920_b33d810_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-directconnect--com.amazonaws__aws-java-sdk-directconnect__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--native_ref-java-natives--com.github.fommil.netlib__native_ref-java-natives__1.1.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-api-services-sts_shaded.jar":"System Classpath","/databricks/jars/spark--dbutils--dbutils-api-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.linecorp.armeria_armeria-grpc-protocol_com.linecorp.armeria__armeria-grpc-protocol__1.9.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--core--core-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--third_party--mssql--stax__stax__1.2.0_shaded.jar":"System Classpath","/databricks/jars/common--cloudstorage--presigned-url-spark_3.1_2.12_deploy.jar":"*********(redacted)","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hive.shims--hive-shims-scheduler--org.apache.hive.shims__hive-shims-scheduler__2.3.7.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.github.fommil.netlib--netlib-native_system-linux-x86_64-natives--com.github.fommil.netlib__netlib-native_system-linux-x86_64-natives__1.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--core--libcore_resources.jar":"System Classpath","/databricks/jars/third_party--gcp-java--j2objc-annotations_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--org.apache.htrace__htrace-core__3.1.0-incubating_2.12_shaded_20180625_3682417_spark_3.1.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--core--core-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/api-base--api-base-spark_3.1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-webapp--org.eclipse.jetty__jetty-webapp__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/third_party--gcp-java--proto-google-iam-v1_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--io.dropwizard.metrics--metrics-jmx--io.dropwizard.metrics__metrics-jmx__4.1.1.jar":"System Classpath","/databricks/jars/third_party--opencensus-shaded--commons-logging__commons-logging__1.2_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-emr--com.amazonaws__aws-java-sdk-emr__1.11.655.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--common--network-shuffle--network-shuffle-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/third_party--gcs-private--gcs-shaded-jetty9-hadoop1_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--sql--hive-thriftserver--hive-thriftserver-protocol-hive-2.3__hadoop-2.7_2.12_deploy.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.arrow--arrow-memory-netty--org.apache.arrow__arrow-memory-netty__2.0.0.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty--jetty-servlet--org.eclipse.jetty__jetty-servlet__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.directory.server--apacheds-kerberos-codec--org.apache.directory.server__apacheds-kerberos-codec__2.0.0-M15.jar":"System Classpath","/databricks/jars/third_party--gcp-java--google-oauth-client_shaded.jar":"System Classpath","/databricks/jars/third_party--hadoop-azure-2.7.3-abfs--com.google.guava__guava__11.0.2_2.12_shaded_20180920_b33d810_spark_3.1.jar":"System Classpath","/databricks/jars/third_party--armeria--maven-trees_armeria_com.google.protobuf_protobuf-java_com.google.protobuf__protobuf-java__3.14.0_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.apache.hadoop--hadoop-mapreduce-client-common--org.apache.hadoop__hadoop-mapreduce-client-common__2.7.4.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--org.eclipse.jetty.websocket--websocket-api--org.eclipse.jetty.websocket__websocket-api__9.4.42.v20210604.jar":"System Classpath","/databricks/jars/third_party--gcs-private--util-hadoop2_shaded.jar":"System Classpath","/databricks/jars/third_party--gcs-private--auto-value-annotations_shaded.jar":"System Classpath","/databricks/jars/----workspace_spark_3_1--maven-trees--hive-2.3__hadoop-2.7--com.amazonaws--aws-java-sdk-config--com.amazonaws__aws-java-sdk-config__1.11.655.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"Databricks Shell","App ID":"app-20220304222044-0000","Timestamp":1646432441343,"User":"root"} +{"Event":"SparkListenerExecutorAdded","Timestamp":1646432448005,"Executor ID":"0","Executor Info":{"Host":"10.77.255.181","Total Cores":16,"Log Urls":{"stdout":"http://10.77.255.181:40000/logPage/?appId=app-20220304222044-0000&executorId=0&logType=stdout","stderr":"http://10.77.255.181:40000/logPage/?appId=app-20220304222044-0000&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"10.77.255.181","Port":42251},"Maximum Memory":26104509235,"Timestamp":1646432448325,"Maximum Onheap Memory":26104509235,"Maximum Offheap Memory":0} +{"Event":"SparkListenerExecutorAdded","Timestamp":1646432448688,"Executor ID":"1","Executor Info":{"Host":"10.77.224.250","Total Cores":16,"Log Urls":{"stdout":"http://10.77.224.250:40000/logPage/?appId=app-20220304222044-0000&executorId=1&logType=stdout","stderr":"http://10.77.224.250:40000/logPage/?appId=app-20220304222044-0000&executorId=1&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"10.77.224.250","Port":35671},"Maximum Memory":26104509235,"Timestamp":1646432449041,"Maximum Onheap Memory":26104509235,"Maximum Offheap Memory":0} +{"Event":"SparkListenerExecutorAdded","Timestamp":1646432449339,"Executor ID":"2","Executor Info":{"Host":"10.77.253.227","Total Cores":16,"Log Urls":{"stdout":"http://10.77.253.227:40000/logPage/?appId=app-20220304222044-0000&executorId=2&logType=stdout","stderr":"http://10.77.253.227:40000/logPage/?appId=app-20220304222044-0000&executorId=2&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":"10.77.253.227","Port":37003},"Maximum Memory":26104509235,"Timestamp":1646432449680,"Maximum Onheap Memory":26104509235,"Maximum Offheap Memory":0} +{"Event":"SparkListenerExecutorAdded","Timestamp":1646432450668,"Executor ID":"3","Executor Info":{"Host":"10.77.234.35","Total Cores":16,"Log Urls":{"stdout":"http://10.77.234.35:40000/logPage/?appId=app-20220304222044-0000&executorId=3&logType=stdout","stderr":"http://10.77.234.35:40000/logPage/?appId=app-20220304222044-0000&executorId=3&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"3","Host":"10.77.234.35","Port":32967},"Maximum Memory":26104509235,"Timestamp":1646432451051,"Maximum Onheap Memory":26104509235,"Maximum Offheap Memory":0} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:964)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nCollectLimit (3)\n+- * Filter (2)\n +- Scan text (1)\n\n\n(1) Scan text \nOutput [1]: [value#2]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz/2009.gz, ... 9 entries]\nReadSchema: struct\n\n(2) Filter [codegen id : 1]\nInput [1]: [value#2]\nCondition : (length(trim(value#2, None)) > 0)\n\n(3) CollectLimit\nInput [1]: [value#2]\nArguments: 1\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 1","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter (length(trim(value#2, None)) > 0)","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan text ","simpleString":"FileScan text [value#2] Batched: false, DataFilters: [(length(trim(value#2, None)) > 0)], Format: Text, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz/2009.gz, s3://job-bank/ml-rec/i..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz/2009.gz, s3://job-bank/ml-rec/input-data/airlinedelay/gz/2010.gz]","ReadSchema":"struct","Format":"Text","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[(length(trim(value#2, None)) > 0)]"},"metrics":[{"name":"file sorting by size time","accumulatorId":21,"metricType":"timing"},{"name":"number of files read","accumulatorId":15,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":20,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":19,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":18,"metricType":"timing"},{"name":"metadata time","accumulatorId":16,"metricType":"timing"},{"name":"size of files read","accumulatorId":17,"metricType":"size"},{"name":"number of output rows","accumulatorId":14,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":13,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":12,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":10,"metricType":"sum"},{"name":"records read","accumulatorId":8,"metricType":"sum"},{"name":"local bytes read","accumulatorId":6,"metricType":"size"},{"name":"fetch wait time","accumulatorId":7,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4,"metricType":"size"},{"name":"local blocks read","accumulatorId":3,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":5,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":9,"metricType":"size"}]},"time":1646432477796} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[21,2],[15,10],[16,7],[17,1904190334]]} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1646432478937,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"csv at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_22\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:964)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"csv at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_22\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:964)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432479049,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"0","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1646432479209,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1646432479209,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432482539,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":18,"Name":"filesystem read time (sampled)","Update":"124","Value":"124","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":19,"Name":"filesystem read data size (sampled)","Update":"7760","Value":"7760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"filesystem read data size","Update":"7760","Value":"7760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":22,"Name":"internal.metrics.executorDeserializeTime","Update":431,"Value":431,"Internal":true,"Count Failed Values":true},{"ID":23,"Name":"internal.metrics.executorDeserializeCpuTime","Update":416176308,"Value":416176308,"Internal":true,"Count Failed Values":true},{"ID":24,"Name":"internal.metrics.executorRunTime","Update":2535,"Value":2535,"Internal":true,"Count Failed Values":true},{"ID":25,"Name":"internal.metrics.executorCpuTime","Update":1942473274,"Value":1942473274,"Internal":true,"Count Failed Values":true},{"ID":26,"Name":"internal.metrics.resultSize","Update":1576,"Value":1576,"Internal":true,"Count Failed Values":true},{"ID":27,"Name":"internal.metrics.jvmGCTime","Update":108,"Value":108,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.input.bytesRead","Update":7760,"Value":7760,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.input.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.input.sampledTimeReadNano","Update":124674828,"Value":124674828,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.input.sampledBytesRead","Update":7760,"Value":7760,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":196486376,"JVMOffHeapMemory":78231144,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":888434,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":888434,"OffHeapUnifiedMemory":0,"DirectPoolMemory":14688,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":5,"MinorGCTime":41,"MajorGCCount":3,"MajorGCTime":131},"Task Metrics":{"Executor Deserialize Time":431,"Executor Deserialize CPU Time":416176308,"Executor Run Time":2535,"Executor CPU Time":1942473274,"Peak Execution Memory":0,"Result Size":1576,"JVM GC Time":108,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":7760,"Records Read":1},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"csv at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_22\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:964)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432479049,"Completion Time":1646432482548,"Accumulables":[{"ID":13,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":14,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":18,"Name":"filesystem read time (sampled)","Value":"124","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":19,"Name":"filesystem read data size (sampled)","Value":"7760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":20,"Name":"filesystem read data size","Value":"7760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":22,"Name":"internal.metrics.executorDeserializeTime","Value":431,"Internal":true,"Count Failed Values":true},{"ID":23,"Name":"internal.metrics.executorDeserializeCpuTime","Value":416176308,"Internal":true,"Count Failed Values":true},{"ID":24,"Name":"internal.metrics.executorRunTime","Value":2535,"Internal":true,"Count Failed Values":true},{"ID":25,"Name":"internal.metrics.executorCpuTime","Value":1942473274,"Internal":true,"Count Failed Values":true},{"ID":26,"Name":"internal.metrics.resultSize","Value":1576,"Internal":true,"Count Failed Values":true},{"ID":27,"Name":"internal.metrics.jvmGCTime","Value":108,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.input.bytesRead","Value":7760,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.input.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.input.sampledTimeReadNano","Value":124674828,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.input.sampledBytesRead","Value":7760,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1646432482556,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1646432482660,"errorMessage":""} +{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1646432483227,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"csv at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"mapPartitions\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_29\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_30\",\"name\":\"DeserializeToObject\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"mapPartitions\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"SQLExecutionRDD","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_29\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:964)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"9\",\"name\":\"aggregate\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"csv at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"mapPartitions\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_29\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_30\",\"name\":\"DeserializeToObject\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"mapPartitions\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"SQLExecutionRDD","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_29\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:964)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432483232,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"9\",\"name\":\"aggregate\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1646432483302,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":1,"Attempt":0,"Launch Time":1646432483302,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":2,"Attempt":0,"Launch Time":1646432483302,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":3,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":4,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":5,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":6,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":7,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":8,"Attempt":0,"Launch Time":1646432483304,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":9,"Attempt":0,"Launch Time":1646432483304,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":6,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432511242,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"5819080","Value":"5819080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1548","Value":"1548","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":345,"Value":345,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":160679796,"Value":160679796,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":27524,"Value":27524,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":25716429146,"Value":25716429146,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1632,"Value":1632,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":284,"Value":284,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":5819080,"Value":5819080,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1548881311,"Value":1548881311,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3054549168,"JVMOffHeapMemory":113813520,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1610948,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1610948,"OffHeapUnifiedMemory":0,"DirectPoolMemory":359182,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":42,"MinorGCTime":203,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":345,"Executor Deserialize CPU Time":160679796,"Executor Run Time":27524,"Executor CPU Time":25716429146,"Peak Execution Memory":0,"Result Size":1632,"JVM GC Time":284,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819080},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":8,"Attempt":0,"Launch Time":1646432483304,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432512656,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"5674622","Value":"11493702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1699","Value":"3247","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"360454509","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"178217644","Value":"360454509","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":811,"Value":1156,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":420894961,"Value":581574757,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":28280,"Value":55804,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":26408321513,"Value":52124750659,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1632,"Value":3264,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":345,"Value":629,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":360454509,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":5674622,"Value":11493702,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1699138727,"Value":3248020038,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":360454509,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3963009096,"JVMOffHeapMemory":113183136,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1048737,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1048737,"OffHeapUnifiedMemory":0,"DirectPoolMemory":413191,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":36,"MinorGCTime":186,"MajorGCCount":4,"MajorGCTime":198},"Task Metrics":{"Executor Deserialize Time":811,"Executor Deserialize CPU Time":420894961,"Executor Run Time":28280,"Executor CPU Time":26408321513,"Peak Execution Memory":0,"Result Size":1632,"JVM GC Time":345,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674622},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":7,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432513286,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"5819812","Value":"17313514","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1402","Value":"4649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"541148352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"180693843","Value":"541148352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":791,"Value":1947,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":509454080,"Value":1091028837,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":28936,"Value":84740,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":27705917588,"Value":79830668247,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1632,"Value":4896,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":303,"Value":932,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":541148352,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":5819812,"Value":17313514,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1402793830,"Value":4650813868,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":541148352,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2957130312,"JVMOffHeapMemory":113133128,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1048737,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1048737,"OffHeapUnifiedMemory":0,"DirectPoolMemory":343541,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":30,"MinorGCTime":156,"MajorGCCount":4,"MajorGCTime":188},"Task Metrics":{"Executor Deserialize Time":791,"Executor Deserialize CPU Time":509454080,"Executor Run Time":28936,"Executor CPU Time":27705917588,"Peak Execution Memory":0,"Result Size":1632,"JVM GC Time":303,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819812},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":9,"Attempt":0,"Launch Time":1646432483304,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432513542,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"5617659","Value":"22931173","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1613","Value":"6262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"717081177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"175932825","Value":"717081177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":920,"Value":2867,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":502685186,"Value":1593714023,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":29030,"Value":113770,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":25697213793,"Value":105527882040,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1632,"Value":6528,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":377,"Value":1309,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":717081177,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":5617659,"Value":22931173,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1613585067,"Value":6264398935,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":717081177,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3156789392,"JVMOffHeapMemory":112969136,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1048737,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1048737,"OffHeapUnifiedMemory":0,"DirectPoolMemory":409093,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":32,"MinorGCTime":190,"MajorGCCount":4,"MajorGCTime":222},"Task Metrics":{"Executor Deserialize Time":920,"Executor Deserialize CPU Time":502685186,"Executor Run Time":29030,"Executor CPU Time":25697213793,"Peak Execution Memory":0,"Result Size":1632,"JVM GC Time":377,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617659},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":4,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432514030,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"6066651","Value":"28997824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1584","Value":"7846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"902117057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"185035880","Value":"902117057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":811,"Value":3678,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":429666967,"Value":2023380990,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":29663,"Value":143433,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":26758939014,"Value":132286821054,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1594,"Value":8122,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":345,"Value":1654,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":902117057,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":6066651,"Value":28997824,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1584143797,"Value":7848542732,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":902117057,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3963009096,"JVMOffHeapMemory":113183136,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1048737,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1048737,"OffHeapUnifiedMemory":0,"DirectPoolMemory":413191,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":36,"MinorGCTime":186,"MajorGCCount":4,"MajorGCTime":198},"Task Metrics":{"Executor Deserialize Time":811,"Executor Deserialize CPU Time":429666967,"Executor Run Time":29663,"Executor CPU Time":26758939014,"Peak Execution Memory":0,"Result Size":1594,"JVM GC Time":345,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066651},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":2,"Attempt":0,"Launch Time":1646432483302,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432514276,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"6450118","Value":"35447942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1633","Value":"9479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1098931357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"196814300","Value":"1098931357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":344,"Value":4022,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":268963827,"Value":2292344817,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":30564,"Value":173997,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":29847329325,"Value":162134150379,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1594,"Value":9716,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":288,"Value":1942,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1098931357,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":6450118,"Value":35447942,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1633177197,"Value":9481719929,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1098931357,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3054549168,"JVMOffHeapMemory":113813520,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1610948,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1610948,"OffHeapUnifiedMemory":0,"DirectPoolMemory":359182,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":42,"MinorGCTime":203,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":344,"Executor Deserialize CPU Time":268963827,"Executor Run Time":30564,"Executor CPU Time":29847329325,"Peak Execution Memory":0,"Result Size":1594,"JVM GC Time":288,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450118},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":3,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432515526,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"6429339","Value":"41877281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1576","Value":"11055","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1294005672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"195074315","Value":"1294005672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":791,"Value":4813,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":429393080,"Value":2721737897,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":31183,"Value":205180,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":28735465278,"Value":190869615657,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1594,"Value":11310,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":307,"Value":2249,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1294005672,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":6429339,"Value":41877281,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1576573948,"Value":11058293877,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1294005672,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2957130312,"JVMOffHeapMemory":113133128,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1048737,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1048737,"OffHeapUnifiedMemory":0,"DirectPoolMemory":344558,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":37,"MinorGCTime":180,"MajorGCCount":4,"MajorGCTime":188},"Task Metrics":{"Executor Deserialize Time":791,"Executor Deserialize CPU Time":429393080,"Executor Run Time":31183,"Executor CPU Time":28735465278,"Peak Execution Memory":0,"Result Size":1594,"JVM GC Time":307,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429339},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":5,"Attempt":0,"Launch Time":1646432483303,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432516399,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"6096763","Value":"47974044","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1612","Value":"12667","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"183939508","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":920,"Value":5733,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":416334421,"Value":3138072318,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":31897,"Value":237077,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":29811562608,"Value":220681178265,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1594,"Value":12904,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":385,"Value":2634,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":1477945180,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":6096763,"Value":47974044,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1612439072,"Value":12670732949,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":1477945180,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6046474248,"JVMOffHeapMemory":113323544,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1048737,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1048737,"OffHeapUnifiedMemory":0,"DirectPoolMemory":409661,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":40,"MinorGCTime":223,"MajorGCCount":4,"MajorGCTime":222},"Task Metrics":{"Executor Deserialize Time":920,"Executor Deserialize CPU Time":416334421,"Executor Run Time":31897,"Executor CPU Time":29811562608,"Peak Execution Memory":0,"Result Size":1594,"JVM GC Time":385,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096763},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":1,"Attempt":0,"Launch Time":1646432483302,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432516740,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"6369483","Value":"54343527","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1812","Value":"14479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":920,"Value":6653,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":451841079,"Value":3589913397,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":32239,"Value":269316,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":28943853374,"Value":249625031639,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1594,"Value":14498,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":385,"Value":3019,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1675583714,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":6369483,"Value":54343527,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1812477312,"Value":14483210261,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1675583714,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6046474248,"JVMOffHeapMemory":113323544,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1048737,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1048737,"OffHeapUnifiedMemory":0,"DirectPoolMemory":409661,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":40,"MinorGCTime":223,"MajorGCCount":4,"MajorGCTime":222},"Task Metrics":{"Executor Deserialize Time":920,"Executor Deserialize CPU Time":451841079,"Executor Run Time":32239,"Executor CPU Time":28943853374,"Peak Execution Memory":0,"Result Size":1594,"JVM GC Time":385,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369483},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1646432483302,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432518929,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"7213447","Value":"61556974","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Update":"1833","Value":"16312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Update":811,"Value":7464,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Update":382180849,"Value":3972094246,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Update":34561,"Value":303877,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Update":31762724099,"Value":281387755738,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Update":1594,"Value":16092,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Update":353,"Value":3372,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Update":7213447,"Value":61556974,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1833541282,"Value":16316751543,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3963009096,"JVMOffHeapMemory":113276560,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1048737,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1048737,"OffHeapUnifiedMemory":0,"DirectPoolMemory":413758,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":43,"MinorGCTime":211,"MajorGCCount":4,"MajorGCTime":198},"Task Metrics":{"Executor Deserialize Time":811,"Executor Deserialize CPU Time":382180849,"Executor Run Time":34561,"Executor CPU Time":31762724099,"Peak Execution Memory":0,"Result Size":1594,"JVM GC Time":353,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213447},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"csv at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"mapPartitions\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_29\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_30\",\"name\":\"DeserializeToObject\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"mapPartitions\"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"SQLExecutionRDD","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_29\",\"name\":\"Scan text \"}","Callsite":"csv at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:964)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432483232,"Completion Time":1646432518930,"Accumulables":[{"ID":52,"Name":"number of output rows","Value":"61556974","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"filesystem read time (sampled)","Value":"16312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":58,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"internal.metrics.executorDeserializeTime","Value":7464,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.executorDeserializeCpuTime","Value":3972094246,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.executorRunTime","Value":303877,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.executorCpuTime","Value":281387755738,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.resultSize","Value":16092,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.jvmGCTime","Value":3372,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.resultSerializationTime","Value":9,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.input.recordsRead","Value":61556974,"Internal":true,"Count Failed Values":true},{"ID":85,"Name":"internal.metrics.input.sampledTimeReadNano","Value":16316751543,"Internal":true,"Count Failed Values":true},{"ID":86,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1646432518930,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.Dataset.createOrReplaceTempView(Dataset.scala:3394)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateViewCommand (1)\n +- CreateViewCommand (2)\n +- Project (4)\n +- LogicalRelation (3)\n\n\n(1) Execute CreateViewCommand\nOutput: []\n\n(2) CreateViewCommand\nArguments: `input`, false, true, LocalTempView\n\n(3) LogicalRelation\nArguments: csv, [FL_DATE#18, OP_CARRIER#19, OP_CARRIER_FL_NUM#20, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_TIME#24, DEP_DELAY#25, TAXI_OUT#26, WHEELS_OFF#27, WHEELS_ON#28, TAXI_IN#29, CRS_ARR_TIME#30, ARR_TIME#31, ARR_DELAY#32, CANCELLED#33, CANCELLATION_CODE#34, DIVERTED#35, CRS_ELAPSED_TIME#36, ACTUAL_ELAPSED_TIME#37, AIR_TIME#38, DISTANCE#39, CARRIER_DELAY#40, WEATHER_DELAY#41, ... 4 more fields], false\n\n(4) Project\nArguments: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\n\n","sparkPlanInfo":{"nodeName":"Execute CreateViewCommand","simpleString":"Execute CreateViewCommand","children":[],"metadata":{},"metrics":[]},"time":1646432519104} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1646432519114,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.Dataset.createOrReplaceTempView(Dataset.scala:3394)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateViewCommand (1)\n +- CreateViewCommand (2)\n +- Project (4)\n +- LogicalRelation (3)\n\n\n(1) Execute CreateViewCommand\nOutput: []\n\n(2) CreateViewCommand\nArguments: `input`, false, true, LocalTempView\n\n(3) LogicalRelation\nArguments: csv, [FL_DATE#18, OP_CARRIER#19, OP_CARRIER_FL_NUM#20, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_TIME#24, DEP_DELAY#25, TAXI_OUT#26, WHEELS_OFF#27, WHEELS_ON#28, TAXI_IN#29, CRS_ARR_TIME#30, ARR_TIME#31, ARR_DELAY#32, CANCELLED#33, CANCELLATION_CODE#34, DIVERTED#35, CRS_ELAPSED_TIME#36, ACTUAL_ELAPSED_TIME#37, AIR_TIME#38, DISTANCE#39, CARRIER_DELAY#40, WEATHER_DELAY#41, ... 4 more fields], false\n\n(4) Project\nArguments: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\n\n","sparkPlanInfo":{"nodeName":"Execute CreateViewCommand","simpleString":"Execute CreateViewCommand","children":[],"metadata":{},"metrics":[]},"time":1646432519135} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1646432519144,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3097)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (5)\n+- HashAggregate (4)\n +- Exchange (3)\n +- HashAggregate (2)\n +- Scan csv (1)\n\n\n(1) Scan csv \nOutput: []\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct<>\n\n(2) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1) AS count#90L]\nAggregate Attributes [1]: [count#89L]\nResults [1]: [count#90L]\n\n(3) Exchange\nInput [1]: [count#90L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#52]\n\n(4) HashAggregate\nInput [1]: [count#90L]\nKeys: []\nFunctions [1]: [finalmerge_count(merge count#90L) AS count(1)#86L]\nAggregate Attributes [1]: [count(1)#86L]\nResults [1]: [count(1)#86L AS count#87L]\n\n(5) AdaptiveSparkPlan\nOutput [1]: [count#87L]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[finalmerge_count(merge count#90L) AS count(1)#86L])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#52]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1) AS count#90L])","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct<>","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},"time":1646432519413} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":3,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (9)\n+- == Current Plan ==\n HashAggregate (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * HashAggregate (2)\n +- Scan csv (1)\n+- == Initial Plan ==\n HashAggregate (8)\n +- Exchange (7)\n +- HashAggregate (6)\n +- Scan csv (1)\n\n\n(1) Scan csv \nOutput: []\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1) AS count#90L]\nAggregate Attributes [1]: [count#89L]\nResults [1]: [count#90L]\n\n(3) Exchange\nInput [1]: [count#90L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#94]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#90L]\nArguments: 0, Statistics(sizeInBytes=3.5 GiB)\n\n(5) HashAggregate\nInput [1]: [count#90L]\nKeys: []\nFunctions [1]: [finalmerge_count(merge count#90L) AS count(1)#86L]\nAggregate Attributes [1]: [count(1)#86L]\nResults [1]: [count(1)#86L AS count#87L]\n\n(6) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1) AS count#90L]\nAggregate Attributes [1]: [count#89L]\nResults [1]: [count#90L]\n\n(7) Exchange\nInput [1]: [count#90L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#52]\n\n(8) HashAggregate\nInput [1]: [count#90L]\nKeys: []\nFunctions [1]: [finalmerge_count(merge count#90L) AS count(1)#86L]\nAggregate Attributes [1]: [count(1)#86L]\nResults [1]: [count(1)#86L AS count#87L]\n\n(9) AdaptiveSparkPlan\nOutput [1]: [count#87L]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[finalmerge_count(merge count#90L) AS count(1)#86L])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=3.5 GiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#94]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1) AS count#90L])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct<>","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":118,"metricType":"timing"},{"name":"number of files read","accumulatorId":112,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":117,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":116,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":115,"metricType":"timing"},{"name":"metadata time","accumulatorId":113,"metricType":"timing"},{"name":"size of files read","accumulatorId":114,"metricType":"size"},{"name":"number of output rows","accumulatorId":111,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":133,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":134,"metricType":"timing"},{"name":"peak memory","accumulatorId":132,"metricType":"size"},{"name":"number of output rows","accumulatorId":131,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":135,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":130,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":128,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":129,"metricType":"nsTiming"},{"name":"records read","accumulatorId":126,"metricType":"sum"},{"name":"local bytes read","accumulatorId":124,"metricType":"size"},{"name":"fetch wait time","accumulatorId":125,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":122,"metricType":"size"},{"name":"local blocks read","accumulatorId":121,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":120,"metricType":"sum"},{"name":"data size","accumulatorId":119,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":123,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":127,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":3,"accumUpdates":[[118,0],[112,10],[113,0],[114,1904190334]]} +{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1646432519595,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_94\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_48\",\"name\":\"Scan csv \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_48\",\"name\":\"Scan csv \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_93\",\"name\":\"WholeStageCodegen\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3097)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[2],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_94\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"3","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_94\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_48\",\"name\":\"Scan csv \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_48\",\"name\":\"Scan csv \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_93\",\"name\":\"WholeStageCodegen\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3097)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432519604,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_94\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"3","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Launch Time":1646432519633,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":1,"Attempt":0,"Launch Time":1646432519634,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":2,"Attempt":0,"Launch Time":1646432519634,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":3,"Attempt":0,"Launch Time":1646432519634,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":4,"Attempt":0,"Launch Time":1646432519634,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":5,"Attempt":0,"Launch Time":1646432519635,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":6,"Attempt":0,"Launch Time":1646432519635,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":7,"Attempt":0,"Launch Time":1646432519635,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":8,"Attempt":0,"Launch Time":1646432519635,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":9,"Attempt":0,"Launch Time":1646432519636,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":8,"Attempt":0,"Launch Time":1646432519635,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432526677,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"5674621","Value":"5674621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1194","Value":"1194","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"178217644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"178217644","Value":"178217644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"9155499","Value":"9155499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"6889","Value":"6889","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"6870","Value":"6870","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":44,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19611517,"Value":19611517,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":6962,"Value":6962,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":6747032624,"Value":6747032624,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1544,"Value":1544,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":9155499,"Value":9155499,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":178217644,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":5674621,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1194108829,"Value":1194108829,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":178217644,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1271446480,"JVMOffHeapMemory":116353496,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":479726,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":39,"MinorGCTime":189,"MajorGCCount":4,"MajorGCTime":188},"Task Metrics":{"Executor Deserialize Time":44,"Executor Deserialize CPU Time":19611517,"Executor Run Time":6962,"Executor CPU Time":6747032624,"Peak Execution Memory":0,"Result Size":1544,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":9155499,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":9,"Attempt":0,"Launch Time":1646432519636,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432526703,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"5617658","Value":"11292279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1255","Value":"2449","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"175932825","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"10966587","Value":"20122086","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"6880","Value":"13769","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"6857","Value":"13727","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":57,"Value":101,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":27677571,"Value":47289088,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":6979,"Value":13941,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":6779499107,"Value":13526531731,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1543,"Value":3087,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":11,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":118,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":10966587,"Value":20122086,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":354150469,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":11292279,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1255916462,"Value":2450025291,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":354150469,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2011857544,"JVMOffHeapMemory":116689632,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":417853,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":43,"MinorGCTime":237,"MajorGCCount":4,"MajorGCTime":222},"Task Metrics":{"Executor Deserialize Time":57,"Executor Deserialize CPU Time":27677571,"Executor Run Time":6979,"Executor CPU Time":6779499107,"Peak Execution Memory":0,"Result Size":1543,"JVM GC Time":11,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":10966587,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":4,"Attempt":0,"Launch Time":1646432519634,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432526903,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"6066650","Value":"17358929","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1275","Value":"3724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"539186349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"185035880","Value":"539186349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"919357","Value":"21041443","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"7129","Value":"20898","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"7115","Value":"20842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":44,"Value":145,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":38550206,"Value":85839294,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":7191,"Value":21132,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":6999571263,"Value":20526102994,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1544,"Value":4631,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":9,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":177,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":919357,"Value":21041443,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":539186349,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":17358929,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1275026951,"Value":3725052242,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":539186349,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1271446480,"JVMOffHeapMemory":116353496,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":479726,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":39,"MinorGCTime":189,"MajorGCCount":4,"MajorGCTime":188},"Task Metrics":{"Executor Deserialize Time":44,"Executor Deserialize CPU Time":38550206,"Executor Run Time":7191,"Executor CPU Time":6999571263,"Peak Execution Memory":0,"Result Size":1544,"JVM GC Time":9,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":919357,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":6,"Attempt":0,"Launch Time":1646432519635,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432526934,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"5819079","Value":"23178008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1361","Value":"5085","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"721423214","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"182236865","Value":"721423214","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"9499062","Value":"30540505","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"7145","Value":"28043","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"7125","Value":"27967","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":191,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":37184384,"Value":123023678,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":7221,"Value":28353,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":7068011949,"Value":27594114943,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1544,"Value":6175,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":5,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":236,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":9499062,"Value":30540505,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":721423214,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":23178008,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1361610764,"Value":5086663006,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":721423214,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2577035128,"JVMOffHeapMemory":116161456,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":417854,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":45,"MinorGCTime":220,"MajorGCCount":4,"MajorGCTime":198},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":37184384,"Executor Run Time":7221,"Executor CPU Time":7068011949,"Peak Execution Memory":0,"Result Size":1544,"JVM GC Time":5,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":9499062,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":7,"Attempt":0,"Launch Time":1646432519635,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432527128,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"5819811","Value":"28997819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1485","Value":"6570","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"902117057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"180693843","Value":"902117057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"295","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"9451138","Value":"39991643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"7372","Value":"35415","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"7354","Value":"35321","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":40,"Value":231,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":25627245,"Value":148650923,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":7420,"Value":35773,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":7070999136,"Value":34665114079,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1544,"Value":7719,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":8,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":295,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":9451138,"Value":39991643,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":902117057,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":28997819,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1485808937,"Value":6572471943,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":902117057,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1554807328,"JVMOffHeapMemory":115988680,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":363334,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":44,"MinorGCTime":211,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":40,"Executor Deserialize CPU Time":25627245,"Executor Run Time":7420,"Executor CPU Time":7070999136,"Peak Execution Memory":0,"Result Size":1544,"JVM GC Time":8,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":9451138,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":5,"Attempt":0,"Launch Time":1646432519635,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432527263,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"6096762","Value":"35094581","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1305","Value":"7875","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1086056565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"183939508","Value":"1086056565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"96","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"354","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"1182949","Value":"41174592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"7456","Value":"42871","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"7438","Value":"42759","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":57,"Value":288,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":27423984,"Value":176074907,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":7541,"Value":43314,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":7295476109,"Value":41960590188,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1543,"Value":9262,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":11,"Value":53,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":354,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":1182949,"Value":41174592,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":1086056565,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":35094581,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1305156878,"Value":7877628821,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":1086056565,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2011857544,"JVMOffHeapMemory":116689632,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":417853,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":43,"MinorGCTime":237,"MajorGCCount":4,"MajorGCTime":222},"Task Metrics":{"Executor Deserialize Time":57,"Executor Deserialize CPU Time":27423984,"Executor Run Time":7541,"Executor CPU Time":7295476109,"Peak Execution Memory":0,"Result Size":1543,"JVM GC Time":11,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":1182949,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":2,"Attempt":0,"Launch Time":1646432519634,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432527441,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"6450117","Value":"41544698","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1344","Value":"9219","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1282870865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"196814300","Value":"1282870865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"660292","Value":"41834884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"7664","Value":"50535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"7649","Value":"50408","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":334,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":24668069,"Value":200742976,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":7730,"Value":51044,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":7567945911,"Value":49528536099,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1544,"Value":10806,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":5,"Value":58,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":413,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":660292,"Value":41834884,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1282870865,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":41544698,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1344559140,"Value":9222187961,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1282870865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2577035128,"JVMOffHeapMemory":116161456,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":417854,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":45,"MinorGCTime":220,"MajorGCCount":4,"MajorGCTime":198},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":24668069,"Executor Run Time":7730,"Executor CPU Time":7567945911,"Peak Execution Memory":0,"Result Size":1544,"JVM GC Time":5,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":660292,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":3,"Attempt":0,"Launch Time":1646432519634,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432527614,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"6429338","Value":"47974036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1364","Value":"10583","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"195074315","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"740051","Value":"42574935","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"7870","Value":"58405","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"7857","Value":"58265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":40,"Value":374,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":26504465,"Value":227247441,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":7907,"Value":58951,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":7750999612,"Value":57279535711,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1544,"Value":12350,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":70,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":472,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":740051,"Value":42574935,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1477945180,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":47974036,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1364849099,"Value":10587037060,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1477945180,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1554807328,"JVMOffHeapMemory":115988680,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":363334,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":44,"MinorGCTime":211,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":40,"Executor Deserialize CPU Time":26504465,"Executor Run Time":7907,"Executor CPU Time":7750999612,"Peak Execution Memory":0,"Result Size":1544,"JVM GC Time":12,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":740051,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":1,"Attempt":0,"Launch Time":1646432519634,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432528104,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"6369482","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1558","Value":"12141","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"531","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"545463","Value":"43120398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"8296","Value":"66701","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"8279","Value":"66544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":57,"Value":431,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":38902086,"Value":266149527,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":8380,"Value":67331,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":8222178271,"Value":65501713982,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1543,"Value":13893,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":11,"Value":81,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":531,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":545463,"Value":43120398,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1675583714,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":54343518,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1558564086,"Value":12145601146,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1675583714,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2011857544,"JVMOffHeapMemory":116689632,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":417853,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":43,"MinorGCTime":237,"MajorGCCount":4,"MajorGCTime":222},"Task Metrics":{"Executor Deserialize Time":57,"Executor Deserialize CPU Time":38902086,"Executor Run Time":8380,"Executor CPU Time":8222178271,"Peak Execution Memory":0,"Result Size":1543,"JVM GC Time":11,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":545463,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Launch Time":1646432519633,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432528701,"Failed":false,"Killed":false,"Accumulables":[{"ID":111,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Update":"1578","Value":"13719","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Update":"16","Value":"160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Update":"59","Value":"590","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Update":"569359","Value":"43689757","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Update":"8926","Value":"75627","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Update":"8912","Value":"75456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Update":44,"Value":475,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19234282,"Value":285383809,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Update":8989,"Value":76320,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Update":8869035669,"Value":74370749651,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Update":1544,"Value":15437,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Update":13,"Value":94,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":590,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Update":569359,"Value":43689757,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1578094207,"Value":13723695353,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1271446480,"JVMOffHeapMemory":116353496,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":1739065,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1739065,"OffHeapUnifiedMemory":0,"DirectPoolMemory":479726,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":39,"MinorGCTime":189,"MajorGCCount":4,"MajorGCTime":188},"Task Metrics":{"Executor Deserialize Time":44,"Executor Deserialize CPU Time":19234282,"Executor Run Time":8989,"Executor CPU Time":8869035669,"Peak Execution Memory":0,"Result Size":1544,"JVM GC Time":13,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":569359,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_94\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_48\",\"name\":\"Scan csv \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_48\",\"name\":\"Scan csv \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_93\",\"name\":\"WholeStageCodegen\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3097)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432519604,"Completion Time":1646432528701,"Accumulables":[{"ID":111,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":115,"Name":"filesystem read time (sampled)","Value":"13719","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":116,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":117,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":119,"Name":"data size","Value":"160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":127,"Name":"shuffle bytes written","Value":"590","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"shuffle records written","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"shuffle write time","Value":"43689757","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":130,"Name":"duration","Value":"75627","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":131,"Name":"number of output rows","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":134,"Name":"time in aggregation build","Value":"75456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":136,"Name":"internal.metrics.executorDeserializeTime","Value":475,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeCpuTime","Value":285383809,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorRunTime","Value":76320,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorCpuTime","Value":74370749651,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.resultSize","Value":15437,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.jvmGCTime","Value":94,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":590,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":10,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.shuffle.write.writeTime","Value":43689757,"Internal":true,"Count Failed Values":true},{"ID":159,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":160,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":161,"Name":"internal.metrics.input.sampledTimeReadNano","Value":13723695353,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1646432528708,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":3,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (9)\n+- == Final Plan ==\n * HashAggregate (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * HashAggregate (2)\n +- Scan csv (1)\n+- == Initial Plan ==\n HashAggregate (8)\n +- Exchange (7)\n +- HashAggregate (6)\n +- Scan csv (1)\n\n\n(1) Scan csv \nOutput: []\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1) AS count#90L]\nAggregate Attributes [1]: [count#89L]\nResults [1]: [count#90L]\n\n(3) Exchange\nInput [1]: [count#90L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#94]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#90L]\nArguments: 0, Statistics(sizeInBytes=160.0 B, rowCount=10, isRuntime=true)\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [count#90L]\nKeys: []\nFunctions [1]: [finalmerge_count(merge count#90L) AS count(1)#86L]\nAggregate Attributes [1]: [count(1)#86L]\nResults [1]: [count(1)#86L AS count#87L]\n\n(6) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1) AS count#90L]\nAggregate Attributes [1]: [count#89L]\nResults [1]: [count#90L]\n\n(7) Exchange\nInput [1]: [count#90L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#52]\n\n(8) HashAggregate\nInput [1]: [count#90L]\nKeys: []\nFunctions [1]: [finalmerge_count(merge count#90L) AS count(1)#86L]\nAggregate Attributes [1]: [count(1)#86L]\nResults [1]: [count(1)#86L AS count#87L]\n\n(9) AdaptiveSparkPlan\nOutput [1]: [count#87L]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[finalmerge_count(merge count#90L) AS count(1)#86L])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=160.0 B, rowCount=10, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#94]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1) AS count#90L])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct<>","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":118,"metricType":"timing"},{"name":"number of files read","accumulatorId":112,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":117,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":116,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":115,"metricType":"timing"},{"name":"metadata time","accumulatorId":113,"metricType":"timing"},{"name":"size of files read","accumulatorId":114,"metricType":"size"},{"name":"number of output rows","accumulatorId":111,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":133,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":134,"metricType":"timing"},{"name":"peak memory","accumulatorId":132,"metricType":"size"},{"name":"number of output rows","accumulatorId":131,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":135,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":130,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":128,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":129,"metricType":"nsTiming"},{"name":"records read","accumulatorId":126,"metricType":"sum"},{"name":"local bytes read","accumulatorId":124,"metricType":"size"},{"name":"fetch wait time","accumulatorId":125,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":122,"metricType":"size"},{"name":"local blocks read","accumulatorId":121,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":120,"metricType":"sum"},{"name":"data size","accumulatorId":119,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":123,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":127,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":169,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":170,"metricType":"timing"},{"name":"peak memory","accumulatorId":168,"metricType":"size"},{"name":"number of output rows","accumulatorId":167,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":171,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":166,"metricType":"timing"}]}],"metadata":{},"metrics":[]}} +{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1646432528775,"Stage Infos":[{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_94\",\"name\":\"Exchange\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_48\",\"name\":\"Scan csv \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_48\",\"name\":\"Scan csv \"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_93\",\"name\":\"WholeStageCodegen\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3097)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_114\",\"name\":\"WholeStageCodegen\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_0\",\"name\":\"ShuffleQueryStage\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3097)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[3,4],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"3","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_114\",\"name\":\"WholeStageCodegen\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_0\",\"name\":\"ShuffleQueryStage\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3097)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432528782,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"3","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Launch Time":1646432528789,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Launch Time":1646432528789,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432528941,"Failed":false,"Killed":false,"Accumulables":[{"ID":120,"Name":"remote blocks read","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":121,"Name":"local blocks read","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":122,"Name":"remote bytes read","Update":"472","Value":"472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":124,"Name":"local bytes read","Update":"118","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":125,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":126,"Name":"records read","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":172,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10649000,"Value":10649000,"Internal":true,"Count Failed Values":true},{"ID":174,"Name":"internal.metrics.executorRunTime","Update":120,"Value":120,"Internal":true,"Count Failed Values":true},{"ID":175,"Name":"internal.metrics.executorCpuTime","Update":98407025,"Value":98407025,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.resultSize","Update":1762,"Value":1762,"Internal":true,"Count Failed Values":true},{"ID":185,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":186,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":472,"Value":472,"Internal":true,"Count Failed Values":true},{"ID":188,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":118,"Value":118,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.recordsRead","Update":10,"Value":10,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":10649000,"Executor Run Time":120,"Executor CPU Time":98407025,"Peak Execution Memory":0,"Result Size":1762,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":472,"Remote Bytes Read To Disk":0,"Local Bytes Read":118,"Total Records Read":10},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"count at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_114\",\"name\":\"WholeStageCodegen\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_0\",\"name\":\"ShuffleQueryStage\"}","Callsite":"count at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3097)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432528782,"Completion Time":1646432528942,"Accumulables":[{"ID":120,"Name":"remote blocks read","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":121,"Name":"local blocks read","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":122,"Name":"remote bytes read","Value":"472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":124,"Name":"local bytes read","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":125,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":126,"Name":"records read","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":166,"Name":"duration","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":167,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"time in aggregation build","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":172,"Name":"internal.metrics.executorDeserializeTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.executorDeserializeCpuTime","Value":10649000,"Internal":true,"Count Failed Values":true},{"ID":174,"Name":"internal.metrics.executorRunTime","Value":120,"Internal":true,"Count Failed Values":true},{"ID":175,"Name":"internal.metrics.executorCpuTime","Value":98407025,"Internal":true,"Count Failed Values":true},{"ID":176,"Name":"internal.metrics.resultSize","Value":1762,"Internal":true,"Count Failed Values":true},{"ID":185,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":8,"Internal":true,"Count Failed Values":true},{"ID":186,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":472,"Internal":true,"Count Failed Values":true},{"ID":188,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":118,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.recordsRead","Value":10,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1646432528944,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1646432528951,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":4,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nCollectLimit (3)\n+- * Project (2)\n +- Scan csv (1)\n\n\n(1) Scan csv \nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(2) Project [codegen id : 1]\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#101, cast(DEP_DELAY#25 as string) AS DEP_DELAY#102]\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\n\n(3) CollectLimit\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#101, DEP_DELAY#102]\nArguments: 11\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 11","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#101, cast(DEP_DELAY#25 as string) AS DEP_DELAY#102]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [FL_DATE#18,OP_CARRIER#19,ORIGIN#21,DEST#22,CRS_DEP_TIME#23,DEP_DELAY#25] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":220,"metricType":"timing"},{"name":"number of files read","accumulatorId":214,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":219,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":218,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":217,"metricType":"timing"},{"name":"metadata time","accumulatorId":215,"metricType":"timing"},{"name":"size of files read","accumulatorId":216,"metricType":"size"},{"name":"number of output rows","accumulatorId":213,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":212,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":210,"metricType":"sum"},{"name":"records read","accumulatorId":208,"metricType":"sum"},{"name":"local bytes read","accumulatorId":206,"metricType":"size"},{"name":"fetch wait time","accumulatorId":207,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":204,"metricType":"size"},{"name":"local blocks read","accumulatorId":203,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":202,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":205,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":209,"metricType":"size"}]},"time":1646432529010} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[220,0],[214,10],[215,0],[216,1904190334]]} +{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1646432529063,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_137\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_120\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_120\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[5],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"4","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_137\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_120\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_120\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432529066,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"4","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Launch Time":1646432529080,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Launch Time":1646432529080,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432529255,"Failed":false,"Killed":false,"Accumulables":[{"ID":213,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":217,"Name":"filesystem read time (sampled)","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":218,"Name":"filesystem read data size (sampled)","Update":"1616","Value":"1616","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":219,"Name":"filesystem read data size","Update":"1616","Value":"1616","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":221,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":222,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10098402,"Value":10098402,"Internal":true,"Count Failed Values":true},{"ID":223,"Name":"internal.metrics.executorRunTime","Update":140,"Value":140,"Internal":true,"Count Failed Values":true},{"ID":224,"Name":"internal.metrics.executorCpuTime","Update":86117088,"Value":86117088,"Internal":true,"Count Failed Values":true},{"ID":225,"Name":"internal.metrics.resultSize","Update":1612,"Value":1612,"Internal":true,"Count Failed Values":true},{"ID":244,"Name":"internal.metrics.input.bytesRead","Update":1616,"Value":1616,"Internal":true,"Count Failed Values":true},{"ID":245,"Name":"internal.metrics.input.recordsRead","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":246,"Name":"internal.metrics.input.sampledTimeReadNano","Update":23033366,"Value":23033366,"Internal":true,"Count Failed Values":true},{"ID":247,"Name":"internal.metrics.input.sampledBytesRead","Update":1616,"Value":1616,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":10098402,"Executor Run Time":140,"Executor CPU Time":86117088,"Peak Execution Memory":0,"Result Size":1612,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":1616,"Records Read":11},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_137\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_120\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_120\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432529066,"Completion Time":1646432529256,"Accumulables":[{"ID":213,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":217,"Name":"filesystem read time (sampled)","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":218,"Name":"filesystem read data size (sampled)","Value":"1616","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":219,"Name":"filesystem read data size","Value":"1616","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":221,"Name":"internal.metrics.executorDeserializeTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":222,"Name":"internal.metrics.executorDeserializeCpuTime","Value":10098402,"Internal":true,"Count Failed Values":true},{"ID":223,"Name":"internal.metrics.executorRunTime","Value":140,"Internal":true,"Count Failed Values":true},{"ID":224,"Name":"internal.metrics.executorCpuTime","Value":86117088,"Internal":true,"Count Failed Values":true},{"ID":225,"Name":"internal.metrics.resultSize","Value":1612,"Internal":true,"Count Failed Values":true},{"ID":244,"Name":"internal.metrics.input.bytesRead","Value":1616,"Internal":true,"Count Failed Values":true},{"ID":245,"Name":"internal.metrics.input.recordsRead","Value":11,"Internal":true,"Count Failed Values":true},{"ID":246,"Name":"internal.metrics.input.sampledTimeReadNano","Value":23033366,"Internal":true,"Count Failed Values":true},{"ID":247,"Name":"internal.metrics.input.sampledBytesRead","Value":1616,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1646432529259,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":4,"time":1646432529303,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":5,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.Dataset.createOrReplaceTempView(Dataset.scala:3394)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateViewCommand (1)\n +- CreateViewCommand (2)\n +- Window (8)\n +- Project (7)\n +- Deduplicate (6)\n +- Project (5)\n +- Project (4)\n +- LogicalRelation (3)\n\n\n(1) Execute CreateViewCommand\nOutput: []\n\n(2) CreateViewCommand\nArguments: `carriers`, false, true, LocalTempView\n\n(3) LogicalRelation\nArguments: csv, [FL_DATE#18, OP_CARRIER#19, OP_CARRIER_FL_NUM#20, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_TIME#24, DEP_DELAY#25, TAXI_OUT#26, WHEELS_OFF#27, WHEELS_ON#28, TAXI_IN#29, CRS_ARR_TIME#30, ARR_TIME#31, ARR_DELAY#32, CANCELLED#33, CANCELLATION_CODE#34, DIVERTED#35, CRS_ELAPSED_TIME#36, ACTUAL_ELAPSED_TIME#37, AIR_TIME#38, DISTANCE#39, CARRIER_DELAY#40, WEATHER_DELAY#41, ... 4 more fields], false\n\n(4) Project\nArguments: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\n\n(5) Project\nArguments: [OP_CARRIER#19]\n\n(6) Deduplicate\nArguments: [OP_CARRIER#19]\n\n(7) Project\nArguments: [OP_CARRIER#19]\n\n(8) Window\nArguments: [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST]\n\n","sparkPlanInfo":{"nodeName":"Execute CreateViewCommand","simpleString":"Execute CreateViewCommand","children":[],"metadata":{},"metrics":[]},"time":1646432529422} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":5,"time":1646432529424,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":6,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (9)\n+- TakeOrderedAndProject (8)\n +- RunningWindowFunction (7)\n +- Sort (6)\n +- Exchange (5)\n +- HashAggregate (4)\n +- Exchange (3)\n +- HashAggregate (2)\n +- Scan csv (1)\n\n\n(1) Scan csv \nOutput [1]: [OP_CARRIER#19]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(2) HashAggregate\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(3) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#161]\n\n(4) HashAggregate\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(5) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#164]\n\n(6) Sort\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(7) RunningWindowFunction\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false\n\n(8) TakeOrderedAndProject\nInput [2]: [OP_CARRIER#19, carrier_id#123]\nArguments: 21, [OP_CARRIER#19 ASC NULLS FIRST], [OP_CARRIER#19, cast(carrier_id#123 as string) AS carrier_id#130]\n\n(9) AdaptiveSparkPlan\nOutput [2]: [OP_CARRIER#19, carrier_id#130]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"TakeOrderedAndProject","simpleString":"TakeOrderedAndProject(limit=21, orderBy=[OP_CARRIER#19 ASC NULLS FIRST], output=[OP_CARRIER#19,carrier_id#130])","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#19 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#164]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#19], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#161]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#19], functions=[])","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#19] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},"time":1646432529683} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":6,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (17)\n+- == Current Plan ==\n TakeOrderedAndProject (9)\n +- RunningWindowFunction (8)\n +- Sort (7)\n +- Exchange (6)\n +- HashAggregate (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * HashAggregate (2)\n +- Scan csv (1)\n+- == Initial Plan ==\n TakeOrderedAndProject (16)\n +- RunningWindowFunction (15)\n +- Sort (14)\n +- Exchange (13)\n +- HashAggregate (12)\n +- Exchange (11)\n +- HashAggregate (10)\n +- Scan csv (1)\n\n\n(1) Scan csv \nOutput [1]: [OP_CARRIER#19]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(2) HashAggregate [codegen id : 1]\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(3) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#223]\n\n(4) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#19]\nArguments: 0, Statistics(sizeInBytes=1816.0 MiB)\n\n(5) HashAggregate\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(6) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#233]\n\n(7) Sort\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(8) RunningWindowFunction\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false\n\n(9) TakeOrderedAndProject\nInput [2]: [OP_CARRIER#19, carrier_id#123]\nArguments: 21, [OP_CARRIER#19 ASC NULLS FIRST], [OP_CARRIER#19, cast(carrier_id#123 as string) AS carrier_id#130]\n\n(10) HashAggregate\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(11) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#161]\n\n(12) HashAggregate\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(13) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#164]\n\n(14) Sort\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(15) RunningWindowFunction\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false\n\n(16) TakeOrderedAndProject\nInput [2]: [OP_CARRIER#19, carrier_id#123]\nArguments: 21, [OP_CARRIER#19 ASC NULLS FIRST], [OP_CARRIER#19, cast(carrier_id#123 as string) AS carrier_id#130]\n\n(17) AdaptiveSparkPlan\nOutput [2]: [OP_CARRIER#19, carrier_id#130]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"TakeOrderedAndProject","simpleString":"TakeOrderedAndProject(limit=21, orderBy=[OP_CARRIER#19 ASC NULLS FIRST], output=[OP_CARRIER#19,carrier_id#130])","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#19 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#233]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#19], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=1816.0 MiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#223]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#19], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#19] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":308,"metricType":"timing"},{"name":"number of files read","accumulatorId":302,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":307,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":306,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":305,"metricType":"timing"},{"name":"metadata time","accumulatorId":303,"metricType":"timing"},{"name":"size of files read","accumulatorId":304,"metricType":"size"},{"name":"number of output rows","accumulatorId":301,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":323,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":324,"metricType":"timing"},{"name":"peak memory","accumulatorId":322,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":326,"metricType":"sum"},{"name":"number of output rows","accumulatorId":321,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":325,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":320,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":318,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":319,"metricType":"nsTiming"},{"name":"records read","accumulatorId":316,"metricType":"sum"},{"name":"local bytes read","accumulatorId":314,"metricType":"size"},{"name":"fetch wait time","accumulatorId":315,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":312,"metricType":"size"},{"name":"local blocks read","accumulatorId":311,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":310,"metricType":"sum"},{"name":"data size","accumulatorId":309,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":313,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":317,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[308,0],[302,10],[303,0],[304,1904190334]]} +{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1646432529806,"Stage Infos":[{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_223\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_222\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[6],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_223\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"6","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_223\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_222\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432529816,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_223\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"6","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":0,"Attempt":0,"Launch Time":1646432529906,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":1,"Attempt":0,"Launch Time":1646432529906,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":2,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":3,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":4,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":5,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":6,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":7,"Attempt":0,"Launch Time":1646432529908,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":8,"Attempt":0,"Launch Time":1646432529908,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":9,"Attempt":0,"Launch Time":1646432529908,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":9,"Attempt":0,"Launch Time":1646432529908,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432542060,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"5617658","Value":"5617658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"1221","Value":"1221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"288","Value":"288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"816","Value":"816","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"12","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"10484346","Value":"10484346","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"12017","Value":"12017","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"12","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"11912","Value":"11912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":44,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9542724,"Value":9542724,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":12067,"Value":12067,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":11859188511,"Value":11859188511,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2133,"Value":2133,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":90,"Value":90,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":816,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":10484346,"Value":10484346,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":5617658,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1221165838,"Value":1221165838,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2042091888,"JVMOffHeapMemory":119725384,"OnHeapExecutionMemory":202113024,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":203027282,"OffHeapUnifiedMemory":0,"DirectPoolMemory":487929,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":47,"MinorGCTime":305,"MajorGCCount":4,"MajorGCTime":198},"Task Metrics":{"Executor Deserialize Time":44,"Executor Deserialize CPU Time":9542724,"Executor Run Time":12067,"Executor CPU Time":11859188511,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":90,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":10484346,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":8,"Attempt":0,"Launch Time":1646432529908,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432542078,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"5674621","Value":"11292279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"1317","Value":"2538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"178217644","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"288","Value":"576","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"816","Value":"1632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"12","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"13384246","Value":"23868592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"12026","Value":"24043","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"12","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"11896","Value":"23808","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":54,"Value":98,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19931966,"Value":29474690,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":12075,"Value":24142,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":11841930388,"Value":23701118899,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2132,"Value":4265,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":88,"Value":178,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":1632,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":13384246,"Value":23868592,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":354150469,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":11292279,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1317946898,"Value":2539112736,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":354150469,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1733490320,"JVMOffHeapMemory":120182776,"OnHeapExecutionMemory":202113024,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":203027282,"OffHeapUnifiedMemory":0,"DirectPoolMemory":450446,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":45,"MinorGCTime":325,"MajorGCCount":4,"MajorGCTime":222},"Task Metrics":{"Executor Deserialize Time":54,"Executor Deserialize CPU Time":19931966,"Executor Run Time":12075,"Executor CPU Time":11841930388,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":88,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":13384246,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":5,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432542353,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"6096762","Value":"17389041","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"1253","Value":"3791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"538089977","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"183939508","Value":"538089977","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"360","Value":"936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"1020","Value":"2652","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"15","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"6910086","Value":"30778678","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"12309","Value":"36352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"15","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"12208","Value":"36016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":44,"Value":142,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42709063,"Value":72183753,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":12360,"Value":36502,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":12049944723,"Value":35751063622,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2133,"Value":6398,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":90,"Value":268,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1020,"Value":2652,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":15,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":6910086,"Value":30778678,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":538089977,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":17389041,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1253458807,"Value":3792571543,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":538089977,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2042091888,"JVMOffHeapMemory":119725384,"OnHeapExecutionMemory":202113024,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":203027282,"OffHeapUnifiedMemory":0,"DirectPoolMemory":487929,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":47,"MinorGCTime":305,"MajorGCCount":4,"MajorGCTime":198},"Task Metrics":{"Executor Deserialize Time":44,"Executor Deserialize CPU Time":42709063,"Executor Run Time":12360,"Executor CPU Time":12049944723,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":90,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1020,"Shuffle Write Time":6910086,"Shuffle Records Written":15},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":7,"Attempt":0,"Launch Time":1646432529908,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432542483,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"5819811","Value":"23208852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"1239","Value":"5030","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"718783820","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"180693843","Value":"718783820","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"336","Value":"1272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"952","Value":"3604","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"14","Value":"53","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"12484523","Value":"43263201","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"12407","Value":"48759","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"14","Value":"53","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"12306","Value":"48322","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"4000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":41,"Value":183,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":33736181,"Value":105919934,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":12465,"Value":48967,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":12267573789,"Value":48018637411,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2133,"Value":8531,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":82,"Value":350,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":3604,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":53,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":12484523,"Value":43263201,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":718783820,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":23208852,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1239511843,"Value":5032083386,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":718783820,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1770867736,"JVMOffHeapMemory":119759744,"OnHeapExecutionMemory":134742016,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":135656274,"OffHeapUnifiedMemory":0,"DirectPoolMemory":483882,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":41,"MinorGCTime":270,"MajorGCCount":4,"MajorGCTime":188},"Task Metrics":{"Executor Deserialize Time":41,"Executor Deserialize CPU Time":33736181,"Executor Run Time":12465,"Executor CPU Time":12267573789,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":82,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":12484523,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":6,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432542535,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"5819079","Value":"29027931","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"1251","Value":"6281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"901020685","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"182236865","Value":"901020685","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"336","Value":"1608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"952","Value":"4556","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"14","Value":"67","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"12450483","Value":"55713684","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"12465","Value":"61224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"14","Value":"67","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"12360","Value":"60682","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":45,"Value":228,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":24373059,"Value":130292993,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":12516,"Value":61483,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":12348943468,"Value":60367580879,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2133,"Value":10664,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":84,"Value":434,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":4556,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":67,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":12450483,"Value":55713684,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":901020685,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":29027931,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1251125974,"Value":6283209360,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":901020685,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3781181960,"JVMOffHeapMemory":119233528,"OnHeapExecutionMemory":134742016,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":135656274,"OffHeapUnifiedMemory":0,"DirectPoolMemory":363791,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":48,"MinorGCTime":303,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":45,"Executor Deserialize CPU Time":24373059,"Executor Run Time":12516,"Executor CPU Time":12348943468,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":84,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":12450483,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":1,"Attempt":0,"Launch Time":1646432529906,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432543056,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"6369482","Value":"35397413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"1341","Value":"7622","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1098659219","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"197638534","Value":"1098659219","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"384","Value":"1992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"1088","Value":"5644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"16","Value":"83","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"4965590","Value":"60679274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"13013","Value":"74237","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"16","Value":"83","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"12913","Value":"73595","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":42,"Value":270,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8419333,"Value":138712326,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":13064,"Value":74547,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":12818142208,"Value":73185723087,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2133,"Value":12797,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":90,"Value":524,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1088,"Value":5644,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":83,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":4965590,"Value":60679274,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1098659219,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":35397413,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1341805504,"Value":7625014864,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1098659219,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2042091888,"JVMOffHeapMemory":119725384,"OnHeapExecutionMemory":202113024,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":203027282,"OffHeapUnifiedMemory":0,"DirectPoolMemory":487929,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":47,"MinorGCTime":305,"MajorGCCount":4,"MajorGCTime":198},"Task Metrics":{"Executor Deserialize Time":42,"Executor Deserialize CPU Time":8419333,"Executor Run Time":13064,"Executor CPU Time":12818142208,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":90,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1088,"Shuffle Write Time":4965590,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":4,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432543219,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"6066650","Value":"41464063","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"1290","Value":"8912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1283695099","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"185035880","Value":"1283695099","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"384","Value":"2376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"1028","Value":"6672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"16","Value":"99","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"6847082","Value":"67526356","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"13166","Value":"87403","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"16","Value":"99","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"13044","Value":"86639","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":54,"Value":324,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42130952,"Value":180843278,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":13216,"Value":87763,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":12945744267,"Value":86131467354,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2132,"Value":14929,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":95,"Value":619,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1028,"Value":6672,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":99,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":6847082,"Value":67526356,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1283695099,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":41464063,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1290603012,"Value":8915617876,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1283695099,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1733490320,"JVMOffHeapMemory":120182776,"OnHeapExecutionMemory":202113024,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":203027282,"OffHeapUnifiedMemory":0,"DirectPoolMemory":450446,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":45,"MinorGCTime":325,"MajorGCCount":4,"MajorGCTime":222},"Task Metrics":{"Executor Deserialize Time":54,"Executor Deserialize CPU Time":42130952,"Executor Run Time":13216,"Executor CPU Time":12945744267,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":95,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1028,"Shuffle Write Time":6847082,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":3,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432543704,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"6429338","Value":"47893401","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"1312","Value":"10224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"195074315","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"456","Value":"2832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"1232","Value":"7904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"19","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"5622444","Value":"73148800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"13630","Value":"101033","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"19","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"13535","Value":"100174","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":41,"Value":365,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":15985318,"Value":196828596,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":13686,"Value":101449,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":13448638225,"Value":99580105579,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2133,"Value":17062,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":82,"Value":701,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1232,"Value":7904,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":19,"Value":118,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":5622444,"Value":73148800,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1478769414,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":47893401,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1312052076,"Value":10227669952,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1478769414,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1770867736,"JVMOffHeapMemory":119759744,"OnHeapExecutionMemory":134742016,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":135656274,"OffHeapUnifiedMemory":0,"DirectPoolMemory":483882,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":41,"MinorGCTime":270,"MajorGCCount":4,"MajorGCTime":188},"Task Metrics":{"Executor Deserialize Time":41,"Executor Deserialize CPU Time":15985318,"Executor Run Time":13686,"Executor CPU Time":13448638225,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":82,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1232,"Shuffle Write Time":5622444,"Shuffle Records Written":19},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":2,"Attempt":0,"Launch Time":1646432529907,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432543855,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"6450117","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"1365","Value":"11589","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"432","Value":"3264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"1164","Value":"9068","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"18","Value":"136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"6097632","Value":"79246432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"13785","Value":"114818","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"18","Value":"136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"13686","Value":"113860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":45,"Value":410,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":29154583,"Value":225983179,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":13835,"Value":115284,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":13600226178,"Value":113180331757,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2133,"Value":19195,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":89,"Value":790,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1164,"Value":9068,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":136,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":6097632,"Value":79246432,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1675583714,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":54343518,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1365798232,"Value":11593468184,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1675583714,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3781181960,"JVMOffHeapMemory":119233528,"OnHeapExecutionMemory":134742016,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":135656274,"OffHeapUnifiedMemory":0,"DirectPoolMemory":363791,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":48,"MinorGCTime":303,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":45,"Executor Deserialize CPU Time":29154583,"Executor Run Time":13835,"Executor CPU Time":13600226178,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":89,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1164,"Shuffle Write Time":6097632,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":0,"Attempt":0,"Launch Time":1646432529906,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432546277,"Failed":false,"Killed":false,"Accumulables":[{"ID":301,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Update":"2245","Value":"13834","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Update":"432","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Update":"1224","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Update":"18","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Update":"6573488","Value":"85819920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Update":"16225","Value":"131043","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Update":"18","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Update":"16103","Value":"129963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Update":54,"Value":464,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11465440,"Value":237448619,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Update":16274,"Value":131558,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Update":15340955778,"Value":128521287535,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Update":2132,"Value":21327,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Update":95,"Value":885,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1224,"Value":10292,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Update":6573488,"Value":85819920,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Update":2245811606,"Value":13839279790,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1868446456,"JVMOffHeapMemory":120436880,"OnHeapExecutionMemory":202113024,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":914258,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":203027282,"OffHeapUnifiedMemory":0,"DirectPoolMemory":450948,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":47,"MinorGCTime":337,"MajorGCCount":4,"MajorGCTime":222},"Task Metrics":{"Executor Deserialize Time":54,"Executor Deserialize CPU Time":11465440,"Executor Run Time":16274,"Executor CPU Time":15340955778,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":95,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1224,"Shuffle Write Time":6573488,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_223\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_222\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432529816,"Completion Time":1646432546278,"Accumulables":[{"ID":301,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":305,"Name":"filesystem read time (sampled)","Value":"13834","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":306,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":307,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":309,"Name":"data size","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"shuffle bytes written","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":318,"Name":"shuffle records written","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":319,"Name":"shuffle write time","Value":"85819920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":320,"Name":"duration","Value":"131043","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":321,"Name":"number of output rows","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":322,"Name":"peak memory","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"time in aggregation build","Value":"129963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"avg hash probe bucket list iters","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":327,"Name":"internal.metrics.executorDeserializeTime","Value":464,"Internal":true,"Count Failed Values":true},{"ID":328,"Name":"internal.metrics.executorDeserializeCpuTime","Value":237448619,"Internal":true,"Count Failed Values":true},{"ID":329,"Name":"internal.metrics.executorRunTime","Value":131558,"Internal":true,"Count Failed Values":true},{"ID":330,"Name":"internal.metrics.executorCpuTime","Value":128521287535,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.resultSize","Value":21327,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.jvmGCTime","Value":885,"Internal":true,"Count Failed Values":true},{"ID":336,"Name":"internal.metrics.peakExecutionMemory","Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":347,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":10292,"Internal":true,"Count Failed Values":true},{"ID":348,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":154,"Internal":true,"Count Failed Values":true},{"ID":349,"Name":"internal.metrics.shuffle.write.writeTime","Value":85819920,"Internal":true,"Count Failed Values":true},{"ID":350,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":351,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":352,"Name":"internal.metrics.input.sampledTimeReadNano","Value":13839279790,"Internal":true,"Count Failed Values":true},{"ID":353,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1646432546283,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":6,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (19)\n+- == Current Plan ==\n TakeOrderedAndProject (11)\n +- RunningWindowFunction (10)\n +- Sort (9)\n +- ShuffleQueryStage (8)\n +- Exchange (7)\n +- * HashAggregate (6)\n +- CustomShuffleReader (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * HashAggregate (2)\n +- Scan csv (1)\n+- == Initial Plan ==\n TakeOrderedAndProject (18)\n +- RunningWindowFunction (17)\n +- Sort (16)\n +- Exchange (15)\n +- HashAggregate (14)\n +- Exchange (13)\n +- HashAggregate (12)\n +- Scan csv (1)\n\n\n(1) Scan csv \nOutput [1]: [OP_CARRIER#19]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(2) HashAggregate [codegen id : 1]\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(3) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#223]\n\n(4) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#19]\nArguments: 0, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)\n\n(5) CustomShuffleReader\nInput [1]: [OP_CARRIER#19]\nArguments: coalesced\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(7) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#273]\n\n(8) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#19]\nArguments: 1, Statistics(sizeInBytes=4.2 KiB, rowCount=154, isRuntime=true)\n\n(9) Sort\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(10) RunningWindowFunction\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false\n\n(11) TakeOrderedAndProject\nInput [2]: [OP_CARRIER#19, carrier_id#123]\nArguments: 21, [OP_CARRIER#19 ASC NULLS FIRST], [OP_CARRIER#19, cast(carrier_id#123 as string) AS carrier_id#130]\n\n(12) HashAggregate\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(13) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#161]\n\n(14) HashAggregate\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(15) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#164]\n\n(16) Sort\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(17) RunningWindowFunction\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false\n\n(18) TakeOrderedAndProject\nInput [2]: [OP_CARRIER#19, carrier_id#123]\nArguments: 21, [OP_CARRIER#19 ASC NULLS FIRST], [OP_CARRIER#19, cast(carrier_id#123 as string) AS carrier_id#130]\n\n(19) AdaptiveSparkPlan\nOutput [2]: [OP_CARRIER#19, carrier_id#130]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"TakeOrderedAndProject","simpleString":"TakeOrderedAndProject(limit=21, orderBy=[OP_CARRIER#19 ASC NULLS FIRST], output=[OP_CARRIER#19,carrier_id#130])","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#19 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=4.2 KiB, rowCount=154, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#273]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#19], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#223]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#19], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#19] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":308,"metricType":"timing"},{"name":"number of files read","accumulatorId":302,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":307,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":306,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":305,"metricType":"timing"},{"name":"metadata time","accumulatorId":303,"metricType":"timing"},{"name":"size of files read","accumulatorId":304,"metricType":"size"},{"name":"number of output rows","accumulatorId":301,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":323,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":324,"metricType":"timing"},{"name":"peak memory","accumulatorId":322,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":326,"metricType":"sum"},{"name":"number of output rows","accumulatorId":321,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":325,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":320,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":318,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":319,"metricType":"nsTiming"},{"name":"records read","accumulatorId":316,"metricType":"sum"},{"name":"local bytes read","accumulatorId":314,"metricType":"size"},{"name":"fetch wait time","accumulatorId":315,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":312,"metricType":"size"},{"name":"local blocks read","accumulatorId":311,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":310,"metricType":"sum"},{"name":"data size","accumulatorId":309,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":313,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":317,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":374,"metricType":"sum"},{"name":"partition data size","accumulatorId":375,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":371,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":372,"metricType":"timing"},{"name":"peak memory","accumulatorId":370,"metricType":"size"},{"name":"number of output rows","accumulatorId":369,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":373,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":368,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":366,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":367,"metricType":"nsTiming"},{"name":"records read","accumulatorId":364,"metricType":"sum"},{"name":"local bytes read","accumulatorId":362,"metricType":"size"},{"name":"fetch wait time","accumulatorId":363,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":360,"metricType":"size"},{"name":"local blocks read","accumulatorId":359,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":358,"metricType":"sum"},{"name":"data size","accumulatorId":357,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":361,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":365,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[374,1],[375,10896]]} +{"Event":"SparkListenerJobStart","Job ID":6,"Submission Time":1646432546349,"Stage Infos":[{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_223\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_222\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_273\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_267\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_272\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[7],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[7,8],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_273\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"6","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_273\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_267\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_272\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[7],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432546351,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_273\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"6","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":0,"Attempt":0,"Launch Time":1646432546390,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":0,"Attempt":0,"Launch Time":1646432546390,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432546573,"Failed":false,"Killed":false,"Accumulables":[{"ID":310,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":311,"Name":"local blocks read","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":312,"Name":"remote bytes read","Update":"7224","Value":"7224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":314,"Name":"local bytes read","Update":"3068","Value":"3068","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":315,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":316,"Name":"records read","Update":"154","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":357,"Name":"data size","Update":"552","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":365,"Name":"shuffle bytes written","Update":"264","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":366,"Name":"shuffle records written","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":367,"Name":"shuffle write time","Update":"391000","Value":"391000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":368,"Name":"duration","Update":"22","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":369,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":370,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":372,"Name":"time in aggregation build","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":373,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":376,"Name":"internal.metrics.executorDeserializeTime","Update":32,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.executorDeserializeCpuTime","Update":32079296,"Value":32079296,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.executorRunTime","Update":130,"Value":130,"Internal":true,"Count Failed Values":true},{"ID":379,"Name":"internal.metrics.executorCpuTime","Update":117349423,"Value":117349423,"Internal":true,"Count Failed Values":true},{"ID":380,"Name":"internal.metrics.resultSize","Update":3468,"Value":3468,"Internal":true,"Count Failed Values":true},{"ID":385,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":389,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":390,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":391,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":7224,"Value":7224,"Internal":true,"Count Failed Values":true},{"ID":392,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":393,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":3068,"Value":3068,"Internal":true,"Count Failed Values":true},{"ID":394,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":395,"Name":"internal.metrics.shuffle.read.recordsRead","Update":154,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":396,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":264,"Value":264,"Internal":true,"Count Failed Values":true},{"ID":397,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":398,"Name":"internal.metrics.shuffle.write.writeTime","Update":391000,"Value":391000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":32,"Executor Deserialize CPU Time":32079296,"Executor Run Time":130,"Executor CPU Time":117349423,"Peak Execution Memory":67371008,"Result Size":3468,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":3,"Fetch Wait Time":0,"Remote Bytes Read":7224,"Remote Bytes Read To Disk":0,"Local Bytes Read":3068,"Total Records Read":154},"Shuffle Write Metrics":{"Shuffle Bytes Written":264,"Shuffle Write Time":391000,"Shuffle Records Written":23},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_273\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_267\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_272\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[7],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432546351,"Completion Time":1646432546574,"Accumulables":[{"ID":310,"Name":"remote blocks read","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":311,"Name":"local blocks read","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":312,"Name":"remote bytes read","Value":"7224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":314,"Name":"local bytes read","Value":"3068","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":315,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":316,"Name":"records read","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":357,"Name":"data size","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":365,"Name":"shuffle bytes written","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":366,"Name":"shuffle records written","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":367,"Name":"shuffle write time","Value":"391000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":368,"Name":"duration","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":369,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":370,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":372,"Name":"time in aggregation build","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":373,"Name":"avg hash probe bucket list iters","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":376,"Name":"internal.metrics.executorDeserializeTime","Value":32,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.executorDeserializeCpuTime","Value":32079296,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.executorRunTime","Value":130,"Internal":true,"Count Failed Values":true},{"ID":379,"Name":"internal.metrics.executorCpuTime","Value":117349423,"Internal":true,"Count Failed Values":true},{"ID":380,"Name":"internal.metrics.resultSize","Value":3468,"Internal":true,"Count Failed Values":true},{"ID":385,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":389,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":7,"Internal":true,"Count Failed Values":true},{"ID":390,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":3,"Internal":true,"Count Failed Values":true},{"ID":391,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":7224,"Internal":true,"Count Failed Values":true},{"ID":392,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":393,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":3068,"Internal":true,"Count Failed Values":true},{"ID":394,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":395,"Name":"internal.metrics.shuffle.read.recordsRead","Value":154,"Internal":true,"Count Failed Values":true},{"ID":396,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":264,"Internal":true,"Count Failed Values":true},{"ID":397,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23,"Internal":true,"Count Failed Values":true},{"ID":398,"Name":"internal.metrics.shuffle.write.writeTime","Value":391000,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":6,"Completion Time":1646432546574,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":6,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (19)\n+- == Final Plan ==\n TakeOrderedAndProject (11)\n +- * RunningWindowFunction (10)\n +- * Sort (9)\n +- ShuffleQueryStage (8)\n +- Exchange (7)\n +- * HashAggregate (6)\n +- CustomShuffleReader (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * HashAggregate (2)\n +- Scan csv (1)\n+- == Initial Plan ==\n TakeOrderedAndProject (18)\n +- RunningWindowFunction (17)\n +- Sort (16)\n +- Exchange (15)\n +- HashAggregate (14)\n +- Exchange (13)\n +- HashAggregate (12)\n +- Scan csv (1)\n\n\n(1) Scan csv \nOutput [1]: [OP_CARRIER#19]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(2) HashAggregate [codegen id : 1]\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(3) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#223]\n\n(4) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#19]\nArguments: 0, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)\n\n(5) CustomShuffleReader\nInput [1]: [OP_CARRIER#19]\nArguments: coalesced\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(7) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#273]\n\n(8) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#19]\nArguments: 1, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)\n\n(9) Sort [codegen id : 3]\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(10) RunningWindowFunction [codegen id : 3]\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false\n\n(11) TakeOrderedAndProject\nInput [2]: [OP_CARRIER#19, carrier_id#123]\nArguments: 21, [OP_CARRIER#19 ASC NULLS FIRST], [OP_CARRIER#19, cast(carrier_id#123 as string) AS carrier_id#130]\n\n(12) HashAggregate\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(13) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#161]\n\n(14) HashAggregate\nInput [1]: [OP_CARRIER#19]\nKeys [1]: [OP_CARRIER#19]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#19]\n\n(15) Exchange\nInput [1]: [OP_CARRIER#19]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#164]\n\n(16) Sort\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(17) RunningWindowFunction\nInput [1]: [OP_CARRIER#19]\nArguments: [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false\n\n(18) TakeOrderedAndProject\nInput [2]: [OP_CARRIER#19, carrier_id#123]\nArguments: 21, [OP_CARRIER#19 ASC NULLS FIRST], [OP_CARRIER#19, cast(carrier_id#123 as string) AS carrier_id#130]\n\n(19) AdaptiveSparkPlan\nOutput [2]: [OP_CARRIER#19, carrier_id#130]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"TakeOrderedAndProject","simpleString":"TakeOrderedAndProject(limit=21, orderBy=[OP_CARRIER#19 ASC NULLS FIRST], output=[OP_CARRIER#19,carrier_id#130])","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#19, row_number() windowspecdefinition(OP_CARRIER#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#19 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#19 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#273]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#19], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#223]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#19], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#19] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":308,"metricType":"timing"},{"name":"number of files read","accumulatorId":302,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":307,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":306,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":305,"metricType":"timing"},{"name":"metadata time","accumulatorId":303,"metricType":"timing"},{"name":"size of files read","accumulatorId":304,"metricType":"size"},{"name":"number of output rows","accumulatorId":301,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":323,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":324,"metricType":"timing"},{"name":"peak memory","accumulatorId":322,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":326,"metricType":"sum"},{"name":"number of output rows","accumulatorId":321,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":325,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":320,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":318,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":319,"metricType":"nsTiming"},{"name":"records read","accumulatorId":316,"metricType":"sum"},{"name":"local bytes read","accumulatorId":314,"metricType":"size"},{"name":"fetch wait time","accumulatorId":315,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":312,"metricType":"size"},{"name":"local blocks read","accumulatorId":311,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":310,"metricType":"sum"},{"name":"data size","accumulatorId":309,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":313,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":317,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":374,"metricType":"sum"},{"name":"partition data size","accumulatorId":375,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":371,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":372,"metricType":"timing"},{"name":"peak memory","accumulatorId":370,"metricType":"size"},{"name":"number of output rows","accumulatorId":369,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":373,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":368,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":366,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":367,"metricType":"nsTiming"},{"name":"records read","accumulatorId":364,"metricType":"sum"},{"name":"local bytes read","accumulatorId":362,"metricType":"size"},{"name":"fetch wait time","accumulatorId":363,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":360,"metricType":"size"},{"name":"local blocks read","accumulatorId":359,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":358,"metricType":"sum"},{"name":"data size","accumulatorId":357,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":361,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":365,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":419,"metricType":"size"},{"name":"cumulative time","accumulatorId":420,"metricType":"timing"},{"name":"peak memory","accumulatorId":418,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":422,"metricType":"sum"},{"name":"sort time","accumulatorId":417,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":423,"metricType":"sum"},{"name":"spill write time","accumulatorId":421,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":416,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":414,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":415,"metricType":"nsTiming"},{"name":"records read","accumulatorId":412,"metricType":"sum"},{"name":"local bytes read","accumulatorId":410,"metricType":"size"},{"name":"fetch wait time","accumulatorId":411,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":408,"metricType":"size"},{"name":"local blocks read","accumulatorId":407,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":406,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":409,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":413,"metricType":"size"}]}],"metadata":{},"metrics":[]}} +{"Event":"SparkListenerJobStart","Job ID":7,"Submission Time":1646432546698,"Stage Infos":[{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_223\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_222\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_155\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":10,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_273\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_267\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_272\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[9],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"37\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_303\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[9,10,11],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"38\",\"name\":\"takeOrdered\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"6","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"37\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_303\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432546702,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"38\",\"name\":\"takeOrdered\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"6","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":11,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":0,"Attempt":0,"Launch Time":1646432546727,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":11,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":0,"Attempt":0,"Launch Time":1646432546727,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432547022,"Failed":false,"Killed":false,"Accumulables":[{"ID":358,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":360,"Name":"remote bytes read","Update":"264","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":363,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":364,"Name":"records read","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":416,"Name":"duration","Update":"73","Value":"73","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":417,"Name":"sort time","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":418,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":419,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":420,"Name":"cumulative time","Update":"17","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":421,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":422,"Name":"num prefix comparisons","Update":"75","Value":"75","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":424,"Name":"internal.metrics.executorDeserializeTime","Update":93,"Value":93,"Internal":true,"Count Failed Values":true},{"ID":425,"Name":"internal.metrics.executorDeserializeCpuTime","Update":90819288,"Value":90819288,"Internal":true,"Count Failed Values":true},{"ID":426,"Name":"internal.metrics.executorRunTime","Update":179,"Value":179,"Internal":true,"Count Failed Values":true},{"ID":427,"Name":"internal.metrics.executorCpuTime","Update":145455219,"Value":145455219,"Internal":true,"Count Failed Values":true},{"ID":428,"Name":"internal.metrics.resultSize","Update":5115,"Value":5115,"Internal":true,"Count Failed Values":true},{"ID":433,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":437,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":438,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":439,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":264,"Value":264,"Internal":true,"Count Failed Values":true},{"ID":440,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":441,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":442,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":443,"Name":"internal.metrics.shuffle.read.recordsRead","Update":23,"Value":23,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":93,"Executor Deserialize CPU Time":90819288,"Executor Run Time":179,"Executor CPU Time":145455219,"Peak Execution Memory":67174400,"Result Size":5115,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":264,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":23},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"37\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_303\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432546702,"Completion Time":1646432547022,"Accumulables":[{"ID":358,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":360,"Name":"remote bytes read","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":363,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":364,"Name":"records read","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":416,"Name":"duration","Value":"73","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":417,"Name":"sort time","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":418,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":419,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":420,"Name":"cumulative time","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":421,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":422,"Name":"num prefix comparisons","Value":"75","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":424,"Name":"internal.metrics.executorDeserializeTime","Value":93,"Internal":true,"Count Failed Values":true},{"ID":425,"Name":"internal.metrics.executorDeserializeCpuTime","Value":90819288,"Internal":true,"Count Failed Values":true},{"ID":426,"Name":"internal.metrics.executorRunTime","Value":179,"Internal":true,"Count Failed Values":true},{"ID":427,"Name":"internal.metrics.executorCpuTime","Value":145455219,"Internal":true,"Count Failed Values":true},{"ID":428,"Name":"internal.metrics.resultSize","Value":5115,"Internal":true,"Count Failed Values":true},{"ID":433,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":437,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":438,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":439,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":264,"Internal":true,"Count Failed Values":true},{"ID":440,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":441,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":442,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":443,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":7,"Completion Time":1646432547023,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":6,"time":1646432547086,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":7,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.Dataset.createOrReplaceTempView(Dataset.scala:3394)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateViewCommand (1)\n +- CreateViewCommand (2)\n +- Window (10)\n +- Project (9)\n +- Deduplicate (8)\n +- Union (7)\n :- Project (5)\n : +- Project (4)\n : +- LogicalRelation (3)\n +- Project (6)\n +- Project (4)\n +- LogicalRelation (3)\n\n\n(1) Execute CreateViewCommand\nOutput: []\n\n(2) CreateViewCommand\nArguments: `airports`, false, true, LocalTempView\n\n(3) LogicalRelation\nArguments: csv, [FL_DATE#18, OP_CARRIER#19, OP_CARRIER_FL_NUM#20, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_TIME#24, DEP_DELAY#25, TAXI_OUT#26, WHEELS_OFF#27, WHEELS_ON#28, TAXI_IN#29, CRS_ARR_TIME#30, ARR_TIME#31, ARR_DELAY#32, CANCELLED#33, CANCELLATION_CODE#34, DIVERTED#35, CRS_ELAPSED_TIME#36, ACTUAL_ELAPSED_TIME#37, AIR_TIME#38, DISTANCE#39, CARRIER_DELAY#40, WEATHER_DELAY#41, ... 4 more fields], false\n\n(4) Project\nArguments: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\n\n(5) Project\nArguments: [ORIGIN#21 AS airport#144]\n\n(6) Project\nArguments: [DEST#22 AS airport#146]\n\n(7) Union\nArguments: false, false\n\n(8) Deduplicate\nArguments: [airport#144]\n\n(9) Project\nArguments: [airport#144]\n\n(10) Window\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST]\n\n","sparkPlanInfo":{"nodeName":"Execute CreateViewCommand","simpleString":"Execute CreateViewCommand","children":[],"metadata":{},"metrics":[]},"time":1646432547180} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":7,"time":1646432547180,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":8,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (14)\n+- GlobalLimit (13)\n +- LocalLimit (12)\n +- Window (11)\n +- Sort (10)\n +- Exchange (9)\n +- HashAggregate (8)\n +- Exchange (7)\n +- HashAggregate (6)\n +- Union (5)\n :- Project (2)\n : +- Scan csv (1)\n +- Project (4)\n +- Scan csv (3)\n\n\n(1) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(2) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(3) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(4) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(5) Union\n\n(6) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(7) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#341]\n\n(8) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(9) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#344]\n\n(10) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(11) Window\nInput [1]: [airport#144]\nArguments: [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21\n\n(12) LocalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(13) GlobalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(14) AdaptiveSparkPlan\nOutput [2]: [airport#144, airport_id#156]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"GlobalLimit","simpleString":"GlobalLimit 21","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Window","simpleString":"Window [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#344]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#341]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},"time":1646432547307} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":8,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (26)\n+- == Current Plan ==\n GlobalLimit (14)\n +- LocalLimit (13)\n +- Window (12)\n +- Sort (11)\n +- Exchange (10)\n +- HashAggregate (9)\n +- ShuffleQueryStage (8)\n +- Exchange (7)\n +- * HashAggregate (6)\n +- Union (5)\n :- * Project (2)\n : +- Scan csv (1)\n +- * Project (4)\n +- Scan csv (3)\n+- == Initial Plan ==\n GlobalLimit (25)\n +- LocalLimit (24)\n +- Window (23)\n +- Sort (22)\n +- Exchange (21)\n +- HashAggregate (20)\n +- Exchange (19)\n +- HashAggregate (18)\n +- Union (17)\n :- Project (15)\n : +- Scan csv (1)\n +- Project (16)\n +- Scan csv (3)\n\n\n(1) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(2) Project [codegen id : 1]\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(3) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(4) Project [codegen id : 2]\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(5) Union\n\n(6) HashAggregate [codegen id : 3]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(7) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#438]\n\n(8) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 0, Statistics(sizeInBytes=3.5 GiB)\n\n(9) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(10) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#457]\n\n(11) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(12) Window\nInput [1]: [airport#144]\nArguments: [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21\n\n(13) LocalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(14) GlobalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(15) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(16) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(17) Union\n\n(18) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(19) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#341]\n\n(20) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(21) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#344]\n\n(22) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(23) Window\nInput [1]: [airport#144]\nArguments: [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21\n\n(24) LocalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(25) GlobalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(26) AdaptiveSparkPlan\nOutput [2]: [airport#144, airport_id#156]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"GlobalLimit","simpleString":"GlobalLimit 21","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Window","simpleString":"Window [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#457]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=3.5 GiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#438]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":501,"metricType":"timing"},{"name":"number of files read","accumulatorId":495,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":500,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":499,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":498,"metricType":"timing"},{"name":"metadata time","accumulatorId":496,"metricType":"timing"},{"name":"size of files read","accumulatorId":497,"metricType":"size"},{"name":"number of output rows","accumulatorId":494,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":528,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":509,"metricType":"timing"},{"name":"number of files read","accumulatorId":503,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":508,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":507,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":506,"metricType":"timing"},{"name":"metadata time","accumulatorId":504,"metricType":"timing"},{"name":"size of files read","accumulatorId":505,"metricType":"size"},{"name":"number of output rows","accumulatorId":502,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":529,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":524,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":525,"metricType":"timing"},{"name":"peak memory","accumulatorId":523,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":527,"metricType":"sum"},{"name":"number of output rows","accumulatorId":522,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":526,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":521,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":519,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":520,"metricType":"nsTiming"},{"name":"records read","accumulatorId":517,"metricType":"sum"},{"name":"local bytes read","accumulatorId":515,"metricType":"size"},{"name":"fetch wait time","accumulatorId":516,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":513,"metricType":"size"},{"name":"local blocks read","accumulatorId":512,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":511,"metricType":"sum"},{"name":"data size","accumulatorId":510,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":514,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":518,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[501,0],[495,10],[496,0],[497,1904190334]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[509,0],[503,10],[504,0],[505,1904190334]]} +{"Event":"SparkListenerJobStart","Job ID":8,"Submission Time":1646432547407,"Stage Infos":[{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_438\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_430\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_434\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[30,33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_433\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_437\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[12],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_438\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"8","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_438\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_430\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_434\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[30,33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_433\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_437\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432547411,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_438\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"8","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":0,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":1,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":2,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":3,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":39,"Index":4,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":40,"Index":5,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":41,"Index":6,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":42,"Index":7,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":43,"Index":8,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":44,"Index":9,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":45,"Index":10,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":46,"Index":11,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":47,"Index":12,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":48,"Index":13,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":49,"Index":14,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":50,"Index":15,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":51,"Index":16,"Attempt":0,"Launch Time":1646432547452,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":52,"Index":17,"Attempt":0,"Launch Time":1646432547452,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":53,"Index":18,"Attempt":0,"Launch Time":1646432547452,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":54,"Index":19,"Attempt":0,"Launch Time":1646432547452,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":44,"Index":9,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432559896,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"5617658","Value":"5617658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1231","Value":"1231","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7512","Value":"7512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11907","Value":"11907","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"313","Value":"313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"20413426","Value":"20413426","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"12305","Value":"12305","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"313","Value":"313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"12262","Value":"12262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"12304","Value":"12304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":13434254,"Value":13434254,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":12386,"Value":12386,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12085400408,"Value":12085400408,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2562,"Value":2562,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":184,"Value":184,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11907,"Value":11907,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":313,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":20413426,"Value":20413426,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":5617658,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1231708518,"Value":1231708518,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4434779688,"JVMOffHeapMemory":123722664,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2360178,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339215218,"OffHeapUnifiedMemory":0,"DirectPoolMemory":722906,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":49,"MinorGCTime":408,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":13434254,"Executor Run Time":12386,"Executor CPU Time":12085400408,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":184,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11907,"Shuffle Write Time":20413426,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":53,"Index":18,"Attempt":0,"Launch Time":1646432547452,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432560284,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"5674621","Value":"5674621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1198","Value":"1198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"178217644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"178217644","Value":"178217644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7680","Value":"15192","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"12025","Value":"23932","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"320","Value":"633","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"19015748","Value":"39429174","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"12697","Value":"25002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"320","Value":"633","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"12637","Value":"24899","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"12688","Value":"12688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":13790340,"Value":27224594,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":12773,"Value":25159,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12232157750,"Value":24317558158,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":5125,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":388,"Value":572,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12025,"Value":23932,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":633,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":19015748,"Value":39429174,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":354150469,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":11292279,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1198744442,"Value":2430452960,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":354150469,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6036018648,"JVMOffHeapMemory":121943064,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":769728,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":50,"MinorGCTime":390,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":13790340,"Executor Run Time":12773,"Executor CPU Time":12232157750,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":388,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12025,"Shuffle Write Time":19015748,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":51,"Index":16,"Attempt":0,"Launch Time":1646432547452,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432560287,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"5819079","Value":"11493700","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1221","Value":"2419","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"360454509","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"182236865","Value":"360454509","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7728","Value":"22920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"12099","Value":"36031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"322","Value":"955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"17443345","Value":"56872519","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"12711","Value":"37713","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"322","Value":"955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"12659","Value":"37558","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"12701","Value":"25389","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":22,"Value":71,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":20155275,"Value":47379869,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":12778,"Value":37937,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12493688452,"Value":36811246610,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":7688,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":197,"Value":769,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12099,"Value":36031,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":955,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":17443345,"Value":56872519,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":536387334,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":17111358,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1221029617,"Value":3651482577,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":536387334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1961600336,"JVMOffHeapMemory":120994408,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":793855,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":46,"MinorGCTime":365,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":22,"Executor Deserialize CPU Time":20155275,"Executor Run Time":12778,"Executor CPU Time":12493688452,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":197,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12099,"Shuffle Write Time":17443345,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":54,"Index":19,"Attempt":0,"Launch Time":1646432547452,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432560365,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"5617658","Value":"17111358","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1245","Value":"3664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"536387334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"175932825","Value":"536387334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7440","Value":"30360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11887","Value":"47918","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"310","Value":"1265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"17001847","Value":"73874366","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"12794","Value":"50507","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"310","Value":"1265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"12732","Value":"50290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"4007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"12782","Value":"38171","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":88,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11776738,"Value":59156607,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":12864,"Value":50801,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12514769165,"Value":49326015775,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":10251,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":187,"Value":956,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11887,"Value":47918,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":310,"Value":1265,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":17001847,"Value":73874366,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":712320159,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":22729016,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1245651239,"Value":4897133816,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":712320159,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1248963096,"JVMOffHeapMemory":124106296,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2370057,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339225097,"OffHeapUnifiedMemory":0,"DirectPoolMemory":761825,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":52,"MinorGCTime":401,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":11776738,"Executor Run Time":12864,"Executor CPU Time":12514769165,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":187,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11887,"Shuffle Write Time":17001847,"Shuffle Records Written":310},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":43,"Index":8,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432560372,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"5674621","Value":"11292279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1182","Value":"2413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"178217644","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7680","Value":"38040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"12024","Value":"59942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"320","Value":"1585","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"17341186","Value":"91215552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"12797","Value":"63304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"320","Value":"1585","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"12770","Value":"63060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"12813","Value":"25117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":108,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10070386,"Value":69226993,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":12866,"Value":63667,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12518390044,"Value":61844405819,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":12814,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":197,"Value":1153,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12024,"Value":59942,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":1585,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":17341186,"Value":91215552,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":890537803,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":28403637,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1182799689,"Value":6079933505,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":890537803,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1961600336,"JVMOffHeapMemory":120994408,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":793855,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":46,"MinorGCTime":365,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":10070386,"Executor Run Time":12866,"Executor CPU Time":12518390044,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":197,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12024,"Shuffle Write Time":17341186,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":52,"Index":17,"Attempt":0,"Launch Time":1646432547452,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432560374,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"5819811","Value":"22931169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1387","Value":"5051","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"717081177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"180693843","Value":"717081177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7776","Value":"45816","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"12053","Value":"71995","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"324","Value":"1909","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"23316916","Value":"114532468","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"12784","Value":"76088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"324","Value":"1909","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"12757","Value":"75817","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"6014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"12802","Value":"50973","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":131,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10683488,"Value":79910481,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":12866,"Value":76533,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12570533297,"Value":74414939116,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2562,"Value":15376,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":184,"Value":1337,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12053,"Value":71995,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":324,"Value":1909,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":23316916,"Value":114532468,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":1071231646,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":34223448,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1387636661,"Value":7467570166,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":1071231646,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4434779688,"JVMOffHeapMemory":123722664,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2360178,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339215218,"OffHeapUnifiedMemory":0,"DirectPoolMemory":722906,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":49,"MinorGCTime":408,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":10683488,"Executor Run Time":12866,"Executor CPU Time":12570533297,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":184,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12053,"Shuffle Write Time":23316916,"Shuffle Records Written":324},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":42,"Index":7,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432560408,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"5819811","Value":"17112090","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1217","Value":"3630","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"534844312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"180693843","Value":"534844312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7800","Value":"53616","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"12061","Value":"84056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"325","Value":"2234","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"18838399","Value":"133370867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"12834","Value":"88922","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"325","Value":"2234","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"12801","Value":"88618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"7021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"12848","Value":"37965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":147,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8507147,"Value":88417628,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":12906,"Value":89439,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12616742628,"Value":87031681744,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":17939,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":187,"Value":1524,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12061,"Value":84056,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":325,"Value":2234,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":18838399,"Value":133370867,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":1251925489,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":40043259,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1217352991,"Value":8684923157,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":1251925489,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1248963096,"JVMOffHeapMemory":124106296,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2370057,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339225097,"OffHeapUnifiedMemory":0,"DirectPoolMemory":761825,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":52,"MinorGCTime":401,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":8507147,"Executor Run Time":12906,"Executor CPU Time":12616742628,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":187,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12061,"Shuffle Write Time":18838399,"Shuffle Records Written":325},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":41,"Index":6,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432560765,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"5819079","Value":"22931169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1237","Value":"4867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"717081177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"182236865","Value":"717081177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7728","Value":"61344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"12103","Value":"96159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"322","Value":"2556","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"23391820","Value":"156762687","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"13144","Value":"102066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"322","Value":"2556","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"13114","Value":"101732","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"13165","Value":"51130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":172,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":13372809,"Value":101790437,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":13224,"Value":102663,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12691047327,"Value":99722729071,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":20502,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":388,"Value":1912,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12103,"Value":96159,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":2556,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":23391820,"Value":156762687,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":1434162354,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":45862338,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1237193470,"Value":9922116627,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":1434162354,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6036018648,"JVMOffHeapMemory":121943064,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":769728,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":50,"MinorGCTime":390,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":13372809,"Executor Run Time":13224,"Executor CPU Time":12691047327,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":388,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12103,"Shuffle Write Time":23391820,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":39,"Index":4,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432560784,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"6066650","Value":"28997819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1274","Value":"6141","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"902117057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"185035880","Value":"902117057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7176","Value":"68520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11443","Value":"107602","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"299","Value":"2855","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"13885122","Value":"170647809","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"13213","Value":"115279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"299","Value":"2855","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"13192","Value":"114924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"13234","Value":"64364","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":19,"Value":191,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10492676,"Value":112283113,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":13277,"Value":115940,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12958061336,"Value":112680790407,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":23065,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":197,"Value":2109,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11443,"Value":107602,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":299,"Value":2855,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":13885122,"Value":170647809,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1619198234,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":51928988,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1274922652,"Value":11197039279,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1619198234,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1961600336,"JVMOffHeapMemory":120994408,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":793855,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":46,"MinorGCTime":365,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":19,"Executor Deserialize CPU Time":10492676,"Executor Run Time":13277,"Executor CPU Time":12958061336,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":197,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11443,"Shuffle Write Time":13885122,"Shuffle Records Written":299},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":50,"Index":15,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432560966,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"6096762","Value":"29027931","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1444","Value":"6495","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"901020685","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"183939508","Value":"901020685","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7512","Value":"76032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11789","Value":"119391","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"313","Value":"3168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"15185568","Value":"185833377","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"13399","Value":"128678","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"313","Value":"3168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"13367","Value":"128291","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"13418","Value":"64391","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":208,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9228510,"Value":121511623,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":13465,"Value":129405,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12996966035,"Value":125677756442,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":25628,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":187,"Value":2296,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11789,"Value":119391,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":3168,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":15185568,"Value":185833377,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":1803137742,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":58025750,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1444917748,"Value":12641957027,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":1803137742,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1248963096,"JVMOffHeapMemory":124106296,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2370057,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339225097,"OffHeapUnifiedMemory":0,"DirectPoolMemory":761825,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":52,"MinorGCTime":401,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":9228510,"Executor Run Time":13465,"Executor CPU Time":12996966035,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":187,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11789,"Shuffle Write Time":15185568,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":49,"Index":14,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432561071,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"6066650","Value":"35094581","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1533","Value":"8028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1086056565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"185035880","Value":"1086056565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7224","Value":"83256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11463","Value":"130854","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"301","Value":"3469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"15274509","Value":"201107886","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"13489","Value":"142167","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"301","Value":"3469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"741081088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"13462","Value":"141753","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"11021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"13513","Value":"77904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":232,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":15599263,"Value":137110886,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":13560,"Value":142965,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":12881132503,"Value":138558888945,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":28191,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":388,"Value":2684,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":741081088,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11463,"Value":130854,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":301,"Value":3469,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":15274509,"Value":201107886,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1988173622,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":64092400,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1533058984,"Value":14175016011,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1988173622,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6036018648,"JVMOffHeapMemory":121943064,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":769728,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":50,"MinorGCTime":390,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":15599263,"Executor Run Time":13560,"Executor CPU Time":12881132503,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":388,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11463,"Shuffle Write Time":15274509,"Shuffle Records Written":301},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":3,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432561204,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"6429338","Value":"35427157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1315","Value":"7456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1097191372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"195074315","Value":"1097191372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7104","Value":"90360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11482","Value":"142336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"296","Value":"3765","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"11673388","Value":"212781274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"13641","Value":"155808","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"296","Value":"3765","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"808452096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"13613","Value":"155366","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"12028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"13663","Value":"78027","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":250,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11686980,"Value":148797866,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":13703,"Value":156668,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":13410078894,"Value":151968967839,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":30754,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":187,"Value":2871,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":808452096,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11482,"Value":142336,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":3765,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":11673388,"Value":212781274,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":2183247937,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":70521738,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1315625175,"Value":15490641186,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":2183247937,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1248963096,"JVMOffHeapMemory":124106296,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2370057,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339225097,"OffHeapUnifiedMemory":0,"DirectPoolMemory":761825,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":52,"MinorGCTime":401,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":11686980,"Executor Run Time":13703,"Executor CPU Time":13410078894,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":187,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11482,"Shuffle Write Time":11673388,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":40,"Index":5,"Attempt":0,"Launch Time":1646432547450,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432561227,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"6096762","Value":"41523919","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1543","Value":"8999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1281130880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"183939508","Value":"1281130880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7488","Value":"97848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11723","Value":"154059","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"312","Value":"4077","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"15408490","Value":"228189764","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"13644","Value":"169452","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"312","Value":"4077","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"875823104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"13622","Value":"168988","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"13028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"13665","Value":"91692","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":274,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":17755125,"Value":166552991,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":13718,"Value":170386,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":13169788373,"Value":165138756212,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2562,"Value":33316,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":184,"Value":3055,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":875823104,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11723,"Value":154059,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":312,"Value":4077,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":15408490,"Value":228189764,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":2367187445,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":76618500,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1543631908,"Value":17034273094,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":2367187445,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4434779688,"JVMOffHeapMemory":123722664,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2360178,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339215218,"OffHeapUnifiedMemory":0,"DirectPoolMemory":722906,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":49,"MinorGCTime":408,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":17755125,"Executor Run Time":13718,"Executor CPU Time":13169788373,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":184,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11723,"Shuffle Write Time":15408490,"Shuffle Records Written":312},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":47,"Index":12,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432561284,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"6450117","Value":"41544698","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1329","Value":"9357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1282870865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"196814300","Value":"1282870865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7320","Value":"105168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11668","Value":"165727","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"305","Value":"4382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"12877616","Value":"241067380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"13717","Value":"183169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"305","Value":"4382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"943194112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"13694","Value":"182682","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"14028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"13737","Value":"91641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":294,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11052776,"Value":177605767,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":13779,"Value":184165,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":13486072752,"Value":178624828964,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":35879,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":197,"Value":3252,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":943194112,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":165727,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":4382,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":12877616,"Value":241067380,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":2564001745,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":83068617,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1329943417,"Value":18364216511,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":2564001745,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1961600336,"JVMOffHeapMemory":120994408,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":793855,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":46,"MinorGCTime":365,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":11052776,"Executor Run Time":13779,"Executor CPU Time":13486072752,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":197,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":12877616,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":46,"Index":11,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432561451,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"6369482","Value":"47914180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1334","Value":"10691","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1480509399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"197638534","Value":"1480509399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7632","Value":"112800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11947","Value":"177674","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"318","Value":"4700","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"12683792","Value":"253751172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"13886","Value":"197055","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"318","Value":"4700","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"1010565120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"13856","Value":"196538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"15028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"13906","Value":"105547","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":311,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9929438,"Value":187535205,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":13950,"Value":198115,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":13625561833,"Value":192250390797,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":38442,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":187,"Value":3439,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1010565120,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11947,"Value":177674,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":318,"Value":4700,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":12683792,"Value":253751172,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":2761640279,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":89438099,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1334191492,"Value":19698408003,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":2761640279,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1248963096,"JVMOffHeapMemory":124106296,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2370057,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339225097,"OffHeapUnifiedMemory":0,"DirectPoolMemory":761825,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":52,"MinorGCTime":401,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":9929438,"Executor Run Time":13950,"Executor CPU Time":13625561833,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":187,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11947,"Shuffle Write Time":12683792,"Shuffle Records Written":318},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":1,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432561669,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"6369482","Value":"47893401","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1456","Value":"10455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"197638534","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7680","Value":"120480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11962","Value":"189636","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"320","Value":"5020","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"12594853","Value":"266346025","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"14090","Value":"211145","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"320","Value":"5020","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"1077936128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"14070","Value":"210608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"16028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"14111","Value":"105803","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":335,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":13648576,"Value":201183781,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":14161,"Value":212276,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":13803162344,"Value":206053553141,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2562,"Value":41004,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":184,"Value":3623,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1077936128,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11962,"Value":189636,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":5020,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":12594853,"Value":266346025,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":2959278813,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":95807581,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1456441782,"Value":21154849785,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":2959278813,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4434779688,"JVMOffHeapMemory":123722664,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2360178,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339215218,"OffHeapUnifiedMemory":0,"DirectPoolMemory":722906,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":49,"MinorGCTime":408,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":13648576,"Executor Run Time":14161,"Executor CPU Time":13803162344,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":184,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11962,"Shuffle Write Time":12594853,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":2,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432561754,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"6450117","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1374","Value":"11829","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7320","Value":"127800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11668","Value":"201304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"305","Value":"5325","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"12563060","Value":"278909085","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"14175","Value":"225320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"305","Value":"5325","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"1145307136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"14150","Value":"224758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"17028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"14199","Value":"120002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":358,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":12420783,"Value":213604564,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":14244,"Value":226520,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":13715718067,"Value":219769271208,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":43567,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":388,"Value":4011,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1145307136,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":201304,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":5325,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":12563060,"Value":278909085,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":3156093113,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":102257698,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1374586439,"Value":22529436224,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":3156093113,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6036018648,"JVMOffHeapMemory":122282768,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":770161,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":696,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":12420783,"Executor Run Time":14244,"Executor CPU Time":13715718067,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":388,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":12563060,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":48,"Index":13,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432561803,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"6429338","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1636","Value":"12327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"195074315","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"7104","Value":"134904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"11481","Value":"212785","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"296","Value":"5621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"13086329","Value":"291995414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"14224","Value":"239544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"296","Value":"5621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"1212678144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"14201","Value":"238959","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"18035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"14242","Value":"119789","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":382,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11366084,"Value":224970648,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":14295,"Value":240815,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":13804409703,"Value":233573680911,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2562,"Value":46129,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":184,"Value":4195,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1212678144,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11481,"Value":212785,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":5621,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":13086329,"Value":291995414,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":3351167428,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":108687036,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1636062255,"Value":24165498479,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":3351167428,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4434779688,"JVMOffHeapMemory":123722664,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2360178,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339215218,"OffHeapUnifiedMemory":0,"DirectPoolMemory":722906,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":49,"MinorGCTime":408,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":11366084,"Executor Run Time":14295,"Executor CPU Time":13804409703,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":184,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11481,"Shuffle Write Time":13086329,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":45,"Index":10,"Attempt":0,"Launch Time":1646432547451,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432563342,"Failed":false,"Killed":false,"Accumulables":[{"ID":502,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Update":"1713","Value":"14040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"8592","Value":"143496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"12847","Value":"225632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"358","Value":"5979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"13245971","Value":"305241385","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"15762","Value":"255306","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"358","Value":"5979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"1280049152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"15740","Value":"254699","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"19035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Update":"15786","Value":"135575","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":405,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9817092,"Value":234787740,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":15832,"Value":256647,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":15320135614,"Value":248893816525,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":48692,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":388,"Value":4583,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1280049152,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12847,"Value":225632,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":5979,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":13245971,"Value":305241385,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":3579774048,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":115900482,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1713768636,"Value":25879267115,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":3579774048,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6036018648,"JVMOffHeapMemory":122282768,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":770161,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":696,"MajorGCCount":4,"MajorGCTime":253},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":9817092,"Executor Run Time":15832,"Executor CPU Time":15320135614,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":388,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12847,"Shuffle Write Time":13245971,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":0,"Attempt":0,"Launch Time":1646432547449,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432564166,"Failed":false,"Killed":false,"Accumulables":[{"ID":494,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Update":"1557","Value":"13386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Update":"8592","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Update":"12849","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Update":"358","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Update":"11776693","Value":"317018078","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Update":"16600","Value":"271906","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Update":"358","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Update":"67371008","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Update":"16579","Value":"271278","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Update":"16620","Value":"136622","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":425,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10194123,"Value":244981863,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Update":16661,"Value":273308,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Update":16313529942,"Value":265207346467,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Update":2563,"Value":51255,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Update":202,"Value":4785,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12849,"Value":238481,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Update":11776693,"Value":317018078,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1557139037,"Value":27436406152,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":3808380668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":1961600336,"JVMOffHeapMemory":120994408,"OnHeapExecutionMemory":336855040,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":2201302,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":339056342,"OffHeapUnifiedMemory":0,"DirectPoolMemory":793855,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":46,"MinorGCTime":365,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":10194123,"Executor Run Time":16661,"Executor CPU Time":16313529942,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":202,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12849,"Shuffle Write Time":11776693,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_438\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_430\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_434\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[30,33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_433\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_437\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432547411,"Completion Time":1646432564167,"Accumulables":[{"ID":494,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":498,"Name":"filesystem read time (sampled)","Value":"13386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":502,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"filesystem read time (sampled)","Value":"14040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":510,"Name":"data size","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":518,"Name":"shuffle bytes written","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle records written","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle write time","Value":"317018078","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"duration","Value":"271906","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"number of output rows","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"peak memory","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":525,"Name":"time in aggregation build","Value":"271278","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"avg hash probe bucket list iters","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":528,"Name":"duration","Value":"136622","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"duration","Value":"135575","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"internal.metrics.executorDeserializeTime","Value":425,"Internal":true,"Count Failed Values":true},{"ID":531,"Name":"internal.metrics.executorDeserializeCpuTime","Value":244981863,"Internal":true,"Count Failed Values":true},{"ID":532,"Name":"internal.metrics.executorRunTime","Value":273308,"Internal":true,"Count Failed Values":true},{"ID":533,"Name":"internal.metrics.executorCpuTime","Value":265207346467,"Internal":true,"Count Failed Values":true},{"ID":534,"Name":"internal.metrics.resultSize","Value":51255,"Internal":true,"Count Failed Values":true},{"ID":535,"Name":"internal.metrics.jvmGCTime","Value":4785,"Internal":true,"Count Failed Values":true},{"ID":539,"Name":"internal.metrics.peakExecutionMemory","Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":550,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":238481,"Internal":true,"Count Failed Values":true},{"ID":551,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":552,"Name":"internal.metrics.shuffle.write.writeTime","Value":317018078,"Internal":true,"Count Failed Values":true},{"ID":553,"Name":"internal.metrics.input.bytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":554,"Name":"internal.metrics.input.recordsRead","Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":555,"Name":"internal.metrics.input.sampledTimeReadNano","Value":27436406152,"Internal":true,"Count Failed Values":true},{"ID":556,"Name":"internal.metrics.input.sampledBytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":8,"Completion Time":1646432564170,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":8,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (28)\n+- == Current Plan ==\n GlobalLimit (16)\n +- LocalLimit (15)\n +- Window (14)\n +- Sort (13)\n +- ShuffleQueryStage (12)\n +- Exchange (11)\n +- * HashAggregate (10)\n +- CustomShuffleReader (9)\n +- ShuffleQueryStage (8)\n +- Exchange (7)\n +- * HashAggregate (6)\n +- Union (5)\n :- * Project (2)\n : +- Scan csv (1)\n +- * Project (4)\n +- Scan csv (3)\n+- == Initial Plan ==\n GlobalLimit (27)\n +- LocalLimit (26)\n +- Window (25)\n +- Sort (24)\n +- Exchange (23)\n +- HashAggregate (22)\n +- Exchange (21)\n +- HashAggregate (20)\n +- Union (19)\n :- Project (17)\n : +- Scan csv (1)\n +- Project (18)\n +- Scan csv (3)\n\n\n(1) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(2) Project [codegen id : 1]\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(3) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(4) Project [codegen id : 2]\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(5) Union\n\n(6) HashAggregate [codegen id : 3]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(7) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#438]\n\n(8) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 0, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)\n\n(9) CustomShuffleReader\nInput [1]: [airport#144]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 4]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(11) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#501]\n\n(12) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 1, Statistics(sizeInBytes=173.3 KiB, rowCount=6.34E+3, isRuntime=true)\n\n(13) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(14) Window\nInput [1]: [airport#144]\nArguments: [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21\n\n(15) LocalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(16) GlobalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(17) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(18) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(19) Union\n\n(20) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(21) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#341]\n\n(22) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(23) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#344]\n\n(24) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(25) Window\nInput [1]: [airport#144]\nArguments: [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21\n\n(26) LocalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(27) GlobalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(28) AdaptiveSparkPlan\nOutput [2]: [airport#144, airport_id#156]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"GlobalLimit","simpleString":"GlobalLimit 21","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Window","simpleString":"Window [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=173.3 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#501]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#438]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":501,"metricType":"timing"},{"name":"number of files read","accumulatorId":495,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":500,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":499,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":498,"metricType":"timing"},{"name":"metadata time","accumulatorId":496,"metricType":"timing"},{"name":"size of files read","accumulatorId":497,"metricType":"size"},{"name":"number of output rows","accumulatorId":494,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":528,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":509,"metricType":"timing"},{"name":"number of files read","accumulatorId":503,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":508,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":507,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":506,"metricType":"timing"},{"name":"metadata time","accumulatorId":504,"metricType":"timing"},{"name":"size of files read","accumulatorId":505,"metricType":"size"},{"name":"number of output rows","accumulatorId":502,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":529,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":524,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":525,"metricType":"timing"},{"name":"peak memory","accumulatorId":523,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":527,"metricType":"sum"},{"name":"number of output rows","accumulatorId":522,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":526,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":521,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":519,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":520,"metricType":"nsTiming"},{"name":"records read","accumulatorId":517,"metricType":"sum"},{"name":"local bytes read","accumulatorId":515,"metricType":"size"},{"name":"fetch wait time","accumulatorId":516,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":513,"metricType":"size"},{"name":"local blocks read","accumulatorId":512,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":511,"metricType":"sum"},{"name":"data size","accumulatorId":510,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":514,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":518,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":577,"metricType":"sum"},{"name":"partition data size","accumulatorId":578,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":574,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":575,"metricType":"timing"},{"name":"peak memory","accumulatorId":573,"metricType":"size"},{"name":"number of output rows","accumulatorId":572,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":576,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":571,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":569,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":570,"metricType":"nsTiming"},{"name":"records read","accumulatorId":567,"metricType":"sum"},{"name":"local bytes read","accumulatorId":565,"metricType":"size"},{"name":"fetch wait time","accumulatorId":566,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":563,"metricType":"size"},{"name":"local blocks read","accumulatorId":562,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":561,"metricType":"sum"},{"name":"data size","accumulatorId":560,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":564,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":568,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[577,1],[578,252030]]} +{"Event":"SparkListenerJobStart","Job ID":9,"Submission Time":1646432564209,"Stage Infos":[{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_438\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_430\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_434\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[30,33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_433\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_437\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_501\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_500\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_495\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[13],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[13,14],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_501\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"8","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_501\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_500\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_495\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[13],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432564213,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_501\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"8","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":14,"Stage Attempt ID":0,"Task Info":{"Task ID":55,"Index":0,"Attempt":0,"Launch Time":1646432564244,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":14,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":55,"Index":0,"Attempt":0,"Launch Time":1646432564244,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432564344,"Failed":false,"Killed":false,"Accumulables":[{"ID":511,"Name":"remote blocks read","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":512,"Name":"local blocks read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":513,"Name":"remote bytes read","Update":"178375","Value":"178375","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":515,"Name":"local bytes read","Update":"60106","Value":"60106","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":516,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":517,"Name":"records read","Update":"6337","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":560,"Name":"data size","Update":"9144","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":568,"Name":"shuffle bytes written","Update":"2249","Value":"2249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":569,"Name":"shuffle records written","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":570,"Name":"shuffle write time","Update":"305990","Value":"305990","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":571,"Name":"duration","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":572,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":573,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":575,"Name":"time in aggregation build","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":576,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":579,"Name":"internal.metrics.executorDeserializeTime","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":580,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19463225,"Value":19463225,"Internal":true,"Count Failed Values":true},{"ID":581,"Name":"internal.metrics.executorRunTime","Update":61,"Value":61,"Internal":true,"Count Failed Values":true},{"ID":582,"Name":"internal.metrics.executorCpuTime","Update":52909928,"Value":52909928,"Internal":true,"Count Failed Values":true},{"ID":583,"Name":"internal.metrics.resultSize","Update":3899,"Value":3899,"Internal":true,"Count Failed Values":true},{"ID":588,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":592,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":593,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":594,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":178375,"Value":178375,"Internal":true,"Count Failed Values":true},{"ID":595,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":596,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":60106,"Value":60106,"Internal":true,"Count Failed Values":true},{"ID":597,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":598,"Name":"internal.metrics.shuffle.read.recordsRead","Update":6337,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2249,"Value":2249,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":381,"Value":381,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.shuffle.write.writeTime","Update":305990,"Value":305990,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":19,"Executor Deserialize CPU Time":19463225,"Executor Run Time":61,"Executor CPU Time":52909928,"Peak Execution Memory":67371008,"Result Size":3899,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":15,"Local Blocks Fetched":5,"Fetch Wait Time":0,"Remote Bytes Read":178375,"Remote Bytes Read To Disk":0,"Local Bytes Read":60106,"Total Records Read":6337},"Shuffle Write Metrics":{"Shuffle Bytes Written":2249,"Shuffle Write Time":305990,"Shuffle Records Written":381},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_501\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_500\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_495\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[13],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432564213,"Completion Time":1646432564345,"Accumulables":[{"ID":511,"Name":"remote blocks read","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":512,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":513,"Name":"remote bytes read","Value":"178375","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":515,"Name":"local bytes read","Value":"60106","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":516,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":517,"Name":"records read","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":560,"Name":"data size","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":568,"Name":"shuffle bytes written","Value":"2249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":569,"Name":"shuffle records written","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":570,"Name":"shuffle write time","Value":"305990","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":571,"Name":"duration","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":572,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":573,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":575,"Name":"time in aggregation build","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":576,"Name":"avg hash probe bucket list iters","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":579,"Name":"internal.metrics.executorDeserializeTime","Value":19,"Internal":true,"Count Failed Values":true},{"ID":580,"Name":"internal.metrics.executorDeserializeCpuTime","Value":19463225,"Internal":true,"Count Failed Values":true},{"ID":581,"Name":"internal.metrics.executorRunTime","Value":61,"Internal":true,"Count Failed Values":true},{"ID":582,"Name":"internal.metrics.executorCpuTime","Value":52909928,"Internal":true,"Count Failed Values":true},{"ID":583,"Name":"internal.metrics.resultSize","Value":3899,"Internal":true,"Count Failed Values":true},{"ID":588,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":592,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":15,"Internal":true,"Count Failed Values":true},{"ID":593,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":594,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":178375,"Internal":true,"Count Failed Values":true},{"ID":595,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":596,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":60106,"Internal":true,"Count Failed Values":true},{"ID":597,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":598,"Name":"internal.metrics.shuffle.read.recordsRead","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2249,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":381,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.shuffle.write.writeTime","Value":305990,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":9,"Completion Time":1646432564349,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":8,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (28)\n+- == Final Plan ==\n * GlobalLimit (16)\n +- * LocalLimit (15)\n +- Window (14)\n +- Sort (13)\n +- ShuffleQueryStage (12)\n +- Exchange (11)\n +- * HashAggregate (10)\n +- CustomShuffleReader (9)\n +- ShuffleQueryStage (8)\n +- Exchange (7)\n +- * HashAggregate (6)\n +- Union (5)\n :- * Project (2)\n : +- Scan csv (1)\n +- * Project (4)\n +- Scan csv (3)\n+- == Initial Plan ==\n GlobalLimit (27)\n +- LocalLimit (26)\n +- Window (25)\n +- Sort (24)\n +- Exchange (23)\n +- HashAggregate (22)\n +- Exchange (21)\n +- HashAggregate (20)\n +- Union (19)\n :- Project (17)\n : +- Scan csv (1)\n +- Project (18)\n +- Scan csv (3)\n\n\n(1) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(2) Project [codegen id : 1]\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(3) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(4) Project [codegen id : 2]\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(5) Union\n\n(6) HashAggregate [codegen id : 3]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(7) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#438]\n\n(8) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 0, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)\n\n(9) CustomShuffleReader\nInput [1]: [airport#144]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 4]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(11) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#501]\n\n(12) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 1, Statistics(sizeInBytes=8.9 KiB, rowCount=381, isRuntime=true)\n\n(13) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(14) Window\nInput [1]: [airport#144]\nArguments: [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21\n\n(15) LocalLimit [codegen id : 5]\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(16) GlobalLimit [codegen id : 5]\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(17) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(18) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(19) Union\n\n(20) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(21) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#341]\n\n(22) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(23) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#344]\n\n(24) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(25) Window\nInput [1]: [airport#144]\nArguments: [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21\n\n(26) LocalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(27) GlobalLimit\nInput [2]: [airport#144, airport_id#156]\nArguments: 21\n\n(28) AdaptiveSparkPlan\nOutput [2]: [airport#144, airport_id#156]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen","children":[{"nodeName":"GlobalLimit","simpleString":"GlobalLimit 21","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Window","simpleString":"Window [airport#144, cast(row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) as string) AS airport_id#156], [airport#144 ASC NULLS FIRST], 21","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=8.9 KiB, rowCount=381, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#501]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#438]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":501,"metricType":"timing"},{"name":"number of files read","accumulatorId":495,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":500,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":499,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":498,"metricType":"timing"},{"name":"metadata time","accumulatorId":496,"metricType":"timing"},{"name":"size of files read","accumulatorId":497,"metricType":"size"},{"name":"number of output rows","accumulatorId":494,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":528,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":509,"metricType":"timing"},{"name":"number of files read","accumulatorId":503,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":508,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":507,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":506,"metricType":"timing"},{"name":"metadata time","accumulatorId":504,"metricType":"timing"},{"name":"size of files read","accumulatorId":505,"metricType":"size"},{"name":"number of output rows","accumulatorId":502,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":529,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":524,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":525,"metricType":"timing"},{"name":"peak memory","accumulatorId":523,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":527,"metricType":"sum"},{"name":"number of output rows","accumulatorId":522,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":526,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":521,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":519,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":520,"metricType":"nsTiming"},{"name":"records read","accumulatorId":517,"metricType":"sum"},{"name":"local bytes read","accumulatorId":515,"metricType":"size"},{"name":"fetch wait time","accumulatorId":516,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":513,"metricType":"size"},{"name":"local blocks read","accumulatorId":512,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":511,"metricType":"sum"},{"name":"data size","accumulatorId":510,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":514,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":518,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":577,"metricType":"sum"},{"name":"partition data size","accumulatorId":578,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":574,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":575,"metricType":"timing"},{"name":"peak memory","accumulatorId":573,"metricType":"size"},{"name":"number of output rows","accumulatorId":572,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":576,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":571,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":569,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":570,"metricType":"nsTiming"},{"name":"records read","accumulatorId":567,"metricType":"sum"},{"name":"local bytes read","accumulatorId":565,"metricType":"size"},{"name":"fetch wait time","accumulatorId":566,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":563,"metricType":"size"},{"name":"local blocks read","accumulatorId":562,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":561,"metricType":"sum"},{"name":"data size","accumulatorId":560,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":564,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":568,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":612,"metricType":"size"},{"name":"cumulative time","accumulatorId":613,"metricType":"timing"},{"name":"peak memory","accumulatorId":611,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":615,"metricType":"sum"},{"name":"sort time","accumulatorId":610,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":616,"metricType":"sum"},{"name":"spill write time","accumulatorId":614,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":609,"metricType":"timing"}]}],"metadata":{},"metrics":[]}} +{"Event":"SparkListenerJobStart","Job ID":10,"Submission Time":1646432564391,"Stage Infos":[{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_438\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_430\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_434\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[30,33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_332\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_330\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_433\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_437\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":16,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_501\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_500\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_495\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[15],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":17,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":43,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_533\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1\",\"name\":\"ShuffleQueryStage\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_509\",\"name\":\"Sort\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_510\",\"name\":\"Window\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[16],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[15,16,17],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"8","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":17,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":43,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_533\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1\",\"name\":\"ShuffleQueryStage\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_509\",\"name\":\"Sort\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_510\",\"name\":\"Window\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[16],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432564394,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"8","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":17,"Stage Attempt ID":0,"Task Info":{"Task ID":56,"Index":0,"Attempt":0,"Launch Time":1646432564425,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":17,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":56,"Index":0,"Attempt":0,"Launch Time":1646432564425,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432564721,"Failed":false,"Killed":false,"Accumulables":[{"ID":562,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":565,"Name":"local bytes read","Update":"2249","Value":"2249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":567,"Name":"records read","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":609,"Name":"duration","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":610,"Name":"sort time","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":611,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":612,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":613,"Name":"cumulative time","Update":"76","Value":"76","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":614,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":615,"Name":"num prefix comparisons","Update":"2773","Value":"2773","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":617,"Name":"internal.metrics.executorDeserializeTime","Update":85,"Value":85,"Internal":true,"Count Failed Values":true},{"ID":618,"Name":"internal.metrics.executorDeserializeCpuTime","Update":84894808,"Value":84894808,"Internal":true,"Count Failed Values":true},{"ID":619,"Name":"internal.metrics.executorRunTime","Update":191,"Value":191,"Internal":true,"Count Failed Values":true},{"ID":620,"Name":"internal.metrics.executorCpuTime","Update":168798972,"Value":168798972,"Internal":true,"Count Failed Values":true},{"ID":621,"Name":"internal.metrics.resultSize","Update":4853,"Value":4853,"Internal":true,"Count Failed Values":true},{"ID":626,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":630,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":631,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":632,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":633,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":634,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2249,"Value":2249,"Internal":true,"Count Failed Values":true},{"ID":635,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":636,"Name":"internal.metrics.shuffle.read.recordsRead","Update":381,"Value":381,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":85,"Executor Deserialize CPU Time":84894808,"Executor Run Time":191,"Executor CPU Time":168798972,"Peak Execution Memory":67174400,"Result Size":4853,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2249,"Total Records Read":381},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":17,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":43,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_533\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1\",\"name\":\"ShuffleQueryStage\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_509\",\"name\":\"Sort\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_510\",\"name\":\"Window\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[16],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432564394,"Completion Time":1646432564721,"Accumulables":[{"ID":562,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":565,"Name":"local bytes read","Value":"2249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":566,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":567,"Name":"records read","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":609,"Name":"duration","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":610,"Name":"sort time","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":611,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":612,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":613,"Name":"cumulative time","Value":"76","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":614,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":615,"Name":"num prefix comparisons","Value":"2773","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":617,"Name":"internal.metrics.executorDeserializeTime","Value":85,"Internal":true,"Count Failed Values":true},{"ID":618,"Name":"internal.metrics.executorDeserializeCpuTime","Value":84894808,"Internal":true,"Count Failed Values":true},{"ID":619,"Name":"internal.metrics.executorRunTime","Value":191,"Internal":true,"Count Failed Values":true},{"ID":620,"Name":"internal.metrics.executorCpuTime","Value":168798972,"Internal":true,"Count Failed Values":true},{"ID":621,"Name":"internal.metrics.resultSize","Value":4853,"Internal":true,"Count Failed Values":true},{"ID":626,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":630,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":631,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":632,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":633,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":634,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2249,"Internal":true,"Count Failed Values":true},{"ID":635,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":636,"Name":"internal.metrics.shuffle.read.recordsRead","Value":381,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":10,"Completion Time":1646432564721,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":8,"time":1646432564761,"errorMessage":""} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":9,"description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","physicalPlanDescription":"== Physical Plan ==\nCollectLimit (54)\n+- AdaptiveSparkPlan (53)\n +- Project (52)\n +- SortMergeJoin Inner (51)\n :- Sort (40)\n : +- Exchange (39)\n : +- Project (38)\n : +- SortMergeJoin Inner (37)\n : :- Sort (22)\n : : +- Exchange (21)\n : : +- Project (20)\n : : +- SortMergeJoin Inner (19)\n : : :- Sort (4)\n : : : +- Exchange (3)\n : : : +- Filter (2)\n : : : +- Scan csv (1)\n : : +- Sort (18)\n : : +- Exchange (17)\n : : +- Filter (16)\n : : +- RunningWindowFunction (15)\n : : +- Sort (14)\n : : +- Exchange (13)\n : : +- HashAggregate (12)\n : : +- Exchange (11)\n : : +- HashAggregate (10)\n : : +- Union (9)\n : : :- Project (6)\n : : : +- Scan csv (5)\n : : +- Project (8)\n : : +- Scan csv (7)\n : +- Sort (36)\n : +- Exchange (35)\n : +- Filter (34)\n : +- RunningWindowFunction (33)\n : +- Sort (32)\n : +- Exchange (31)\n : +- HashAggregate (30)\n : +- Exchange (29)\n : +- HashAggregate (28)\n : +- Union (27)\n : :- Project (24)\n : : +- Scan csv (23)\n : +- Project (26)\n : +- Scan csv (25)\n +- Sort (50)\n +- Exchange (49)\n +- Filter (48)\n +- RunningWindowFunction (47)\n +- Sort (46)\n +- Exchange (45)\n +- HashAggregate (44)\n +- Exchange (43)\n +- HashAggregate (42)\n +- Scan csv (41)\n\n\n(1) Scan csv \nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nPushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]\nReadSchema: struct\n\n(2) Filter\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(3) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#665]\n\n(4) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(5) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(6) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(7) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(8) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(9) Union\n\n(10) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(11) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#656]\n\n(12) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(13) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#659]\n\n(14) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(15) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(16) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(17) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#666]\n\n(18) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(19) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(20) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(21) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#680]\n\n(22) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(23) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(24) Project\nOutput [1]: [ORIGIN#21 AS airport#178]\nInput [1]: [ORIGIN#21]\n\n(25) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(26) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(27) Union\n\n(28) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(29) Exchange\nInput [1]: [airport#178]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#671]\n\n(30) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(31) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#674]\n\n(32) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(33) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(34) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(35) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#681]\n\n(36) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(37) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(38) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(39) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#695]\n\n(40) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(41) Scan csv \nOutput [1]: [OP_CARRIER#180]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(42) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(43) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#686]\n\n(44) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(45) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#689]\n\n(46) Sort\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(47) RunningWindowFunction\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(48) Filter\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(49) Exchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#696]\n\n(50) Sort\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(51) SortMergeJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(52) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(53) AdaptiveSparkPlan\nOutput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: isFinalPlan=false\n\n(54) CollectLimit\nInput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: 11\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 11","children":[{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [OP_CARRIER#19], [OP_CARRIER#180], Inner, AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#19 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#695]","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [DEST#22], [airport#178], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [DEST#22 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#680]","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [ORIGIN#21], [airport#144], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [ORIGIN#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#665]","children":[{"nodeName":"Filter","simpleString":"Filter ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [FL_DATE#18,OP_CARRIER#19,ORIGIN#21,DEST#22,CRS_DEP_TIME#23,DEP_DELAY#25] Batched: false, DataFilters: [isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)], ReadSchema: struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]","DataFilters":"[isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)]"},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#666]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#144)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#659]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#656]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [airport#178 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#681]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#178)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#178 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#674]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#178], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#671]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#178], functions=[])","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#178]","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#180 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#696]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(OP_CARRIER#180)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#180 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#689]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#686]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#180] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":655,"metricType":"sum"},{"name":"records read","accumulatorId":653,"metricType":"sum"},{"name":"local bytes read","accumulatorId":651,"metricType":"size"},{"name":"fetch wait time","accumulatorId":652,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":649,"metricType":"size"},{"name":"local blocks read","accumulatorId":648,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":647,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":650,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":654,"metricType":"size"}]},"time":1646432565262} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":9,"physicalPlanDescription":"== Physical Plan ==\nCollectLimit (100)\n+- AdaptiveSparkPlan (99)\n +- == Current Plan ==\n Project (50)\n +- SortMergeJoin Inner (49)\n :- Sort (37)\n : +- Exchange (36)\n : +- Project (35)\n : +- SortMergeJoin Inner (34)\n : :- Sort (24)\n : : +- Exchange (23)\n : : +- Project (22)\n : : +- SortMergeJoin Inner (21)\n : : :- Sort (5)\n : : : +- ShuffleQueryStage (4)\n : : : +- Exchange (3)\n : : : +- * Filter (2)\n : : : +- Scan csv (1)\n : : +- Sort (20)\n : : +- Exchange (19)\n : : +- Filter (18)\n : : +- RunningWindowFunction (17)\n : : +- Sort (16)\n : : +- Exchange (15)\n : : +- HashAggregate (14)\n : : +- ShuffleQueryStage (13)\n : : +- Exchange (12)\n : : +- * HashAggregate (11)\n : : +- Union (10)\n : : :- * Project (7)\n : : : +- Scan csv (6)\n : : +- * Project (9)\n : : +- Scan csv (8)\n : +- Sort (33)\n : +- Exchange (32)\n : +- Filter (31)\n : +- RunningWindowFunction (30)\n : +- Sort (29)\n : +- Exchange (28)\n : +- HashAggregate (27)\n : +- ShuffleQueryStage (26)\n : +- ReusedExchange (25)\n +- Sort (48)\n +- Exchange (47)\n +- Filter (46)\n +- RunningWindowFunction (45)\n +- Sort (44)\n +- Exchange (43)\n +- HashAggregate (42)\n +- ShuffleQueryStage (41)\n +- Exchange (40)\n +- * HashAggregate (39)\n +- Scan csv (38)\n +- == Initial Plan ==\n Project (98)\n +- SortMergeJoin Inner (97)\n :- Sort (87)\n : +- Exchange (86)\n : +- Project (85)\n : +- SortMergeJoin Inner (84)\n : :- Sort (69)\n : : +- Exchange (68)\n : : +- Project (67)\n : : +- SortMergeJoin Inner (66)\n : : :- Sort (53)\n : : : +- Exchange (52)\n : : : +- Filter (51)\n : : : +- Scan csv (1)\n : : +- Sort (65)\n : : +- Exchange (64)\n : : +- Filter (63)\n : : +- RunningWindowFunction (62)\n : : +- Sort (61)\n : : +- Exchange (60)\n : : +- HashAggregate (59)\n : : +- Exchange (58)\n : : +- HashAggregate (57)\n : : +- Union (56)\n : : :- Project (54)\n : : : +- Scan csv (6)\n : : +- Project (55)\n : : +- Scan csv (8)\n : +- Sort (83)\n : +- Exchange (82)\n : +- Filter (81)\n : +- RunningWindowFunction (80)\n : +- Sort (79)\n : +- Exchange (78)\n : +- HashAggregate (77)\n : +- Exchange (76)\n : +- HashAggregate (75)\n : +- Union (74)\n : :- Project (71)\n : : +- Scan csv (70)\n : +- Project (73)\n : +- Scan csv (72)\n +- Sort (96)\n +- Exchange (95)\n +- Filter (94)\n +- RunningWindowFunction (93)\n +- Sort (92)\n +- Exchange (91)\n +- HashAggregate (90)\n +- Exchange (89)\n +- HashAggregate (88)\n +- Scan csv (38)\n\n\n(1) Scan csv \nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nPushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]\nReadSchema: struct\n\n(2) Filter [codegen id : 1]\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(3) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]\n\n(4) ShuffleQueryStage\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: 0, Statistics(sizeInBytes=1816.0 MiB)\n\n(5) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(6) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(7) Project [codegen id : 2]\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(8) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(9) Project [codegen id : 3]\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(10) Union\n\n(11) HashAggregate [codegen id : 4]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(12) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]\n\n(13) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 1, Statistics(sizeInBytes=3.5 GiB)\n\n(14) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(15) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1086]\n\n(16) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(17) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(18) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(19) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1090]\n\n(20) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(21) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(22) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(23) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#1094]\n\n(24) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(25) ReusedExchange [Reuses operator id: 12]\nOutput [1]: [airport#178]\n\n(26) ShuffleQueryStage\nOutput [1]: [airport#178]\nArguments: 3, Statistics(sizeInBytes=3.5 GiB)\n\n(27) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(28) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1129]\n\n(29) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(30) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(31) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(32) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#1133]\n\n(33) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(34) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(35) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(36) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#1137]\n\n(37) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(38) Scan csv \nOutput [1]: [OP_CARRIER#180]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(39) HashAggregate [codegen id : 8]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(40) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]\n\n(41) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 4, Statistics(sizeInBytes=1816.0 MiB)\n\n(42) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(43) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1153]\n\n(44) Sort\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(45) RunningWindowFunction\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(46) Filter\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(47) Exchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1157]\n\n(48) Sort\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(49) SortMergeJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(50) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(51) Filter\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(52) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#665]\n\n(53) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(54) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(55) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(56) Union\n\n(57) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(58) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#656]\n\n(59) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(60) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#659]\n\n(61) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(62) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(63) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(64) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#666]\n\n(65) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(66) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(67) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(68) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#680]\n\n(69) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(70) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(71) Project\nOutput [1]: [ORIGIN#21 AS airport#178]\nInput [1]: [ORIGIN#21]\n\n(72) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(73) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(74) Union\n\n(75) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(76) Exchange\nInput [1]: [airport#178]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#671]\n\n(77) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(78) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#674]\n\n(79) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(80) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(81) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(82) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#681]\n\n(83) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(84) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(85) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(86) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#695]\n\n(87) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(88) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(89) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#686]\n\n(90) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(91) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#689]\n\n(92) Sort\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(93) RunningWindowFunction\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(94) Filter\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(95) Exchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#696]\n\n(96) Sort\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(97) SortMergeJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(98) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(99) AdaptiveSparkPlan\nOutput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: isFinalPlan=false\n\n(100) CollectLimit\nInput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: 11\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 11","children":[{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [OP_CARRIER#19], [OP_CARRIER#180], Inner, AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#19 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#1137]","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [DEST#22], [airport#178], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [DEST#22 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#1094]","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [ORIGIN#21], [airport#144], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [ORIGIN#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=1816.0 MiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [FL_DATE#18,OP_CARRIER#19,ORIGIN#21,DEST#22,CRS_DEP_TIME#23,DEP_DELAY#25] Batched: false, DataFilters: [isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)], ReadSchema: struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]","DataFilters":"[isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)]"},"metrics":[{"name":"file sorting by size time","accumulatorId":722,"metricType":"timing"},{"name":"number of files read","accumulatorId":716,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":721,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":720,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":719,"metricType":"timing"},{"name":"metadata time","accumulatorId":717,"metricType":"timing"},{"name":"size of files read","accumulatorId":718,"metricType":"size"},{"name":"number of output rows","accumulatorId":715,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":952,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":951,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":949,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":950,"metricType":"nsTiming"},{"name":"records read","accumulatorId":947,"metricType":"sum"},{"name":"local bytes read","accumulatorId":945,"metricType":"size"},{"name":"fetch wait time","accumulatorId":946,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":943,"metricType":"size"},{"name":"local blocks read","accumulatorId":942,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":941,"metricType":"sum"},{"name":"data size","accumulatorId":940,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":944,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":948,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1090]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#144)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1086]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=3.5 GiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [airport#178 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#1133]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#178)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#178 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1129]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#178], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3, Statistics(sizeInBytes=3.5 GiB)","children":[{"nodeName":"ReusedExchange","simpleString":"ReusedExchange [airport#178], Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#180 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1157]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(OP_CARRIER#180)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#180 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1153]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4, Statistics(sizeInBytes=1816.0 MiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#180] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":939,"metricType":"timing"},{"name":"number of files read","accumulatorId":933,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":938,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":937,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":936,"metricType":"timing"},{"name":"metadata time","accumulatorId":934,"metricType":"timing"},{"name":"size of files read","accumulatorId":935,"metricType":"size"},{"name":"number of output rows","accumulatorId":932,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":987,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":988,"metricType":"timing"},{"name":"peak memory","accumulatorId":986,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":990,"metricType":"sum"},{"name":"number of output rows","accumulatorId":985,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":989,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":984,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":982,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":983,"metricType":"nsTiming"},{"name":"records read","accumulatorId":980,"metricType":"sum"},{"name":"local bytes read","accumulatorId":978,"metricType":"size"},{"name":"fetch wait time","accumulatorId":979,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":976,"metricType":"size"},{"name":"local blocks read","accumulatorId":975,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":974,"metricType":"sum"},{"name":"data size","accumulatorId":973,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":977,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":981,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":655,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":656,"metricType":"nsTiming"},{"name":"records read","accumulatorId":653,"metricType":"sum"},{"name":"local bytes read","accumulatorId":651,"metricType":"size"},{"name":"fetch wait time","accumulatorId":652,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":649,"metricType":"size"},{"name":"local blocks read","accumulatorId":648,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":647,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":650,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":654,"metricType":"size"}]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":9,"accumUpdates":[[722,0],[716,10],[717,0],[718,1904190334]]} +{"Event":"SparkListenerJobStart","Job ID":11,"Submission Time":1646432565352,"Stage Infos":[{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1046\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1045\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_616\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_616\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[18],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1046\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1046\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1045\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_616\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_616\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432565356,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1046\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":57,"Index":0,"Attempt":0,"Launch Time":1646432565375,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":58,"Index":1,"Attempt":0,"Launch Time":1646432565375,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":59,"Index":2,"Attempt":0,"Launch Time":1646432565376,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":60,"Index":3,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":61,"Index":4,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":62,"Index":5,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":63,"Index":6,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":64,"Index":7,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":65,"Index":8,"Attempt":0,"Launch Time":1646432565378,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":66,"Index":9,"Attempt":0,"Launch Time":1646432565378,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":9,"accumUpdates":[[789,0],[783,10],[784,0],[785,1904190334]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":9,"accumUpdates":[[797,0],[791,10],[792,0],[793,1904190334]]} +{"Event":"SparkListenerJobStart","Job ID":12,"Submission Time":1646432565455,"Stage Infos":[{"Stage ID":19,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1066\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":54,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_1062\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[50,53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1065\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1061\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1058\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[19],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1066\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":19,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1066\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":54,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_1062\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[50,53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1065\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1061\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1058\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432565464,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1066\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":67,"Index":0,"Attempt":0,"Launch Time":1646432565510,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":68,"Index":1,"Attempt":0,"Launch Time":1646432565510,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":69,"Index":2,"Attempt":0,"Launch Time":1646432565510,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":70,"Index":3,"Attempt":0,"Launch Time":1646432565510,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":71,"Index":4,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":72,"Index":5,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":73,"Index":6,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":74,"Index":7,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":75,"Index":8,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":76,"Index":9,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":77,"Index":10,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":78,"Index":11,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":79,"Index":12,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":80,"Index":13,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":81,"Index":14,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":82,"Index":15,"Attempt":0,"Launch Time":1646432565513,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":83,"Index":16,"Attempt":0,"Launch Time":1646432565513,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":84,"Index":17,"Attempt":0,"Launch Time":1646432565513,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":85,"Index":18,"Attempt":0,"Launch Time":1646432565514,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":19,"Stage Attempt ID":0,"Task Info":{"Task ID":86,"Index":19,"Attempt":0,"Launch Time":1646432565514,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":9,"accumUpdates":[[939,0],[933,10],[934,0],[935,1904190334]]} +{"Event":"SparkListenerJobStart","Job ID":13,"Submission Time":1646432565567,"Stage Infos":[{"Stage ID":20,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1142\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1141\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[20],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1142\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":20,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1142\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1141\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432565571,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1142\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":87,"Index":0,"Attempt":0,"Launch Time":1646432565597,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":88,"Index":1,"Attempt":0,"Launch Time":1646432565597,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":89,"Index":2,"Attempt":0,"Launch Time":1646432565597,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":90,"Index":3,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":91,"Index":4,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":92,"Index":5,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":93,"Index":6,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":94,"Index":7,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":95,"Index":8,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":20,"Stage Attempt ID":0,"Task Info":{"Task ID":96,"Index":9,"Attempt":0,"Launch Time":1646432565599,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":76,"Index":9,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432578478,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"5617658","Value":"5617658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1445","Value":"1445","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7512","Value":"7512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11907","Value":"11907","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"313","Value":"313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"14654074","Value":"14654074","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"12879","Value":"12879","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"313","Value":"313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"12861","Value":"12861","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"12903","Value":"12903","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6128413,"Value":6128413,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":12928,"Value":12928,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":12521938004,"Value":12521938004,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":2563,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":127,"Value":127,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11907,"Value":11907,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":313,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":14654074,"Value":14654074,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":5617658,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1445251821,"Value":1445251821,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4655239904,"JVMOffHeapMemory":127611656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460414,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":522,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":6128413,"Executor Run Time":12928,"Executor CPU Time":12521938004,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":127,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11907,"Shuffle Write Time":14654074,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":94,"Index":7,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432578554,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"5819811","Value":"5819811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1378","Value":"1378","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"336","Value":"336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"952","Value":"952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"6282820","Value":"6282820","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"12881","Value":"12881","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"12869","Value":"12869","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3784000,"Value":3784000,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":12921,"Value":12921,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":12598508599,"Value":12598508599,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2132,"Value":2132,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":159,"Value":159,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":952,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":6282820,"Value":6282820,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":180693843,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":5819811,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1378502923,"Value":1378502923,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":180693843,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":5772991312,"JVMOffHeapMemory":127072720,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1398747,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":56,"MinorGCTime":553,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3784000,"Executor Run Time":12921,"Executor CPU Time":12598508599,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":159,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":6282820,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":74,"Index":7,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432578905,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"5819811","Value":"11437469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1293","Value":"2738","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"356626668","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"180693843","Value":"356626668","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7800","Value":"15312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"12061","Value":"23968","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"325","Value":"638","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"14552557","Value":"29206631","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"13311","Value":"26190","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"325","Value":"638","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"13284","Value":"26145","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"2007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"13332","Value":"26235","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5338619,"Value":11467032,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":13358,"Value":26286,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":13088603316,"Value":25610541320,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":5126,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":129,"Value":256,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12061,"Value":23968,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":325,"Value":638,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":14552557,"Value":29206631,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":356626668,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":11437469,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1293531113,"Value":2738782934,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":356626668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":124878648,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1457840,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":483,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5338619,"Executor Run Time":13358,"Executor CPU Time":13088603316,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":129,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12061,"Shuffle Write Time":14552557,"Shuffle Records Written":325},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":72,"Index":5,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432578925,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"6096762","Value":"17534231","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1260","Value":"3998","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"540566176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"183939508","Value":"540566176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7488","Value":"22800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11723","Value":"35691","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"312","Value":"950","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"13458731","Value":"42665362","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"13328","Value":"39518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"312","Value":"950","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"13310","Value":"39455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"3007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"13353","Value":"39588","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6112524,"Value":17579556,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":13376,"Value":39662,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":13138355004,"Value":38748896324,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":7689,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":127,"Value":383,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11723,"Value":35691,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":312,"Value":950,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":13458731,"Value":42665362,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":540566176,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":17534231,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1260417708,"Value":3999200642,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":540566176,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4655239904,"JVMOffHeapMemory":127611656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460414,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":522,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":6112524,"Executor Run Time":13376,"Executor CPU Time":13138355004,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":127,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11723,"Shuffle Write Time":13458731,"Shuffle Records Written":312},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":86,"Index":19,"Attempt":0,"Launch Time":1646432565514,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432579084,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"5617658","Value":"5617658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1243","Value":"1243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7440","Value":"30240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11887","Value":"47578","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"310","Value":"1260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"18783126","Value":"61448488","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"13486","Value":"53004","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"310","Value":"1260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"13461","Value":"52916","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"4014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"13494","Value":"13494","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5352834,"Value":22932390,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":13537,"Value":53199,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":13308660643,"Value":52057556967,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":10252,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":129,"Value":512,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11887,"Value":47578,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":310,"Value":1260,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":18783126,"Value":61448488,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":716499001,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":23151889,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1243196742,"Value":5242397384,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":716499001,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":124878648,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1457840,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":483,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5352834,"Executor Run Time":13537,"Executor CPU Time":13308660643,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":129,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11887,"Shuffle Write Time":18783126,"Shuffle Records Written":310},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":90,"Index":3,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432579568,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"6429338","Value":"12249149","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1479","Value":"2857","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"375768158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"195074315","Value":"375768158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"456","Value":"792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"1232","Value":"2184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"19","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"4831030","Value":"11113850","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"13897","Value":"26778","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"19","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"13887","Value":"26756","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4026164,"Value":7810164,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":13937,"Value":26858,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":13615400159,"Value":26213908758,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2132,"Value":4264,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":170,"Value":329,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1232,"Value":2184,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":19,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":4831030,"Value":11113850,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":375768158,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":12249149,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1479254855,"Value":2857757778,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":375768158,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":5772991312,"JVMOffHeapMemory":127072720,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1398747,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":56,"MinorGCTime":553,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4026164,"Executor Run Time":13937,"Executor CPU Time":13615400159,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":170,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1232,"Shuffle Write Time":4831030,"Shuffle Records Written":19},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":85,"Index":18,"Attempt":0,"Launch Time":1646432565514,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432579635,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"5674621","Value":"11292279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1445","Value":"2688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"178217644","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7680","Value":"37920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"12025","Value":"59603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"320","Value":"1580","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"16290399","Value":"77738887","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"14015","Value":"67019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"320","Value":"1580","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"13994","Value":"66910","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"14031","Value":"27525","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5115685,"Value":28048075,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":14086,"Value":67285,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":13662757185,"Value":65720314152,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2562,"Value":12814,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":170,"Value":682,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12025,"Value":59603,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":1580,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":16290399,"Value":77738887,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":894716645,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":28826510,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1445921480,"Value":6688318864,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":894716645,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":5772991312,"JVMOffHeapMemory":127072720,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1398747,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":56,"MinorGCTime":553,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":5115685,"Executor Run Time":14086,"Executor CPU Time":13662757185,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":170,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12025,"Shuffle Write Time":16290399,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":71,"Index":4,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432579769,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"6066650","Value":"23600881","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1310","Value":"5308","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"725602056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"185035880","Value":"725602056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7176","Value":"45096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11443","Value":"71046","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"299","Value":"1879","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"13996378","Value":"91735265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"14173","Value":"81192","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"299","Value":"1879","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"14154","Value":"81064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"14197","Value":"53785","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5330225,"Value":33378300,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":14221,"Value":81506,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":13738607718,"Value":79458921870,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":15377,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":343,"Value":1025,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11443,"Value":71046,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":299,"Value":1879,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":13996378,"Value":91735265,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1079752525,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":34893160,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1310943418,"Value":7999262282,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1079752525,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":129174864,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1444437,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":59,"MinorGCTime":890,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":5330225,"Executor Run Time":14221,"Executor CPU Time":13738607718,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":343,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11443,"Shuffle Write Time":13996378,"Shuffle Records Written":299},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":73,"Index":6,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432580240,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"5819079","Value":"29419960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1607","Value":"6915","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"907838921","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"182236865","Value":"907838921","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7728","Value":"52824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"12103","Value":"83149","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"322","Value":"2201","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"13502990","Value":"105238255","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"14620","Value":"95812","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"322","Value":"2201","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"14600","Value":"95664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"14662","Value":"68447","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6495479,"Value":39873779,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":14687,"Value":96193,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":14203625709,"Value":93662547579,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2562,"Value":17939,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":170,"Value":1195,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12103,"Value":83149,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":2201,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":13502990,"Value":105238255,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":1261989390,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":40712239,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1607327044,"Value":9606589326,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":1261989390,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":5772991312,"JVMOffHeapMemory":127072720,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1398747,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":56,"MinorGCTime":553,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":6495479,"Executor Run Time":14687,"Executor CPU Time":14203625709,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":170,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12103,"Shuffle Write Time":13502990,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":80,"Index":13,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432580301,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"6429338","Value":"17721617","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1453","Value":"4141","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"549224784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"195074315","Value":"549224784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7104","Value":"59928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11481","Value":"94630","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"296","Value":"2497","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"12486091","Value":"117724346","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"14704","Value":"110516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"296","Value":"2497","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"14687","Value":"110351","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"8021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"14715","Value":"42240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":53,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5065322,"Value":44939101,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":14753,"Value":110946,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":14470866374,"Value":108133413953,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":20502,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":136,"Value":1331,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11481,"Value":94630,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":2497,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":12486091,"Value":117724346,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1457063705,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":47141577,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1453071418,"Value":11059660744,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1457063705,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4655239904,"JVMOffHeapMemory":127611656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460414,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":522,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":5065322,"Executor Run Time":14753,"Executor CPU Time":14470866374,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":136,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11481,"Shuffle Write Time":12486091,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":95,"Index":8,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432580527,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"5674621","Value":"17923770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1445","Value":"4302","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"553985802","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"178217644","Value":"553985802","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"288","Value":"1080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"816","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"12","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"3856231","Value":"14970081","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"14864","Value":"41642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"12","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"14850","Value":"41606","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3600034,"Value":11410198,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":14896,"Value":41754,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":14651130430,"Value":40865039188,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2133,"Value":6397,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":136,"Value":465,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":3000,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":3856231,"Value":14970081,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":553985802,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":17923770,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1445661963,"Value":4303419741,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":553985802,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4655239904,"JVMOffHeapMemory":127611656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460414,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":522,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3600034,"Executor Run Time":14896,"Executor CPU Time":14651130430,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":136,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":3856231,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":84,"Index":17,"Attempt":0,"Launch Time":1646432565513,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432580924,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"5819811","Value":"23541428","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1430","Value":"5571","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"729918627","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"180693843","Value":"729918627","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7776","Value":"67704","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"12053","Value":"106683","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"324","Value":"2821","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"12222658","Value":"129947004","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"15328","Value":"125844","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"324","Value":"2821","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"15312","Value":"125663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"9028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"15338","Value":"57578","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4615759,"Value":49554860,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":15377,"Value":126323,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":15102115350,"Value":123235529303,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":23065,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":136,"Value":1467,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12053,"Value":106683,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":324,"Value":2821,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":12222658,"Value":129947004,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":1637757548,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":52961388,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1430631780,"Value":12490292524,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":1637757548,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4655239904,"JVMOffHeapMemory":127611656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460414,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":522,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":4615759,"Executor Run Time":15377,"Executor CPU Time":15102115350,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":136,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12053,"Shuffle Write Time":12222658,"Shuffle Records Written":324},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":96,"Index":9,"Attempt":0,"Launch Time":1646432565599,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432581033,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"5617658","Value":"23541428","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1439","Value":"5741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"729918627","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"175932825","Value":"729918627","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"288","Value":"1368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"816","Value":"3816","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"12","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"4032555","Value":"19002636","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"15360","Value":"57002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"12","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"15350","Value":"56956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"4000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3627936,"Value":15038134,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":15389,"Value":57143,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":15161064366,"Value":56026103554,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2133,"Value":8530,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":138,"Value":603,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":3816,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":4032555,"Value":19002636,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":729918627,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":23541428,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1439658942,"Value":5743078683,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":729918627,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":124878648,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1457840,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":483,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":3627936,"Executor Run Time":15389,"Executor CPU Time":15161064366,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":138,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":4032555,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":83,"Index":16,"Attempt":0,"Launch Time":1646432565513,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432581428,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"5819079","Value":"29360507","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1350","Value":"6921","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"912155492","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"182236865","Value":"912155492","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7728","Value":"75432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"12099","Value":"118782","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"322","Value":"3143","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"13838582","Value":"143785586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"15833","Value":"141677","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"322","Value":"3143","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"15816","Value":"141479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"15840","Value":"73418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":64,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4875652,"Value":54430512,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":15882,"Value":142205,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":15398827997,"Value":138634357300,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":25628,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":348,"Value":1815,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12099,"Value":118782,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":3143,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":13838582,"Value":143785586,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":1819994413,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":58780467,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1350621966,"Value":13840914490,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":1819994413,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":129174864,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1445149,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":917,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4875652,"Executor Run Time":15882,"Executor CPU Time":15398827997,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":348,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12099,"Shuffle Write Time":13838582,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":92,"Index":5,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432581573,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"6096762","Value":"29638190","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1473","Value":"7214","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"913858135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"183939508","Value":"913858135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"360","Value":"1728","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"1020","Value":"4836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"15","Value":"72","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"4183618","Value":"23186254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"15899","Value":"72901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"15","Value":"72","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"15889","Value":"72845","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3009033,"Value":18047167,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":15930,"Value":73073,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":15700608130,"Value":71726711684,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2133,"Value":10663,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":138,"Value":741,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1020,"Value":4836,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":15,"Value":72,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":4183618,"Value":23186254,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":913858135,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":29638190,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1473838856,"Value":7216917539,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":913858135,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":124878648,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1457840,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":483,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3009033,"Executor Run Time":15930,"Executor CPU Time":15700608130,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":138,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1020,"Shuffle Write Time":4183618,"Shuffle Records Written":15},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":70,"Index":3,"Attempt":0,"Launch Time":1646432565510,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432581700,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"6429338","Value":"35849298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1596","Value":"8511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1102913236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"195074315","Value":"1102913236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7104","Value":"82536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11482","Value":"130264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"296","Value":"3439","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"12483511","Value":"156269097","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"16109","Value":"157786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"296","Value":"3439","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"741081088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"16079","Value":"157558","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"11035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"16133","Value":"84580","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":69,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5212724,"Value":59643236,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":16154,"Value":158359,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":15797163901,"Value":154431521201,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":28191,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":138,"Value":1953,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":741081088,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11482,"Value":130264,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":3439,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":12483511,"Value":156269097,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":2015068728,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":65209805,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1596676465,"Value":15437590955,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":2015068728,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":124878648,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1457840,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":483,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5212724,"Executor Run Time":16154,"Executor CPU Time":15797163901,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":138,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11482,"Shuffle Write Time":12483511,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":67,"Index":0,"Attempt":0,"Launch Time":1646432565510,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432581802,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"7213446","Value":"43062744","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1593","Value":"10104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1331519856","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"228606620","Value":"1331519856","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"8592","Value":"91128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"12849","Value":"143113","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"358","Value":"3797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"14097619","Value":"170366716","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"16206","Value":"173992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"358","Value":"3797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"808452096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"16189","Value":"173747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"12035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"16231","Value":"100811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":74,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4924575,"Value":64567811,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":16255,"Value":174614,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":15772631451,"Value":170204152652,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":30754,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":348,"Value":2301,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":808452096,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12849,"Value":143113,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":3797,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":14097619,"Value":170366716,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":2243675348,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":72423251,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1593122580,"Value":17030713535,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":2243675348,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":129174864,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1445149,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":917,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4924575,"Executor Run Time":16255,"Executor CPU Time":15772631451,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":348,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12849,"Shuffle Write Time":14097619,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":82,"Index":15,"Attempt":0,"Launch Time":1646432565513,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432582130,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"6096762","Value":"35457269","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1456","Value":"8377","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1096095000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"183939508","Value":"1096095000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7512","Value":"98640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11789","Value":"154902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"313","Value":"4110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"13182965","Value":"183549681","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"16540","Value":"190532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"313","Value":"4110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"875823104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"16498","Value":"190245","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"13035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"16533","Value":"89951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":80,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5358124,"Value":69925935,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":16583,"Value":191197,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":16311370839,"Value":186515523491,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":33317,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":147,"Value":2448,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":875823104,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11789,"Value":154902,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":4110,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":13182965,"Value":183549681,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":2427614856,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":78520013,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1456811811,"Value":18487525346,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":2427614856,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":124878648,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1457840,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":483,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":5358124,"Executor Run Time":16583,"Executor CPU Time":16311370839,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":147,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11789,"Shuffle Write Time":13182965,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":69,"Index":2,"Attempt":0,"Launch Time":1646432565510,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432582238,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"6450117","Value":"49512861","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1721","Value":"11825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1528334156","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"196814300","Value":"1528334156","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7320","Value":"105960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11668","Value":"166570","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"305","Value":"4415","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"12934372","Value":"196484053","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"16620","Value":"207152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"305","Value":"4415","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"943194112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"16601","Value":"206846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"14035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"16665","Value":"117476","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":86,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5913716,"Value":75839651,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":16689,"Value":207886,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":16360327384,"Value":202875850875,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2562,"Value":35879,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":182,"Value":2630,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":943194112,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":166570,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":4415,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":12934372,"Value":196484053,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":2624429156,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":84970130,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1721589343,"Value":20209114689,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":2624429156,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":5772991312,"JVMOffHeapMemory":127072720,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1398747,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":56,"MinorGCTime":553,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":5913716,"Executor Run Time":16689,"Executor CPU Time":16360327384,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":182,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":12934372,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":88,"Index":1,"Attempt":0,"Launch Time":1646432565597,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432582432,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"6369482","Value":"36007672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1507","Value":"8721","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1111496669","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"197638534","Value":"1111496669","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"384","Value":"2112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"1088","Value":"5924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"16","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"3690281","Value":"26876535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"16759","Value":"89660","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"16","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"16748","Value":"89593","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3243439,"Value":21290606,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":16790,"Value":89863,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":16493620975,"Value":88220332659,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2133,"Value":12796,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":147,"Value":888,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1088,"Value":5924,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":88,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":3690281,"Value":26876535,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1111496669,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":36007672,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1507332683,"Value":8724250222,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1111496669,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":124878648,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1457840,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":483,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3243439,"Executor Run Time":16790,"Executor CPU Time":16493620975,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":147,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1088,"Shuffle Write Time":3690281,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":78,"Index":11,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432582467,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"6369482","Value":"41826751","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1578","Value":"9955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1293733534","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"197638534","Value":"1293733534","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7632","Value":"113592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11947","Value":"178517","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"318","Value":"4733","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"11033937","Value":"207517990","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"16878","Value":"224030","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"318","Value":"4733","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"1010565120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"16853","Value":"223699","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"15035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"16887","Value":"106838","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":90,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4128558,"Value":79968209,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":16922,"Value":224808,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":16650489400,"Value":219526340275,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":38442,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":147,"Value":2777,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1010565120,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11947,"Value":178517,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":318,"Value":4733,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":11033937,"Value":207517990,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":2822067690,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":91339612,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1578792553,"Value":21787907242,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":2822067690,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":124878648,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1457840,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":55,"MinorGCTime":483,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4128558,"Executor Run Time":16922,"Executor CPU Time":16650489400,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":147,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11947,"Shuffle Write Time":11033937,"Shuffle Records Written":318},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":91,"Index":4,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432582643,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"6066650","Value":"42074322","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1575","Value":"10296","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1296532549","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"185035880","Value":"1296532549","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"384","Value":"2496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"1028","Value":"6952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"16","Value":"104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"4478101","Value":"31354636","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"16981","Value":"106641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"16","Value":"104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"16964","Value":"106557","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3639490,"Value":24930096,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":17013,"Value":106876,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":16698498056,"Value":104918830715,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2133,"Value":14929,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":1033,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1028,"Value":6952,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":104,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":4478101,"Value":31354636,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1296532549,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":42074322,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1575160039,"Value":10299410261,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1296532549,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4655239904,"JVMOffHeapMemory":127611656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460414,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":522,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3639490,"Executor Run Time":17013,"Executor CPU Time":16698498056,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":145,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1028,"Shuffle Write Time":4478101,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":68,"Index":1,"Attempt":0,"Launch Time":1646432565510,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432582953,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"6369482","Value":"55882343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1649","Value":"13474","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1725972690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"197638534","Value":"1725972690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7680","Value":"121272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11962","Value":"190479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"320","Value":"5053","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"11640271","Value":"219158261","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"17358","Value":"241388","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"320","Value":"5053","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"1077936128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"17341","Value":"241040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"16035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"17384","Value":"134860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":99,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6437273,"Value":86405482,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":17404,"Value":242212,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":17141554109,"Value":236667894384,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":41005,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":2922,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1077936128,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11962,"Value":190479,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":5053,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":11640271,"Value":219158261,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":3019706224,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":97709094,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1649433591,"Value":23437340833,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":3019706224,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4655239904,"JVMOffHeapMemory":127611656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460414,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":522,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":6437273,"Executor Run Time":17404,"Executor CPU Time":17141554109,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":145,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11962,"Shuffle Write Time":11640271,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":93,"Index":6,"Attempt":0,"Launch Time":1646432565598,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432583044,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"5819079","Value":"47893401","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1579","Value":"11875","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"182236865","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"336","Value":"2832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"952","Value":"7904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"14","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"5770046","Value":"37124682","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"17280","Value":"123921","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"14","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"17269","Value":"123826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3170617,"Value":28100713,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":17315,"Value":124191,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":16927539041,"Value":121846369756,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2133,"Value":17062,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":266,"Value":1299,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":7904,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":118,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":5770046,"Value":37124682,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":1478769414,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":47893401,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1579368405,"Value":11878778666,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":1478769414,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":129174864,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1445149,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":917,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3170617,"Executor Run Time":17315,"Executor CPU Time":16927539041,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":266,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":5770046,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":75,"Index":8,"Attempt":0,"Launch Time":1646432565511,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432583078,"Failed":false,"Killed":false,"Accumulables":[{"ID":782,"Name":"number of output rows","Update":"5674621","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Update":"1481","Value":"14955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Update":"178217644","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7680","Value":"128952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"12024","Value":"202503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"320","Value":"5373","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"18092157","Value":"237250418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"17478","Value":"258866","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"320","Value":"5373","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"1145307136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"17457","Value":"258497","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"17035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Update":"17499","Value":"152359","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":107,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5417650,"Value":91823132,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":17529,"Value":259741,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":17064247968,"Value":253732142352,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":43568,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":353,"Value":3275,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1145307136,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12024,"Value":202503,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":5373,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":18092157,"Value":237250418,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":3197923868,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":103383715,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1481589238,"Value":24918930071,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":3197923868,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":129174864,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1445149,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":917,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":5417650,"Executor Run Time":17529,"Executor CPU Time":17064247968,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":353,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12024,"Shuffle Write Time":18092157,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":89,"Index":2,"Attempt":0,"Launch Time":1646432565597,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432583595,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"6450117","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1852","Value":"13727","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"432","Value":"3264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"1164","Value":"9068","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"18","Value":"136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"3897453","Value":"41022135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"17833","Value":"141754","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"18","Value":"136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"17823","Value":"141649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3414922,"Value":31515635,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":17866,"Value":142057,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":17474883096,"Value":139321252852,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2133,"Value":19195,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":266,"Value":1565,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1164,"Value":9068,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":136,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":3897453,"Value":41022135,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1675583714,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":54343518,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1852441847,"Value":13731220513,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1675583714,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":129174864,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1445149,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":917,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3414922,"Executor Run Time":17866,"Executor CPU Time":17474883096,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":266,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1164,"Shuffle Write Time":3897453,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":81,"Index":14,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432583778,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"6066650","Value":"47893401","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1768","Value":"11723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"185035880","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7224","Value":"136176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11463","Value":"213966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"301","Value":"5674","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"12257143","Value":"249507561","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"18162","Value":"277028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"301","Value":"5674","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"1212678144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"18143","Value":"276640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"18035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"18179","Value":"125017","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":115,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6447479,"Value":98270611,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":18227,"Value":277968,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":17808088182,"Value":271540230534,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2562,"Value":46130,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":182,"Value":3457,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1212678144,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11463,"Value":213966,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":301,"Value":5674,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":12257143,"Value":249507561,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":3382959748,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":109450365,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1768391859,"Value":26687321930,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":3382959748,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":5772991312,"JVMOffHeapMemory":127072720,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1398747,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":56,"MinorGCTime":553,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":6447479,"Executor Run Time":18227,"Executor CPU Time":17808088182,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":182,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11463,"Shuffle Write Time":12257143,"Shuffle Records Written":301},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":79,"Index":12,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432583822,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"6450117","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1570","Value":"13293","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"7320","Value":"143496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"11668","Value":"225634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"305","Value":"5979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"11824417","Value":"261331978","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"18229","Value":"295257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"305","Value":"5979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"1280049152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"18210","Value":"294850","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"19035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"18236","Value":"143253","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":121,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4900272,"Value":103170883,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":18275,"Value":296243,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":17801332401,"Value":289341562935,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2563,"Value":48693,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":353,"Value":3810,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1280049152,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":225634,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":5979,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":11824417,"Value":261331978,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":3579774048,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":115900482,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1570282137,"Value":28257604067,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":3579774048,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":129174864,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1445149,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":917,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":4900272,"Executor Run Time":18275,"Executor CPU Time":17801332401,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":353,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":11824417,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":20,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":87,"Index":0,"Attempt":0,"Launch Time":1646432565597,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432584044,"Failed":false,"Killed":false,"Accumulables":[{"ID":932,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Update":"1817","Value":"15544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Update":"432","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Update":"1224","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Update":"18","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Update":"3876166","Value":"44898301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Update":"18382","Value":"160136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Update":"18","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Update":"18372","Value":"160021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3296959,"Value":34812594,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Update":18411,"Value":160468,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Update":18082267487,"Value":157403520339,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Update":2133,"Value":21328,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":1710,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1224,"Value":10292,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Update":3876166,"Value":44898301,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1817234545,"Value":15548455058,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4655239904,"JVMOffHeapMemory":127611656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460414,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":522,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":3296959,"Executor Run Time":18411,"Executor CPU Time":18082267487,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":145,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1224,"Shuffle Write Time":3876166,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":20,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1142\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1141\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432565571,"Completion Time":1646432584044,"Accumulables":[{"ID":932,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":936,"Name":"filesystem read time (sampled)","Value":"15544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":937,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":938,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":973,"Name":"data size","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":981,"Name":"shuffle bytes written","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":982,"Name":"shuffle records written","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"shuffle write time","Value":"44898301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"duration","Value":"160136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"number of output rows","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":986,"Name":"peak memory","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":988,"Name":"time in aggregation build","Value":"160021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":989,"Name":"avg hash probe bucket list iters","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1051,"Name":"internal.metrics.executorDeserializeTime","Value":40,"Internal":true,"Count Failed Values":true},{"ID":1052,"Name":"internal.metrics.executorDeserializeCpuTime","Value":34812594,"Internal":true,"Count Failed Values":true},{"ID":1053,"Name":"internal.metrics.executorRunTime","Value":160468,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.executorCpuTime","Value":157403520339,"Internal":true,"Count Failed Values":true},{"ID":1055,"Name":"internal.metrics.resultSize","Value":21328,"Internal":true,"Count Failed Values":true},{"ID":1056,"Name":"internal.metrics.jvmGCTime","Value":1710,"Internal":true,"Count Failed Values":true},{"ID":1060,"Name":"internal.metrics.peakExecutionMemory","Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":1071,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":10292,"Internal":true,"Count Failed Values":true},{"ID":1072,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":154,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.writeTime","Value":44898301,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1076,"Name":"internal.metrics.input.sampledTimeReadNano","Value":15548455058,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":13,"Completion Time":1646432584045,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":9,"physicalPlanDescription":"== Physical Plan ==\nCollectLimit (99)\n+- AdaptiveSparkPlan (98)\n +- == Current Plan ==\n Project (49)\n +- BroadcastHashJoin Inner BuildRight (48)\n :- Project (35)\n : +- SortMergeJoin Inner (34)\n : :- Sort (24)\n : : +- Exchange (23)\n : : +- Project (22)\n : : +- SortMergeJoin Inner (21)\n : : :- Sort (5)\n : : : +- ShuffleQueryStage (4)\n : : : +- Exchange (3)\n : : : +- * Filter (2)\n : : : +- Scan csv (1)\n : : +- Sort (20)\n : : +- Exchange (19)\n : : +- Filter (18)\n : : +- RunningWindowFunction (17)\n : : +- Sort (16)\n : : +- Exchange (15)\n : : +- HashAggregate (14)\n : : +- ShuffleQueryStage (13)\n : : +- Exchange (12)\n : : +- * HashAggregate (11)\n : : +- Union (10)\n : : :- * Project (7)\n : : : +- Scan csv (6)\n : : +- * Project (9)\n : : +- Scan csv (8)\n : +- Sort (33)\n : +- Exchange (32)\n : +- Filter (31)\n : +- RunningWindowFunction (30)\n : +- Sort (29)\n : +- Exchange (28)\n : +- HashAggregate (27)\n : +- ShuffleQueryStage (26)\n : +- ReusedExchange (25)\n +- BroadcastExchange (47)\n +- Filter (46)\n +- RunningWindowFunction (45)\n +- Sort (44)\n +- ShuffleQueryStage (43)\n +- Exchange (42)\n +- * HashAggregate (41)\n +- CustomShuffleReader (40)\n +- ShuffleQueryStage (39)\n +- Exchange (38)\n +- * HashAggregate (37)\n +- Scan csv (36)\n +- == Initial Plan ==\n Project (97)\n +- SortMergeJoin Inner (96)\n :- Sort (86)\n : +- Exchange (85)\n : +- Project (84)\n : +- SortMergeJoin Inner (83)\n : :- Sort (68)\n : : +- Exchange (67)\n : : +- Project (66)\n : : +- SortMergeJoin Inner (65)\n : : :- Sort (52)\n : : : +- Exchange (51)\n : : : +- Filter (50)\n : : : +- Scan csv (1)\n : : +- Sort (64)\n : : +- Exchange (63)\n : : +- Filter (62)\n : : +- RunningWindowFunction (61)\n : : +- Sort (60)\n : : +- Exchange (59)\n : : +- HashAggregate (58)\n : : +- Exchange (57)\n : : +- HashAggregate (56)\n : : +- Union (55)\n : : :- Project (53)\n : : : +- Scan csv (6)\n : : +- Project (54)\n : : +- Scan csv (8)\n : +- Sort (82)\n : +- Exchange (81)\n : +- Filter (80)\n : +- RunningWindowFunction (79)\n : +- Sort (78)\n : +- Exchange (77)\n : +- HashAggregate (76)\n : +- Exchange (75)\n : +- HashAggregate (74)\n : +- Union (73)\n : :- Project (70)\n : : +- Scan csv (69)\n : +- Project (72)\n : +- Scan csv (71)\n +- Sort (95)\n +- Exchange (94)\n +- Filter (93)\n +- RunningWindowFunction (92)\n +- Sort (91)\n +- Exchange (90)\n +- HashAggregate (89)\n +- Exchange (88)\n +- HashAggregate (87)\n +- Scan csv (36)\n\n\n(1) Scan csv \nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nPushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]\nReadSchema: struct\n\n(2) Filter [codegen id : 1]\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(3) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]\n\n(4) ShuffleQueryStage\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: 0, Statistics(sizeInBytes=1816.0 MiB)\n\n(5) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(6) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(7) Project [codegen id : 2]\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(8) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(9) Project [codegen id : 3]\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(10) Union\n\n(11) HashAggregate [codegen id : 4]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(12) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]\n\n(13) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 1, Statistics(sizeInBytes=3.5 GiB)\n\n(14) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(15) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1259]\n\n(16) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(17) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(18) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(19) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1265]\n\n(20) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(21) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(22) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(23) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#1276]\n\n(24) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(25) ReusedExchange [Reuses operator id: 12]\nOutput [1]: [airport#178]\n\n(26) ShuffleQueryStage\nOutput [1]: [airport#178]\nArguments: 3, Statistics(sizeInBytes=3.5 GiB)\n\n(27) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(28) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1270]\n\n(29) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(30) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(31) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(32) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#1277]\n\n(33) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(34) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(35) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(36) Scan csv \nOutput [1]: [OP_CARRIER#180]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(37) HashAggregate [codegen id : 8]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(38) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]\n\n(39) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)\n\n(40) CustomShuffleReader\nInput [1]: [OP_CARRIER#180]\nArguments: coalesced\n\n(41) HashAggregate [codegen id : 9]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(42) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]\n\n(43) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 5, Statistics(sizeInBytes=4.2 KiB, rowCount=154, isRuntime=true)\n\n(44) Sort\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(45) RunningWindowFunction\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(46) Filter\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(47) BroadcastExchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1469]\n\n(48) BroadcastHashJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(49) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(50) Filter\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(51) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#665]\n\n(52) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(53) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(54) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(55) Union\n\n(56) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(57) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#656]\n\n(58) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(59) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#659]\n\n(60) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(61) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(62) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(63) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#666]\n\n(64) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(65) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(66) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(67) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#680]\n\n(68) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(69) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(70) Project\nOutput [1]: [ORIGIN#21 AS airport#178]\nInput [1]: [ORIGIN#21]\n\n(71) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(72) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(73) Union\n\n(74) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(75) Exchange\nInput [1]: [airport#178]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#671]\n\n(76) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(77) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#674]\n\n(78) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(79) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(80) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(81) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#681]\n\n(82) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(83) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(84) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(85) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#695]\n\n(86) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(87) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(88) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#686]\n\n(89) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(90) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#689]\n\n(91) Sort\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(92) RunningWindowFunction\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(93) Filter\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(94) Exchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#696]\n\n(95) Sort\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(96) SortMergeJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(97) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(98) AdaptiveSparkPlan\nOutput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: isFinalPlan=false\n\n(99) CollectLimit\nInput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: 11\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 11","children":[{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [OP_CARRIER#19], [OP_CARRIER#180], Inner, BuildRight, AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25), false","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [DEST#22], [airport#178], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [DEST#22 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#1276]","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [ORIGIN#21], [airport#144], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [ORIGIN#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=1816.0 MiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [FL_DATE#18,OP_CARRIER#19,ORIGIN#21,DEST#22,CRS_DEP_TIME#23,DEP_DELAY#25] Batched: false, DataFilters: [isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)], ReadSchema: struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]","DataFilters":"[isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)]"},"metrics":[{"name":"file sorting by size time","accumulatorId":722,"metricType":"timing"},{"name":"number of files read","accumulatorId":716,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":721,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":720,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":719,"metricType":"timing"},{"name":"metadata time","accumulatorId":717,"metricType":"timing"},{"name":"size of files read","accumulatorId":718,"metricType":"size"},{"name":"number of output rows","accumulatorId":715,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":952,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":951,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":949,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":950,"metricType":"nsTiming"},{"name":"records read","accumulatorId":947,"metricType":"sum"},{"name":"local bytes read","accumulatorId":945,"metricType":"size"},{"name":"fetch wait time","accumulatorId":946,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":943,"metricType":"size"},{"name":"local blocks read","accumulatorId":942,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":941,"metricType":"sum"},{"name":"data size","accumulatorId":940,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":944,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":948,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1265]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#144)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1259]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=3.5 GiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [airport#178 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#1277]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#178)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#178 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1270]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#178], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3, Statistics(sizeInBytes=3.5 GiB)","children":[{"nodeName":"ReusedExchange","simpleString":"ReusedExchange [airport#178], Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1469]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(OP_CARRIER#180)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#180 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5, Statistics(sizeInBytes=4.2 KiB, rowCount=154, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#180] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":939,"metricType":"timing"},{"name":"number of files read","accumulatorId":933,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":938,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":937,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":936,"metricType":"timing"},{"name":"metadata time","accumulatorId":934,"metricType":"timing"},{"name":"size of files read","accumulatorId":935,"metricType":"size"},{"name":"number of output rows","accumulatorId":932,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":987,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":988,"metricType":"timing"},{"name":"peak memory","accumulatorId":986,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":990,"metricType":"sum"},{"name":"number of output rows","accumulatorId":985,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":989,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":984,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":982,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":983,"metricType":"nsTiming"},{"name":"records read","accumulatorId":980,"metricType":"sum"},{"name":"local bytes read","accumulatorId":978,"metricType":"size"},{"name":"fetch wait time","accumulatorId":979,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":976,"metricType":"size"},{"name":"local blocks read","accumulatorId":975,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":974,"metricType":"sum"},{"name":"data size","accumulatorId":973,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":977,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":981,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1098,"metricType":"sum"},{"name":"partition data size","accumulatorId":1099,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1095,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1096,"metricType":"timing"},{"name":"peak memory","accumulatorId":1094,"metricType":"size"},{"name":"number of output rows","accumulatorId":1093,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1097,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1092,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1090,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1091,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1088,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1086,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1087,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1084,"metricType":"size"},{"name":"local blocks read","accumulatorId":1083,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1082,"metricType":"sum"},{"name":"data size","accumulatorId":1081,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1085,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1089,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":655,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":656,"metricType":"nsTiming"},{"name":"records read","accumulatorId":653,"metricType":"sum"},{"name":"local bytes read","accumulatorId":651,"metricType":"size"},{"name":"fetch wait time","accumulatorId":652,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":649,"metricType":"size"},{"name":"local blocks read","accumulatorId":648,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":647,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":650,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":654,"metricType":"size"}]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":9,"accumUpdates":[[1098,1],[1099,10896]]} +{"Event":"SparkListenerJobStart","Job ID":14,"Submission Time":1646432584166,"Stage Infos":[{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1142\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1141\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":22,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1458\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1457\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1452\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[21],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[21,22],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1458\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":22,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1458\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1457\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1452\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[21],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432584170,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1458\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":22,"Stage Attempt ID":0,"Task Info":{"Task ID":97,"Index":0,"Attempt":0,"Launch Time":1646432584188,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":22,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":97,"Index":0,"Attempt":0,"Launch Time":1646432584188,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432584333,"Failed":false,"Killed":false,"Accumulables":[{"ID":974,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":975,"Name":"local blocks read","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":976,"Name":"remote bytes read","Update":"7368","Value":"7368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":978,"Name":"local bytes read","Update":"2924","Value":"2924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":979,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":980,"Name":"records read","Update":"154","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1081,"Name":"data size","Update":"552","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1089,"Name":"shuffle bytes written","Update":"259","Value":"259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1090,"Name":"shuffle records written","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1091,"Name":"shuffle write time","Update":"262091","Value":"262091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1092,"Name":"duration","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1093,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1094,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1096,"Name":"time in aggregation build","Update":"12","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1097,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1100,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":1101,"Name":"internal.metrics.executorDeserializeCpuTime","Update":23095536,"Value":23095536,"Internal":true,"Count Failed Values":true},{"ID":1102,"Name":"internal.metrics.executorRunTime","Update":97,"Value":97,"Internal":true,"Count Failed Values":true},{"ID":1103,"Name":"internal.metrics.executorCpuTime","Update":86604772,"Value":86604772,"Internal":true,"Count Failed Values":true},{"ID":1104,"Name":"internal.metrics.resultSize","Update":3469,"Value":3469,"Internal":true,"Count Failed Values":true},{"ID":1109,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1113,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":1114,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":1115,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":7368,"Value":7368,"Internal":true,"Count Failed Values":true},{"ID":1116,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1117,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2924,"Value":2924,"Internal":true,"Count Failed Values":true},{"ID":1118,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1119,"Name":"internal.metrics.shuffle.read.recordsRead","Update":154,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":1120,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":259,"Value":259,"Internal":true,"Count Failed Values":true},{"ID":1121,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":1122,"Name":"internal.metrics.shuffle.write.writeTime","Update":262091,"Value":262091,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":23095536,"Executor Run Time":97,"Executor CPU Time":86604772,"Peak Execution Memory":67371008,"Result Size":3469,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":3,"Fetch Wait Time":0,"Remote Bytes Read":7368,"Remote Bytes Read To Disk":0,"Local Bytes Read":2924,"Total Records Read":154},"Shuffle Write Metrics":{"Shuffle Bytes Written":259,"Shuffle Write Time":262091,"Shuffle Records Written":23},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":22,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1458\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1457\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1452\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[21],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432584170,"Completion Time":1646432584333,"Accumulables":[{"ID":974,"Name":"remote blocks read","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":975,"Name":"local blocks read","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":976,"Name":"remote bytes read","Value":"7368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":978,"Name":"local bytes read","Value":"2924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":979,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":980,"Name":"records read","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1081,"Name":"data size","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1089,"Name":"shuffle bytes written","Value":"259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1090,"Name":"shuffle records written","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1091,"Name":"shuffle write time","Value":"262091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1092,"Name":"duration","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1093,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1094,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1096,"Name":"time in aggregation build","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1097,"Name":"avg hash probe bucket list iters","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1100,"Name":"internal.metrics.executorDeserializeTime","Value":23,"Internal":true,"Count Failed Values":true},{"ID":1101,"Name":"internal.metrics.executorDeserializeCpuTime","Value":23095536,"Internal":true,"Count Failed Values":true},{"ID":1102,"Name":"internal.metrics.executorRunTime","Value":97,"Internal":true,"Count Failed Values":true},{"ID":1103,"Name":"internal.metrics.executorCpuTime","Value":86604772,"Internal":true,"Count Failed Values":true},{"ID":1104,"Name":"internal.metrics.resultSize","Value":3469,"Internal":true,"Count Failed Values":true},{"ID":1109,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1113,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":7,"Internal":true,"Count Failed Values":true},{"ID":1114,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":3,"Internal":true,"Count Failed Values":true},{"ID":1115,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":7368,"Internal":true,"Count Failed Values":true},{"ID":1116,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1117,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2924,"Internal":true,"Count Failed Values":true},{"ID":1118,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1119,"Name":"internal.metrics.shuffle.read.recordsRead","Value":154,"Internal":true,"Count Failed Values":true},{"ID":1120,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":259,"Internal":true,"Count Failed Values":true},{"ID":1121,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23,"Internal":true,"Count Failed Values":true},{"ID":1122,"Name":"internal.metrics.shuffle.write.writeTime","Value":262091,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":14,"Completion Time":1646432584334,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":9,"physicalPlanDescription":"== Physical Plan ==\nCollectLimit (100)\n+- AdaptiveSparkPlan (99)\n +- == Current Plan ==\n Project (50)\n +- BroadcastHashJoin Inner BuildRight (49)\n :- Project (35)\n : +- SortMergeJoin Inner (34)\n : :- Sort (24)\n : : +- Exchange (23)\n : : +- Project (22)\n : : +- SortMergeJoin Inner (21)\n : : :- Sort (5)\n : : : +- ShuffleQueryStage (4)\n : : : +- Exchange (3)\n : : : +- * Filter (2)\n : : : +- Scan csv (1)\n : : +- Sort (20)\n : : +- Exchange (19)\n : : +- Filter (18)\n : : +- RunningWindowFunction (17)\n : : +- Sort (16)\n : : +- Exchange (15)\n : : +- HashAggregate (14)\n : : +- ShuffleQueryStage (13)\n : : +- Exchange (12)\n : : +- * HashAggregate (11)\n : : +- Union (10)\n : : :- * Project (7)\n : : : +- Scan csv (6)\n : : +- * Project (9)\n : : +- Scan csv (8)\n : +- Sort (33)\n : +- Exchange (32)\n : +- Filter (31)\n : +- RunningWindowFunction (30)\n : +- Sort (29)\n : +- Exchange (28)\n : +- HashAggregate (27)\n : +- ShuffleQueryStage (26)\n : +- ReusedExchange (25)\n +- BroadcastQueryStage (48)\n +- BroadcastExchange (47)\n +- * Filter (46)\n +- * RunningWindowFunction (45)\n +- * Sort (44)\n +- ShuffleQueryStage (43)\n +- Exchange (42)\n +- * HashAggregate (41)\n +- CustomShuffleReader (40)\n +- ShuffleQueryStage (39)\n +- Exchange (38)\n +- * HashAggregate (37)\n +- Scan csv (36)\n +- == Initial Plan ==\n Project (98)\n +- SortMergeJoin Inner (97)\n :- Sort (87)\n : +- Exchange (86)\n : +- Project (85)\n : +- SortMergeJoin Inner (84)\n : :- Sort (69)\n : : +- Exchange (68)\n : : +- Project (67)\n : : +- SortMergeJoin Inner (66)\n : : :- Sort (53)\n : : : +- Exchange (52)\n : : : +- Filter (51)\n : : : +- Scan csv (1)\n : : +- Sort (65)\n : : +- Exchange (64)\n : : +- Filter (63)\n : : +- RunningWindowFunction (62)\n : : +- Sort (61)\n : : +- Exchange (60)\n : : +- HashAggregate (59)\n : : +- Exchange (58)\n : : +- HashAggregate (57)\n : : +- Union (56)\n : : :- Project (54)\n : : : +- Scan csv (6)\n : : +- Project (55)\n : : +- Scan csv (8)\n : +- Sort (83)\n : +- Exchange (82)\n : +- Filter (81)\n : +- RunningWindowFunction (80)\n : +- Sort (79)\n : +- Exchange (78)\n : +- HashAggregate (77)\n : +- Exchange (76)\n : +- HashAggregate (75)\n : +- Union (74)\n : :- Project (71)\n : : +- Scan csv (70)\n : +- Project (73)\n : +- Scan csv (72)\n +- Sort (96)\n +- Exchange (95)\n +- Filter (94)\n +- RunningWindowFunction (93)\n +- Sort (92)\n +- Exchange (91)\n +- HashAggregate (90)\n +- Exchange (89)\n +- HashAggregate (88)\n +- Scan csv (36)\n\n\n(1) Scan csv \nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nPushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]\nReadSchema: struct\n\n(2) Filter [codegen id : 1]\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(3) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]\n\n(4) ShuffleQueryStage\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: 0, Statistics(sizeInBytes=1816.0 MiB)\n\n(5) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(6) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(7) Project [codegen id : 2]\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(8) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(9) Project [codegen id : 3]\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(10) Union\n\n(11) HashAggregate [codegen id : 4]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(12) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]\n\n(13) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 1, Statistics(sizeInBytes=3.5 GiB)\n\n(14) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(15) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1259]\n\n(16) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(17) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(18) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(19) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1265]\n\n(20) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(21) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(22) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(23) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#1276]\n\n(24) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(25) ReusedExchange [Reuses operator id: 12]\nOutput [1]: [airport#178]\n\n(26) ShuffleQueryStage\nOutput [1]: [airport#178]\nArguments: 3, Statistics(sizeInBytes=3.5 GiB)\n\n(27) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(28) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1270]\n\n(29) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(30) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(31) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(32) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#1277]\n\n(33) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(34) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(35) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(36) Scan csv \nOutput [1]: [OP_CARRIER#180]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(37) HashAggregate [codegen id : 8]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(38) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]\n\n(39) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)\n\n(40) CustomShuffleReader\nInput [1]: [OP_CARRIER#180]\nArguments: coalesced\n\n(41) HashAggregate [codegen id : 9]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(42) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]\n\n(43) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 5, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)\n\n(44) Sort [codegen id : 10]\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(45) RunningWindowFunction [codegen id : 10]\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(46) Filter [codegen id : 10]\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(47) BroadcastExchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1676]\n\n(48) BroadcastQueryStage\nOutput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: 6, Statistics(sizeInBytes=630.0 B, isRuntime=true)\n\n(49) BroadcastHashJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(50) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(51) Filter\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(52) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#665]\n\n(53) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(54) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(55) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(56) Union\n\n(57) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(58) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#656]\n\n(59) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(60) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#659]\n\n(61) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(62) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(63) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(64) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#666]\n\n(65) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(66) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(67) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(68) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#680]\n\n(69) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(70) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(71) Project\nOutput [1]: [ORIGIN#21 AS airport#178]\nInput [1]: [ORIGIN#21]\n\n(72) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(73) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(74) Union\n\n(75) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(76) Exchange\nInput [1]: [airport#178]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#671]\n\n(77) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(78) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#674]\n\n(79) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(80) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(81) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(82) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#681]\n\n(83) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(84) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(85) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(86) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#695]\n\n(87) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(88) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(89) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#686]\n\n(90) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(91) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#689]\n\n(92) Sort\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(93) RunningWindowFunction\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(94) Filter\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(95) Exchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#696]\n\n(96) Sort\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(97) SortMergeJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(98) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(99) AdaptiveSparkPlan\nOutput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: isFinalPlan=false\n\n(100) CollectLimit\nInput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: 11\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 11","children":[{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [OP_CARRIER#19], [OP_CARRIER#180], Inner, BuildRight, AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25), false","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [DEST#22], [airport#178], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [DEST#22 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#1276]","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [ORIGIN#21], [airport#144], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [ORIGIN#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=1816.0 MiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [FL_DATE#18,OP_CARRIER#19,ORIGIN#21,DEST#22,CRS_DEP_TIME#23,DEP_DELAY#25] Batched: false, DataFilters: [isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)], ReadSchema: struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]","DataFilters":"[isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)]"},"metrics":[{"name":"file sorting by size time","accumulatorId":722,"metricType":"timing"},{"name":"number of files read","accumulatorId":716,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":721,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":720,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":719,"metricType":"timing"},{"name":"metadata time","accumulatorId":717,"metricType":"timing"},{"name":"size of files read","accumulatorId":718,"metricType":"size"},{"name":"number of output rows","accumulatorId":715,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":952,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":951,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":949,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":950,"metricType":"nsTiming"},{"name":"records read","accumulatorId":947,"metricType":"sum"},{"name":"local bytes read","accumulatorId":945,"metricType":"size"},{"name":"fetch wait time","accumulatorId":946,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":943,"metricType":"size"},{"name":"local blocks read","accumulatorId":942,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":941,"metricType":"sum"},{"name":"data size","accumulatorId":940,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":944,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":948,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1265]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#144)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1259]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=3.5 GiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [airport#178 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#1277]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#178)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#178 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1270]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#178], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3, Statistics(sizeInBytes=3.5 GiB)","children":[{"nodeName":"ReusedExchange","simpleString":"ReusedExchange [airport#178], Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6, Statistics(sizeInBytes=630.0 B, isRuntime=true)","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1676]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(OP_CARRIER#180)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#180 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#180] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":939,"metricType":"timing"},{"name":"number of files read","accumulatorId":933,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":938,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":937,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":936,"metricType":"timing"},{"name":"metadata time","accumulatorId":934,"metricType":"timing"},{"name":"size of files read","accumulatorId":935,"metricType":"size"},{"name":"number of output rows","accumulatorId":932,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":987,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":988,"metricType":"timing"},{"name":"peak memory","accumulatorId":986,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":990,"metricType":"sum"},{"name":"number of output rows","accumulatorId":985,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":989,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":984,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":982,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":983,"metricType":"nsTiming"},{"name":"records read","accumulatorId":980,"metricType":"sum"},{"name":"local bytes read","accumulatorId":978,"metricType":"size"},{"name":"fetch wait time","accumulatorId":979,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":976,"metricType":"size"},{"name":"local blocks read","accumulatorId":975,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":974,"metricType":"sum"},{"name":"data size","accumulatorId":973,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":977,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":981,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1098,"metricType":"sum"},{"name":"partition data size","accumulatorId":1099,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1095,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1096,"metricType":"timing"},{"name":"peak memory","accumulatorId":1094,"metricType":"size"},{"name":"number of output rows","accumulatorId":1093,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1097,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1092,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1090,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1091,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1088,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1086,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1087,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1084,"metricType":"size"},{"name":"local blocks read","accumulatorId":1083,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1082,"metricType":"sum"},{"name":"data size","accumulatorId":1081,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1085,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1089,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1139,"metricType":"size"},{"name":"cumulative time","accumulatorId":1140,"metricType":"timing"},{"name":"peak memory","accumulatorId":1138,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":1142,"metricType":"sum"},{"name":"sort time","accumulatorId":1137,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":1143,"metricType":"sum"},{"name":"spill write time","accumulatorId":1141,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1136,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1135,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1134,"metricType":"timing"},{"name":"time to build","accumulatorId":1133,"metricType":"timing"},{"name":"time to collect","accumulatorId":1132,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1131,"metricType":"sum"},{"name":"data size","accumulatorId":1130,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":655,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":656,"metricType":"nsTiming"},{"name":"records read","accumulatorId":653,"metricType":"sum"},{"name":"local bytes read","accumulatorId":651,"metricType":"size"},{"name":"fetch wait time","accumulatorId":652,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":649,"metricType":"size"},{"name":"local blocks read","accumulatorId":648,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":647,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":650,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":654,"metricType":"size"}]}} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":64,"Index":7,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432584403,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"5819811","Value":"5819811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1324","Value":"1324","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"558701856","Value":"558701856","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"78151025","Value":"78151025","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"5819811","Value":"5819811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"187630735","Value":"187630735","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"18897","Value":"18897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"5819811","Value":"5819811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4779964,"Value":4779964,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":18999,"Value":18999,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":18724844552,"Value":18724844552,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1703,"Value":1703,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":145,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":78151025,"Value":78151025,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819811,"Value":5819811,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":187630735,"Value":187630735,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":180693843,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":5819811,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1324104170,"Value":1324104170,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":180693843,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4655239904,"JVMOffHeapMemory":127611656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460414,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":522,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4779964,"Executor Run Time":18999,"Executor CPU Time":18724844552,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":145,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":78151025,"Shuffle Write Time":187630735,"Shuffle Records Written":5819811},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":19,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":77,"Index":10,"Attempt":0,"Launch Time":1646432565512,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432584410,"Failed":false,"Killed":false,"Accumulables":[{"ID":790,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Update":"1945","Value":"15238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Update":"8592","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Update":"12847","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Update":"358","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Update":"13374224","Value":"274706202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Update":"18788","Value":"314045","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Update":"358","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Update":"67371008","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Update":"18768","Value":"313618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Update":"18805","Value":"162058","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":129,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6105490,"Value":109276373,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Update":18856,"Value":315099,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Update":18404575890,"Value":307746138825,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Update":2562,"Value":51255,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Update":182,"Value":3992,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12847,"Value":238481,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Update":13374224,"Value":274706202,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1945934992,"Value":30203539059,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":3808380668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":8297822104,"JVMOffHeapMemory":127199904,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":599,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":6105490,"Executor Run Time":18856,"Executor CPU Time":18404575890,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":182,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12847,"Shuffle Write Time":13374224,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":19,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1066\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":54,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_1062\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[50,53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1065\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1061\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1058\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432565464,"Completion Time":1646432584411,"Accumulables":[{"ID":782,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"filesystem read time (sampled)","Value":"14955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":787,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":788,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":790,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":794,"Name":"filesystem read time (sampled)","Value":"15238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":795,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":796,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":953,"Name":"data size","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":961,"Name":"shuffle bytes written","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":962,"Name":"shuffle records written","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":963,"Name":"shuffle write time","Value":"274706202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":964,"Name":"duration","Value":"314045","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":965,"Name":"number of output rows","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":966,"Name":"peak memory","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":968,"Name":"time in aggregation build","Value":"313618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":969,"Name":"avg hash probe bucket list iters","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":971,"Name":"duration","Value":"152359","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":972,"Name":"duration","Value":"162058","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1021,"Name":"internal.metrics.executorDeserializeTime","Value":129,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorDeserializeCpuTime","Value":109276373,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorRunTime","Value":315099,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.executorCpuTime","Value":307746138825,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.resultSize","Value":51255,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.jvmGCTime","Value":3992,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.peakExecutionMemory","Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":238481,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.write.writeTime","Value":274706202,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.input.bytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.input.recordsRead","Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.input.sampledTimeReadNano","Value":30203539059,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.input.sampledBytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":12,"Completion Time":1646432584412,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":15,"Submission Time":1646432584461,"Stage Infos":[{"Stage ID":24,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1458\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1457\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1452\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[23],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1675\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[63],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[24],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1142\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1141\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_641\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[24,25,23],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_6\",\"name\":\"BroadcastQueryStage\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"broadcast exchange (runId fb098b05-da7e-4bc9-9864-594443088fd0)","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"fb098b05-da7e-4bc9-9864-594443088fd0","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1675\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[63],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[24],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432584462,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_6\",\"name\":\"BroadcastQueryStage\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"broadcast exchange (runId fb098b05-da7e-4bc9-9864-594443088fd0)","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"fb098b05-da7e-4bc9-9864-594443088fd0","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":25,"Stage Attempt ID":0,"Task Info":{"Task ID":98,"Index":0,"Attempt":0,"Launch Time":1646432584511,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":9,"physicalPlanDescription":"== Physical Plan ==\nCollectLimit (95)\n+- AdaptiveSparkPlan (94)\n +- == Current Plan ==\n Project (45)\n +- BroadcastHashJoin Inner BuildRight (44)\n :- Project (30)\n : +- BroadcastHashJoin Inner BuildRight (29)\n : :- Project (22)\n : : +- BroadcastHashJoin Inner BuildRight (21)\n : : :- ShuffleQueryStage (4)\n : : : +- Exchange (3)\n : : : +- * Filter (2)\n : : : +- Scan csv (1)\n : : +- BroadcastExchange (20)\n : : +- Filter (19)\n : : +- RunningWindowFunction (18)\n : : +- Sort (17)\n : : +- ShuffleQueryStage (16)\n : : +- Exchange (15)\n : : +- * HashAggregate (14)\n : : +- CustomShuffleReader (13)\n : : +- ShuffleQueryStage (12)\n : : +- Exchange (11)\n : : +- * HashAggregate (10)\n : : +- Union (9)\n : : :- * Project (6)\n : : : +- Scan csv (5)\n : : +- * Project (8)\n : : +- Scan csv (7)\n : +- BroadcastExchange (28)\n : +- Filter (27)\n : +- RunningWindowFunction (26)\n : +- Sort (25)\n : +- ShuffleQueryStage (24)\n : +- ReusedExchange (23)\n +- BroadcastQueryStage (43)\n +- BroadcastExchange (42)\n +- * Filter (41)\n +- * RunningWindowFunction (40)\n +- * Sort (39)\n +- ShuffleQueryStage (38)\n +- Exchange (37)\n +- * HashAggregate (36)\n +- CustomShuffleReader (35)\n +- ShuffleQueryStage (34)\n +- Exchange (33)\n +- * HashAggregate (32)\n +- Scan csv (31)\n +- == Initial Plan ==\n Project (93)\n +- SortMergeJoin Inner (92)\n :- Sort (82)\n : +- Exchange (81)\n : +- Project (80)\n : +- SortMergeJoin Inner (79)\n : :- Sort (64)\n : : +- Exchange (63)\n : : +- Project (62)\n : : +- SortMergeJoin Inner (61)\n : : :- Sort (48)\n : : : +- Exchange (47)\n : : : +- Filter (46)\n : : : +- Scan csv (1)\n : : +- Sort (60)\n : : +- Exchange (59)\n : : +- Filter (58)\n : : +- RunningWindowFunction (57)\n : : +- Sort (56)\n : : +- Exchange (55)\n : : +- HashAggregate (54)\n : : +- Exchange (53)\n : : +- HashAggregate (52)\n : : +- Union (51)\n : : :- Project (49)\n : : : +- Scan csv (5)\n : : +- Project (50)\n : : +- Scan csv (7)\n : +- Sort (78)\n : +- Exchange (77)\n : +- Filter (76)\n : +- RunningWindowFunction (75)\n : +- Sort (74)\n : +- Exchange (73)\n : +- HashAggregate (72)\n : +- Exchange (71)\n : +- HashAggregate (70)\n : +- Union (69)\n : :- Project (66)\n : : +- Scan csv (65)\n : +- Project (68)\n : +- Scan csv (67)\n +- Sort (91)\n +- Exchange (90)\n +- Filter (89)\n +- RunningWindowFunction (88)\n +- Sort (87)\n +- Exchange (86)\n +- HashAggregate (85)\n +- Exchange (84)\n +- HashAggregate (83)\n +- Scan csv (31)\n\n\n(1) Scan csv \nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nPushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]\nReadSchema: struct\n\n(2) Filter [codegen id : 1]\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(3) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]\n\n(4) ShuffleQueryStage\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: 0, Statistics(sizeInBytes=1816.0 MiB)\n\n(5) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(6) Project [codegen id : 2]\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(7) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(8) Project [codegen id : 3]\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(9) Union\n\n(10) HashAggregate [codegen id : 4]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(11) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]\n\n(12) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 1, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)\n\n(13) CustomShuffleReader\nInput [1]: [airport#144]\nArguments: coalesced\n\n(14) HashAggregate [codegen id : 11]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(15) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]\n\n(16) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 7, Statistics(sizeInBytes=173.3 KiB, rowCount=6.34E+3, isRuntime=true)\n\n(17) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(18) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(19) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(20) BroadcastExchange\nInput [2]: [airport#144, airport_id#149]\nArguments: HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1845]\n\n(21) BroadcastHashJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(22) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(23) ReusedExchange [Reuses operator id: 15]\nOutput [1]: [airport#178]\n\n(24) ShuffleQueryStage\nOutput [1]: [airport#178]\nArguments: 9, Statistics(sizeInBytes=173.3 KiB, rowCount=6.34E+3, isRuntime=true)\n\n(25) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(26) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(27) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(28) BroadcastExchange\nInput [2]: [airport#178, airport_id#177]\nArguments: HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1875]\n\n(29) BroadcastHashJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(30) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(31) Scan csv \nOutput [1]: [OP_CARRIER#180]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(32) HashAggregate [codegen id : 8]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(33) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]\n\n(34) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)\n\n(35) CustomShuffleReader\nInput [1]: [OP_CARRIER#180]\nArguments: coalesced\n\n(36) HashAggregate [codegen id : 9]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(37) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]\n\n(38) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 5, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)\n\n(39) Sort [codegen id : 10]\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(40) RunningWindowFunction [codegen id : 10]\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(41) Filter [codegen id : 10]\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(42) BroadcastExchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1676]\n\n(43) BroadcastQueryStage\nOutput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: 6, Statistics(sizeInBytes=630.0 B, isRuntime=true)\n\n(44) BroadcastHashJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(45) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(46) Filter\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(47) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#665]\n\n(48) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(49) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(50) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(51) Union\n\n(52) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(53) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#656]\n\n(54) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(55) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#659]\n\n(56) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(57) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(58) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(59) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#666]\n\n(60) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(61) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(62) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(63) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#680]\n\n(64) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(65) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(66) Project\nOutput [1]: [ORIGIN#21 AS airport#178]\nInput [1]: [ORIGIN#21]\n\n(67) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(68) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(69) Union\n\n(70) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(71) Exchange\nInput [1]: [airport#178]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#671]\n\n(72) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(73) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#674]\n\n(74) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(75) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(76) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(77) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#681]\n\n(78) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(79) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(80) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(81) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#695]\n\n(82) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(83) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(84) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#686]\n\n(85) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(86) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#689]\n\n(87) Sort\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(88) RunningWindowFunction\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(89) Filter\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(90) Exchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#696]\n\n(91) Sort\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(92) SortMergeJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(93) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(94) AdaptiveSparkPlan\nOutput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: isFinalPlan=false\n\n(95) CollectLimit\nInput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: 11\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 11","children":[{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [OP_CARRIER#19], [OP_CARRIER#180], Inner, BuildRight, AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25), false","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [DEST#22], [airport#178], Inner, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [ORIGIN#21], [airport#144], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=1816.0 MiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [FL_DATE#18,OP_CARRIER#19,ORIGIN#21,DEST#22,CRS_DEP_TIME#23,DEP_DELAY#25] Batched: false, DataFilters: [isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)], ReadSchema: struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]","DataFilters":"[isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)]"},"metrics":[{"name":"file sorting by size time","accumulatorId":722,"metricType":"timing"},{"name":"number of files read","accumulatorId":716,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":721,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":720,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":719,"metricType":"timing"},{"name":"metadata time","accumulatorId":717,"metricType":"timing"},{"name":"size of files read","accumulatorId":718,"metricType":"size"},{"name":"number of output rows","accumulatorId":715,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":952,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":951,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":949,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":950,"metricType":"nsTiming"},{"name":"records read","accumulatorId":947,"metricType":"sum"},{"name":"local bytes read","accumulatorId":945,"metricType":"size"},{"name":"fetch wait time","accumulatorId":946,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":943,"metricType":"size"},{"name":"local blocks read","accumulatorId":942,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":941,"metricType":"sum"},{"name":"data size","accumulatorId":940,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":944,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":948,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1845]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#144)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7, Statistics(sizeInBytes=173.3 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1191,"metricType":"sum"},{"name":"partition data size","accumulatorId":1192,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1188,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1189,"metricType":"timing"},{"name":"peak memory","accumulatorId":1187,"metricType":"size"},{"name":"number of output rows","accumulatorId":1186,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1190,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1185,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1183,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1184,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1181,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1179,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1180,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1177,"metricType":"size"},{"name":"local blocks read","accumulatorId":1176,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1175,"metricType":"sum"},{"name":"data size","accumulatorId":1174,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1178,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1182,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1875]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#178)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#178 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9, Statistics(sizeInBytes=173.3 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"ReusedExchange","simpleString":"ReusedExchange [airport#178], Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1191,"metricType":"sum"},{"name":"partition data size","accumulatorId":1192,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1188,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1189,"metricType":"timing"},{"name":"peak memory","accumulatorId":1187,"metricType":"size"},{"name":"number of output rows","accumulatorId":1186,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1190,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1185,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1183,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1184,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1181,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1179,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1180,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1177,"metricType":"size"},{"name":"local blocks read","accumulatorId":1176,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1175,"metricType":"sum"},{"name":"data size","accumulatorId":1174,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1178,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1182,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6, Statistics(sizeInBytes=630.0 B, isRuntime=true)","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1676]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(OP_CARRIER#180)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#180 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#180] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":939,"metricType":"timing"},{"name":"number of files read","accumulatorId":933,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":938,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":937,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":936,"metricType":"timing"},{"name":"metadata time","accumulatorId":934,"metricType":"timing"},{"name":"size of files read","accumulatorId":935,"metricType":"size"},{"name":"number of output rows","accumulatorId":932,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":987,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":988,"metricType":"timing"},{"name":"peak memory","accumulatorId":986,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":990,"metricType":"sum"},{"name":"number of output rows","accumulatorId":985,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":989,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":984,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":982,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":983,"metricType":"nsTiming"},{"name":"records read","accumulatorId":980,"metricType":"sum"},{"name":"local bytes read","accumulatorId":978,"metricType":"size"},{"name":"fetch wait time","accumulatorId":979,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":976,"metricType":"size"},{"name":"local blocks read","accumulatorId":975,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":974,"metricType":"sum"},{"name":"data size","accumulatorId":973,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":977,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":981,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1098,"metricType":"sum"},{"name":"partition data size","accumulatorId":1099,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1095,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1096,"metricType":"timing"},{"name":"peak memory","accumulatorId":1094,"metricType":"size"},{"name":"number of output rows","accumulatorId":1093,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1097,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1092,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1090,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1091,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1088,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1086,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1087,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1084,"metricType":"size"},{"name":"local blocks read","accumulatorId":1083,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1082,"metricType":"sum"},{"name":"data size","accumulatorId":1081,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1085,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1089,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1139,"metricType":"size"},{"name":"cumulative time","accumulatorId":1140,"metricType":"timing"},{"name":"peak memory","accumulatorId":1138,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":1142,"metricType":"sum"},{"name":"sort time","accumulatorId":1137,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":1143,"metricType":"sum"},{"name":"spill write time","accumulatorId":1141,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1136,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1135,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1134,"metricType":"timing"},{"name":"time to build","accumulatorId":1133,"metricType":"timing"},{"name":"time to collect","accumulatorId":1132,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1131,"metricType":"sum"},{"name":"data size","accumulatorId":1130,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":655,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":656,"metricType":"nsTiming"},{"name":"records read","accumulatorId":653,"metricType":"sum"},{"name":"local bytes read","accumulatorId":651,"metricType":"size"},{"name":"fetch wait time","accumulatorId":652,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":649,"metricType":"size"},{"name":"local blocks read","accumulatorId":648,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":647,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":650,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":654,"metricType":"size"}]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":9,"accumUpdates":[[1191,1],[1192,252030]]} +{"Event":"SparkListenerJobStart","Job ID":16,"Submission Time":1646432584548,"Stage Infos":[{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1834\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":66,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1828\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1833\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[26],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1066\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":54,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_1062\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[50,53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1065\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1061\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1058\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[27,26],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1834\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1834\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":66,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1828\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1833\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[26],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432584550,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_1834\",\"name\":\"Exchange\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":27,"Stage Attempt ID":0,"Task Info":{"Task ID":99,"Index":0,"Attempt":0,"Launch Time":1646432584585,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":25,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":98,"Index":0,"Attempt":0,"Launch Time":1646432584511,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432584604,"Failed":false,"Killed":false,"Accumulables":[{"ID":1082,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1084,"Name":"remote bytes read","Update":"259","Value":"259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1087,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1088,"Name":"records read","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1135,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1136,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1137,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1138,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1139,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1140,"Name":"cumulative time","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1141,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1142,"Name":"num prefix comparisons","Update":"79","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1144,"Name":"internal.metrics.executorDeserializeTime","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":1145,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19269705,"Value":19269705,"Internal":true,"Count Failed Values":true},{"ID":1146,"Name":"internal.metrics.executorRunTime","Update":56,"Value":56,"Internal":true,"Count Failed Values":true},{"ID":1147,"Name":"internal.metrics.executorCpuTime","Update":32986219,"Value":32986219,"Internal":true,"Count Failed Values":true},{"ID":1148,"Name":"internal.metrics.resultSize","Update":4523,"Value":4523,"Internal":true,"Count Failed Values":true},{"ID":1153,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1157,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1158,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1159,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":259,"Value":259,"Internal":true,"Count Failed Values":true},{"ID":1160,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1161,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1162,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1163,"Name":"internal.metrics.shuffle.read.recordsRead","Update":23,"Value":23,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":19,"Executor Deserialize CPU Time":19269705,"Executor Run Time":56,"Executor CPU Time":32986219,"Peak Execution Memory":67174400,"Result Size":4523,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":259,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":23},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1675\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[63],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[24],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432584462,"Completion Time":1646432584604,"Accumulables":[{"ID":1082,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1084,"Name":"remote bytes read","Value":"259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1087,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1088,"Name":"records read","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1135,"Name":"duration","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1136,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1137,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1138,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1139,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1140,"Name":"cumulative time","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1141,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1142,"Name":"num prefix comparisons","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1144,"Name":"internal.metrics.executorDeserializeTime","Value":19,"Internal":true,"Count Failed Values":true},{"ID":1145,"Name":"internal.metrics.executorDeserializeCpuTime","Value":19269705,"Internal":true,"Count Failed Values":true},{"ID":1146,"Name":"internal.metrics.executorRunTime","Value":56,"Internal":true,"Count Failed Values":true},{"ID":1147,"Name":"internal.metrics.executorCpuTime","Value":32986219,"Internal":true,"Count Failed Values":true},{"ID":1148,"Name":"internal.metrics.resultSize","Value":4523,"Internal":true,"Count Failed Values":true},{"ID":1153,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1157,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1158,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1159,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":259,"Internal":true,"Count Failed Values":true},{"ID":1160,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1161,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1162,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1163,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":15,"Completion Time":1646432584605,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":9,"accumUpdates":[[1134,5],[1133,29],[1132,200],[1131,23],[1130,4195328]]} +{"Event":"SparkListenerTaskEnd","Stage ID":27,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":99,"Index":0,"Attempt":0,"Launch Time":1646432584585,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432584656,"Failed":false,"Killed":false,"Accumulables":[{"ID":954,"Name":"remote blocks read","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":955,"Name":"local blocks read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":956,"Name":"remote bytes read","Update":"178398","Value":"178398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":958,"Name":"local bytes read","Update":"60083","Value":"60083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":959,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":960,"Name":"records read","Update":"6337","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1174,"Name":"data size","Update":"9144","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1182,"Name":"shuffle bytes written","Update":"2252","Value":"2252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1183,"Name":"shuffle records written","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1184,"Name":"shuffle write time","Update":"384395","Value":"384395","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1185,"Name":"duration","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1186,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1187,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1189,"Name":"time in aggregation build","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1190,"Name":"avg hash probe bucket list iters","Update":"1009","Value":"1009","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1193,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1194,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3423409,"Value":3423409,"Internal":true,"Count Failed Values":true},{"ID":1195,"Name":"internal.metrics.executorRunTime","Update":51,"Value":51,"Internal":true,"Count Failed Values":true},{"ID":1196,"Name":"internal.metrics.executorCpuTime","Update":44922225,"Value":44922225,"Internal":true,"Count Failed Values":true},{"ID":1197,"Name":"internal.metrics.resultSize","Update":3899,"Value":3899,"Internal":true,"Count Failed Values":true},{"ID":1202,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1206,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1207,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":1208,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":178398,"Value":178398,"Internal":true,"Count Failed Values":true},{"ID":1209,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1210,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":60083,"Value":60083,"Internal":true,"Count Failed Values":true},{"ID":1211,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1212,"Name":"internal.metrics.shuffle.read.recordsRead","Update":6337,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1213,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2252,"Value":2252,"Internal":true,"Count Failed Values":true},{"ID":1214,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":381,"Value":381,"Internal":true,"Count Failed Values":true},{"ID":1215,"Name":"internal.metrics.shuffle.write.writeTime","Update":384395,"Value":384395,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3423409,"Executor Run Time":51,"Executor CPU Time":44922225,"Peak Execution Memory":67371008,"Result Size":3899,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":15,"Local Blocks Fetched":5,"Fetch Wait Time":0,"Remote Bytes Read":178398,"Remote Bytes Read To Disk":0,"Local Bytes Read":60083,"Total Records Read":6337},"Shuffle Write Metrics":{"Shuffle Bytes Written":2252,"Shuffle Write Time":384395,"Shuffle Records Written":381},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1834\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":66,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1828\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1833\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[26],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432584550,"Completion Time":1646432584657,"Accumulables":[{"ID":954,"Name":"remote blocks read","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":955,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":956,"Name":"remote bytes read","Value":"178398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":958,"Name":"local bytes read","Value":"60083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":959,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":960,"Name":"records read","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1174,"Name":"data size","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1182,"Name":"shuffle bytes written","Value":"2252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1183,"Name":"shuffle records written","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1184,"Name":"shuffle write time","Value":"384395","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1185,"Name":"duration","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1186,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1187,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1189,"Name":"time in aggregation build","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1190,"Name":"avg hash probe bucket list iters","Value":"1009","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1193,"Name":"internal.metrics.executorDeserializeTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":1194,"Name":"internal.metrics.executorDeserializeCpuTime","Value":3423409,"Internal":true,"Count Failed Values":true},{"ID":1195,"Name":"internal.metrics.executorRunTime","Value":51,"Internal":true,"Count Failed Values":true},{"ID":1196,"Name":"internal.metrics.executorCpuTime","Value":44922225,"Internal":true,"Count Failed Values":true},{"ID":1197,"Name":"internal.metrics.resultSize","Value":3899,"Internal":true,"Count Failed Values":true},{"ID":1202,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1206,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":15,"Internal":true,"Count Failed Values":true},{"ID":1207,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":1208,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":178398,"Internal":true,"Count Failed Values":true},{"ID":1209,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1210,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":60083,"Internal":true,"Count Failed Values":true},{"ID":1211,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1212,"Name":"internal.metrics.shuffle.read.recordsRead","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1213,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2252,"Internal":true,"Count Failed Values":true},{"ID":1214,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":381,"Internal":true,"Count Failed Values":true},{"ID":1215,"Name":"internal.metrics.shuffle.write.writeTime","Value":384395,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":16,"Completion Time":1646432584668,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":9,"physicalPlanDescription":"== Physical Plan ==\nCollectLimit (92)\n+- AdaptiveSparkPlan (91)\n +- == Current Plan ==\n Project (42)\n +- BroadcastHashJoin Inner BuildRight (41)\n :- Project (27)\n : +- BroadcastHashJoin Inner BuildRight (26)\n : :- Project (23)\n : : +- BroadcastHashJoin Inner BuildRight (22)\n : : :- ShuffleQueryStage (4)\n : : : +- Exchange (3)\n : : : +- * Filter (2)\n : : : +- Scan csv (1)\n : : +- BroadcastQueryStage (21)\n : : +- BroadcastExchange (20)\n : : +- * Filter (19)\n : : +- * RunningWindowFunction (18)\n : : +- * Sort (17)\n : : +- ShuffleQueryStage (16)\n : : +- Exchange (15)\n : : +- * HashAggregate (14)\n : : +- CustomShuffleReader (13)\n : : +- ShuffleQueryStage (12)\n : : +- Exchange (11)\n : : +- * HashAggregate (10)\n : : +- Union (9)\n : : :- * Project (6)\n : : : +- Scan csv (5)\n : : +- * Project (8)\n : : +- Scan csv (7)\n : +- BroadcastQueryStage (25)\n : +- ReusedExchange (24)\n +- BroadcastQueryStage (40)\n +- BroadcastExchange (39)\n +- * Filter (38)\n +- * RunningWindowFunction (37)\n +- * Sort (36)\n +- ShuffleQueryStage (35)\n +- Exchange (34)\n +- * HashAggregate (33)\n +- CustomShuffleReader (32)\n +- ShuffleQueryStage (31)\n +- Exchange (30)\n +- * HashAggregate (29)\n +- Scan csv (28)\n +- == Initial Plan ==\n Project (90)\n +- SortMergeJoin Inner (89)\n :- Sort (79)\n : +- Exchange (78)\n : +- Project (77)\n : +- SortMergeJoin Inner (76)\n : :- Sort (61)\n : : +- Exchange (60)\n : : +- Project (59)\n : : +- SortMergeJoin Inner (58)\n : : :- Sort (45)\n : : : +- Exchange (44)\n : : : +- Filter (43)\n : : : +- Scan csv (1)\n : : +- Sort (57)\n : : +- Exchange (56)\n : : +- Filter (55)\n : : +- RunningWindowFunction (54)\n : : +- Sort (53)\n : : +- Exchange (52)\n : : +- HashAggregate (51)\n : : +- Exchange (50)\n : : +- HashAggregate (49)\n : : +- Union (48)\n : : :- Project (46)\n : : : +- Scan csv (5)\n : : +- Project (47)\n : : +- Scan csv (7)\n : +- Sort (75)\n : +- Exchange (74)\n : +- Filter (73)\n : +- RunningWindowFunction (72)\n : +- Sort (71)\n : +- Exchange (70)\n : +- HashAggregate (69)\n : +- Exchange (68)\n : +- HashAggregate (67)\n : +- Union (66)\n : :- Project (63)\n : : +- Scan csv (62)\n : +- Project (65)\n : +- Scan csv (64)\n +- Sort (88)\n +- Exchange (87)\n +- Filter (86)\n +- RunningWindowFunction (85)\n +- Sort (84)\n +- Exchange (83)\n +- HashAggregate (82)\n +- Exchange (81)\n +- HashAggregate (80)\n +- Scan csv (28)\n\n\n(1) Scan csv \nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nPushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]\nReadSchema: struct\n\n(2) Filter [codegen id : 1]\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(3) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]\n\n(4) ShuffleQueryStage\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: 0, Statistics(sizeInBytes=1816.0 MiB)\n\n(5) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(6) Project [codegen id : 2]\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(7) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(8) Project [codegen id : 3]\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(9) Union\n\n(10) HashAggregate [codegen id : 4]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(11) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]\n\n(12) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 1, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)\n\n(13) CustomShuffleReader\nInput [1]: [airport#144]\nArguments: coalesced\n\n(14) HashAggregate [codegen id : 11]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(15) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]\n\n(16) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 7, Statistics(sizeInBytes=8.9 KiB, rowCount=381, isRuntime=true)\n\n(17) Sort [codegen id : 13]\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(18) RunningWindowFunction [codegen id : 13]\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(19) Filter [codegen id : 13]\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(20) BroadcastExchange\nInput [2]: [airport#144, airport_id#149]\nArguments: HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#2105]\n\n(21) BroadcastQueryStage\nOutput [2]: [airport#144, airport_id#149]\nArguments: 10, Statistics(sizeInBytes=10.2 KiB, isRuntime=true)\n\n(22) BroadcastHashJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(23) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(24) ReusedExchange [Reuses operator id: 20]\nOutput [2]: [airport#178, airport_id#177]\n\n(25) BroadcastQueryStage\nOutput [2]: [airport#178, airport_id#177]\nArguments: 12, Statistics(sizeInBytes=10.2 KiB, isRuntime=true)\n\n(26) BroadcastHashJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(27) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(28) Scan csv \nOutput [1]: [OP_CARRIER#180]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(29) HashAggregate [codegen id : 8]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(30) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]\n\n(31) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)\n\n(32) CustomShuffleReader\nInput [1]: [OP_CARRIER#180]\nArguments: coalesced\n\n(33) HashAggregate [codegen id : 9]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(34) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]\n\n(35) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 5, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)\n\n(36) Sort [codegen id : 10]\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(37) RunningWindowFunction [codegen id : 10]\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(38) Filter [codegen id : 10]\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(39) BroadcastExchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1676]\n\n(40) BroadcastQueryStage\nOutput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: 6, Statistics(sizeInBytes=4.0 MiB, rowCount=23, isRuntime=true)\n\n(41) BroadcastHashJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(42) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(43) Filter\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(44) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#665]\n\n(45) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(46) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(47) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(48) Union\n\n(49) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(50) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#656]\n\n(51) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(52) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#659]\n\n(53) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(54) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(55) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(56) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#666]\n\n(57) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(58) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(59) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(60) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#680]\n\n(61) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(62) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(63) Project\nOutput [1]: [ORIGIN#21 AS airport#178]\nInput [1]: [ORIGIN#21]\n\n(64) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(65) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(66) Union\n\n(67) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(68) Exchange\nInput [1]: [airport#178]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#671]\n\n(69) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(70) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#674]\n\n(71) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(72) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(73) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(74) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#681]\n\n(75) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(76) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(77) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(78) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#695]\n\n(79) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(80) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(81) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#686]\n\n(82) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(83) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#689]\n\n(84) Sort\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(85) RunningWindowFunction\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(86) Filter\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(87) Exchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#696]\n\n(88) Sort\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(89) SortMergeJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(90) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(91) AdaptiveSparkPlan\nOutput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: isFinalPlan=false\n\n(92) CollectLimit\nInput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: 11\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 11","children":[{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [OP_CARRIER#19], [OP_CARRIER#180], Inner, BuildRight, AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25), false","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [DEST#22], [airport#178], Inner, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [ORIGIN#21], [airport#144], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=1816.0 MiB)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [FL_DATE#18,OP_CARRIER#19,ORIGIN#21,DEST#22,CRS_DEP_TIME#23,DEP_DELAY#25] Batched: false, DataFilters: [isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)], ReadSchema: struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]","DataFilters":"[isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)]"},"metrics":[{"name":"file sorting by size time","accumulatorId":722,"metricType":"timing"},{"name":"number of files read","accumulatorId":716,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":721,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":720,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":719,"metricType":"timing"},{"name":"metadata time","accumulatorId":717,"metricType":"timing"},{"name":"size of files read","accumulatorId":718,"metricType":"size"},{"name":"number of output rows","accumulatorId":715,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":952,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":951,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":949,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":950,"metricType":"nsTiming"},{"name":"records read","accumulatorId":947,"metricType":"sum"},{"name":"local bytes read","accumulatorId":945,"metricType":"size"},{"name":"fetch wait time","accumulatorId":946,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":943,"metricType":"size"},{"name":"local blocks read","accumulatorId":942,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":941,"metricType":"sum"},{"name":"data size","accumulatorId":940,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":944,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":948,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10, Statistics(sizeInBytes=10.2 KiB, isRuntime=true)","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#2105]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#144)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7, Statistics(sizeInBytes=8.9 KiB, rowCount=381, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1191,"metricType":"sum"},{"name":"partition data size","accumulatorId":1192,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1188,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1189,"metricType":"timing"},{"name":"peak memory","accumulatorId":1187,"metricType":"size"},{"name":"number of output rows","accumulatorId":1186,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1190,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1185,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1183,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1184,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1181,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1179,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1180,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1177,"metricType":"size"},{"name":"local blocks read","accumulatorId":1176,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1175,"metricType":"sum"},{"name":"data size","accumulatorId":1174,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1178,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1182,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1232,"metricType":"size"},{"name":"cumulative time","accumulatorId":1233,"metricType":"timing"},{"name":"peak memory","accumulatorId":1231,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":1235,"metricType":"sum"},{"name":"sort time","accumulatorId":1230,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":1236,"metricType":"sum"},{"name":"spill write time","accumulatorId":1234,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1228,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1227,"metricType":"timing"},{"name":"time to build","accumulatorId":1226,"metricType":"timing"},{"name":"time to collect","accumulatorId":1225,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1224,"metricType":"sum"},{"name":"data size","accumulatorId":1223,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 12, Statistics(sizeInBytes=10.2 KiB, isRuntime=true)","children":[{"nodeName":"ReusedExchange","simpleString":"ReusedExchange [airport#178, airport_id#177], BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#2105]","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#2105]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#144)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7, Statistics(sizeInBytes=8.9 KiB, rowCount=381, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1191,"metricType":"sum"},{"name":"partition data size","accumulatorId":1192,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1188,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1189,"metricType":"timing"},{"name":"peak memory","accumulatorId":1187,"metricType":"size"},{"name":"number of output rows","accumulatorId":1186,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1190,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1185,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1183,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1184,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1181,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1179,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1180,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1177,"metricType":"size"},{"name":"local blocks read","accumulatorId":1176,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1175,"metricType":"sum"},{"name":"data size","accumulatorId":1174,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1178,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1182,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1232,"metricType":"size"},{"name":"cumulative time","accumulatorId":1233,"metricType":"timing"},{"name":"peak memory","accumulatorId":1231,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":1235,"metricType":"sum"},{"name":"sort time","accumulatorId":1230,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":1236,"metricType":"sum"},{"name":"spill write time","accumulatorId":1234,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1228,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1227,"metricType":"timing"},{"name":"time to build","accumulatorId":1226,"metricType":"timing"},{"name":"time to collect","accumulatorId":1225,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1224,"metricType":"sum"},{"name":"data size","accumulatorId":1223,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6, Statistics(sizeInBytes=4.0 MiB, rowCount=23, isRuntime=true)","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1676]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(OP_CARRIER#180)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#180 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#180] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":939,"metricType":"timing"},{"name":"number of files read","accumulatorId":933,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":938,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":937,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":936,"metricType":"timing"},{"name":"metadata time","accumulatorId":934,"metricType":"timing"},{"name":"size of files read","accumulatorId":935,"metricType":"size"},{"name":"number of output rows","accumulatorId":932,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":987,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":988,"metricType":"timing"},{"name":"peak memory","accumulatorId":986,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":990,"metricType":"sum"},{"name":"number of output rows","accumulatorId":985,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":989,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":984,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":982,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":983,"metricType":"nsTiming"},{"name":"records read","accumulatorId":980,"metricType":"sum"},{"name":"local bytes read","accumulatorId":978,"metricType":"size"},{"name":"fetch wait time","accumulatorId":979,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":976,"metricType":"size"},{"name":"local blocks read","accumulatorId":975,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":974,"metricType":"sum"},{"name":"data size","accumulatorId":973,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":977,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":981,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1098,"metricType":"sum"},{"name":"partition data size","accumulatorId":1099,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1095,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1096,"metricType":"timing"},{"name":"peak memory","accumulatorId":1094,"metricType":"size"},{"name":"number of output rows","accumulatorId":1093,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1097,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1092,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1090,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1091,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1088,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1086,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1087,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1084,"metricType":"size"},{"name":"local blocks read","accumulatorId":1083,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1082,"metricType":"sum"},{"name":"data size","accumulatorId":1081,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1085,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1089,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1139,"metricType":"size"},{"name":"cumulative time","accumulatorId":1140,"metricType":"timing"},{"name":"peak memory","accumulatorId":1138,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":1142,"metricType":"sum"},{"name":"sort time","accumulatorId":1137,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":1143,"metricType":"sum"},{"name":"spill write time","accumulatorId":1141,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1136,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1135,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1134,"metricType":"timing"},{"name":"time to build","accumulatorId":1133,"metricType":"timing"},{"name":"time to collect","accumulatorId":1132,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1131,"metricType":"sum"},{"name":"data size","accumulatorId":1130,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":655,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":656,"metricType":"nsTiming"},{"name":"records read","accumulatorId":653,"metricType":"sum"},{"name":"local bytes read","accumulatorId":651,"metricType":"size"},{"name":"fetch wait time","accumulatorId":652,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":649,"metricType":"size"},{"name":"local blocks read","accumulatorId":648,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":647,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":650,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":654,"metricType":"size"}]}} +{"Event":"SparkListenerJobStart","Job ID":17,"Submission Time":1646432584756,"Stage Infos":[{"Stage ID":30,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":70,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2104\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[29],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":20,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1066\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_621\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_619\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":54,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_1062\",\"name\":\"Union\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[50,53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1065\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1061\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1058\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":29,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1834\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":66,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_1828\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1833\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[28],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[30,28,29],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_10\",\"name\":\"BroadcastQueryStage\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"broadcast exchange (runId 15d2eea1-90b9-4e06-b967-93b5c71a4a4b)","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"15d2eea1-90b9-4e06-b967-93b5c71a4a4b","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":30,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":70,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2104\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[29],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432584758,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.rdd.scope":"{\"id\":\"spark_plan_10\",\"name\":\"BroadcastQueryStage\"}","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.rdd.scope.noOverride":"true","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"broadcast exchange (runId 15d2eea1-90b9-4e06-b967-93b5c71a4a4b)","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"15d2eea1-90b9-4e06-b967-93b5c71a4a4b","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":30,"Stage Attempt ID":0,"Task Info":{"Task ID":100,"Index":0,"Attempt":0,"Launch Time":1646432584793,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":30,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":100,"Index":0,"Attempt":0,"Launch Time":1646432584793,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432584869,"Failed":false,"Killed":false,"Accumulables":[{"ID":1175,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1177,"Name":"remote bytes read","Update":"2252","Value":"2252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1180,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1181,"Name":"records read","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1228,"Name":"duration","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1229,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1230,"Name":"sort time","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1231,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1232,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1233,"Name":"cumulative time","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1234,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1235,"Name":"num prefix comparisons","Update":"2797","Value":"2797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1237,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":1238,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18457057,"Value":18457057,"Internal":true,"Count Failed Values":true},{"ID":1239,"Name":"internal.metrics.executorRunTime","Update":41,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":1240,"Name":"internal.metrics.executorCpuTime","Update":33980793,"Value":33980793,"Internal":true,"Count Failed Values":true},{"ID":1241,"Name":"internal.metrics.resultSize","Update":8776,"Value":8776,"Internal":true,"Count Failed Values":true},{"ID":1246,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1250,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1251,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1252,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":2252,"Value":2252,"Internal":true,"Count Failed Values":true},{"ID":1253,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1254,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1255,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1256,"Name":"internal.metrics.shuffle.read.recordsRead","Update":381,"Value":381,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":18457057,"Executor Run Time":41,"Executor CPU Time":33980793,"Peak Execution Memory":67174400,"Result Size":8776,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":2252,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":381},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":30,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":70,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2104\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[29],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432584758,"Completion Time":1646432584870,"Accumulables":[{"ID":1175,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1177,"Name":"remote bytes read","Value":"2252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1180,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1181,"Name":"records read","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1228,"Name":"duration","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1229,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1230,"Name":"sort time","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1231,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1232,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1233,"Name":"cumulative time","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1234,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1235,"Name":"num prefix comparisons","Value":"2797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1237,"Name":"internal.metrics.executorDeserializeTime","Value":18,"Internal":true,"Count Failed Values":true},{"ID":1238,"Name":"internal.metrics.executorDeserializeCpuTime","Value":18457057,"Internal":true,"Count Failed Values":true},{"ID":1239,"Name":"internal.metrics.executorRunTime","Value":41,"Internal":true,"Count Failed Values":true},{"ID":1240,"Name":"internal.metrics.executorCpuTime","Value":33980793,"Internal":true,"Count Failed Values":true},{"ID":1241,"Name":"internal.metrics.resultSize","Value":8776,"Internal":true,"Count Failed Values":true},{"ID":1246,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1250,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1251,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1252,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":2252,"Internal":true,"Count Failed Values":true},{"ID":1253,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1254,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1255,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1256,"Name":"internal.metrics.shuffle.read.recordsRead","Value":381,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":17,"Completion Time":1646432584873,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":9,"accumUpdates":[[1227,5],[1226,11],[1225,163],[1224,381],[1223,4210688]]} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":66,"Index":9,"Attempt":0,"Launch Time":1646432565378,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432585281,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"5617658","Value":"11437469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1615","Value":"2939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"356626668","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"175932825","Value":"356626668","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"539295168","Value":"1097997024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"74785065","Value":"152936090","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"5617658","Value":"11437469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"229177241","Value":"416807976","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"19760","Value":"38657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"5617658","Value":"11437469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4124958,"Value":8904922,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":19874,"Value":38873,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":19344099507,"Value":38068944059,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1702,"Value":3405,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":182,"Value":327,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":74785065,"Value":152936090,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5617658,"Value":11437469,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":229177241,"Value":416807976,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":356626668,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":11437469,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1615512746,"Value":2939616916,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":356626668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":8297822104,"JVMOffHeapMemory":127199904,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":599,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4124958,"Executor Run Time":19874,"Executor CPU Time":19344099507,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":182,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":74785065,"Shuffle Write Time":229177241,"Shuffle Records Written":5617658},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":61,"Index":4,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432585333,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"6066650","Value":"17504119","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1366","Value":"4305","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"541662548","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"185035880","Value":"541662548","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"582398400","Value":"1680395424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"79904590","Value":"232840680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"6066650","Value":"17504119","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"207551361","Value":"624359337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"19821","Value":"58478","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"6066650","Value":"17504119","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3805119,"Value":12710041,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":19927,"Value":58800,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":19405182650,"Value":57474126709,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1703,"Value":5108,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":353,"Value":680,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79904590,"Value":232840680,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6066650,"Value":17504119,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":207551361,"Value":624359337,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":541662548,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":17504119,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1366551125,"Value":4306168041,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":541662548,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":129174864,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1445149,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":917,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":3805119,"Executor Run Time":19927,"Executor CPU Time":19405182650,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":353,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":79904590,"Shuffle Write Time":207551361,"Shuffle Records Written":6066650},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":63,"Index":6,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432585455,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"5819079","Value":"23323198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1380","Value":"5685","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"723899413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"182236865","Value":"723899413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"558631584","Value":"2239027008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"77834569","Value":"310675249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"5819079","Value":"23323198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"188191162","Value":"812550499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"19946","Value":"78424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"5819079","Value":"23323198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4052626,"Value":16762667,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":20050,"Value":78850,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":19757551143,"Value":77231677852,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1703,"Value":6811,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":147,"Value":827,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":77834569,"Value":310675249,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819079,"Value":23323198,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":188191162,"Value":812550499,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":723899413,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":23323198,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1380334188,"Value":5686502229,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":723899413,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":126673080,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":525,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4052626,"Executor Run Time":20050,"Executor CPU Time":19757551143,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":147,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":77834569,"Shuffle Write Time":188191162,"Shuffle Records Written":5819079},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":65,"Index":8,"Attempt":0,"Launch Time":1646432565378,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432586492,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"5674621","Value":"28997819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1417","Value":"7102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"902117057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"178217644","Value":"902117057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"544763616","Value":"2783790624","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"75751912","Value":"386427161","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"5674621","Value":"28997819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"206027893","Value":"1018578392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"20984","Value":"99408","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"5674621","Value":"28997819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4113765,"Value":20876432,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":21089,"Value":99939,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":20553455817,"Value":97785133669,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1703,"Value":8514,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":353,"Value":1180,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":75751912,"Value":386427161,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5674621,"Value":28997819,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":206027893,"Value":1018578392,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":902117057,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":28997819,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1417904648,"Value":7104406877,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":902117057,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":129174864,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1445149,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":917,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4113765,"Executor Run Time":21089,"Executor CPU Time":20553455817,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":353,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":75751912,"Shuffle Write Time":206027893,"Shuffle Records Written":5674621},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":60,"Index":3,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432586878,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"6429338","Value":"35427157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1421","Value":"8523","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1097191372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"195074315","Value":"1097191372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"617216448","Value":"3401007072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"83388371","Value":"469815532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"6429338","Value":"35427157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"199697881","Value":"1218276273","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"21370","Value":"120778","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"6429338","Value":"35427157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3845461,"Value":24721893,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":21474,"Value":121413,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":21162189226,"Value":118947322895,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1703,"Value":10217,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":1325,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83388371,"Value":469815532,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6429338,"Value":35427157,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":199697881,"Value":1218276273,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1097191372,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":35427157,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1421371853,"Value":8525778730,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1097191372,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":9514678176,"JVMOffHeapMemory":128029656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":63,"MinorGCTime":550,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":3845461,"Executor Run Time":21474,"Executor CPU Time":21162189226,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":145,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83388371,"Shuffle Write Time":199697881,"Shuffle Records Written":6429338},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":59,"Index":2,"Attempt":0,"Launch Time":1646432565376,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432587545,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"6450117","Value":"41877274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1489","Value":"10012","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1294005672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"196814300","Value":"1294005672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"619211232","Value":"4020218304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"84724622","Value":"554540154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"6450117","Value":"41877274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"196720841","Value":"1414997114","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"22037","Value":"142815","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"6450117","Value":"41877274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4604028,"Value":29325921,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":22140,"Value":143553,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":21848185433,"Value":140795508328,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1703,"Value":11920,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":147,"Value":1472,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":84724622,"Value":554540154,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6450117,"Value":41877274,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":196720841,"Value":1414997114,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1294005672,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":41877274,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1489658744,"Value":10015437474,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1294005672,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6779866992,"JVMOffHeapMemory":126673080,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543149606,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":525,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4604028,"Executor Run Time":22140,"Executor CPU Time":21848185433,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":147,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":84724622,"Shuffle Write Time":196720841,"Shuffle Records Written":6450117},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":62,"Index":5,"Attempt":0,"Launch Time":1646432565377,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432588248,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"6096762","Value":"47974036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1712","Value":"11724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"183939508","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"585289152","Value":"4605507456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"80392000","Value":"634932154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"6096762","Value":"47974036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"250330611","Value":"1665327725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"22722","Value":"165537","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"6096762","Value":"47974036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4192263,"Value":33518184,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":22841,"Value":166394,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":22300999217,"Value":163096507545,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1702,"Value":13622,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":192,"Value":1664,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":80392000,"Value":634932154,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6096762,"Value":47974036,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":250330611,"Value":1665327725,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":1477945180,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":47974036,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1712468229,"Value":11727905703,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":1477945180,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":8297822104,"JVMOffHeapMemory":127199904,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":599,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4192263,"Executor Run Time":22841,"Executor CPU Time":22300999217,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":192,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":80392000,"Shuffle Write Time":250330611,"Shuffle Records Written":6096762},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":58,"Index":1,"Attempt":0,"Launch Time":1646432565375,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432591263,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"6369482","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1768","Value":"13492","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"611470272","Value":"5216977728","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"85250127","Value":"720182281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"6369482","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"260722640","Value":"1926050365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"25740","Value":"191277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"6369482","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5199334,"Value":38717518,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":25856,"Value":192250,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":25473248532,"Value":188569756077,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1702,"Value":15324,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":192,"Value":1856,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85250127,"Value":720182281,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6369482,"Value":54343518,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":260722640,"Value":1926050365,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1675583714,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":54343518,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1768502272,"Value":13496407975,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1675583714,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":8297822104,"JVMOffHeapMemory":127199904,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4181542,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":475778598,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":60,"MinorGCTime":599,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5199334,"Executor Run Time":25856,"Executor CPU Time":25473248532,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":192,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85250127,"Shuffle Write Time":260722640,"Shuffle Records Written":6369482},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":57,"Index":0,"Attempt":0,"Launch Time":1646432565375,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432591632,"Failed":false,"Killed":false,"Accumulables":[{"ID":715,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Update":"1764","Value":"15256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Update":"692490816","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Update":"98606155","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Update":"257547577","Value":"2183597942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Update":"26107","Value":"217384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5093364,"Value":43810882,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Update":26228,"Value":218478,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Update":25719182347,"Value":214288938424,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Update":1703,"Value":17027,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Update":359,"Value":2215,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":98606155,"Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7213446,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Update":257547577,"Value":2183597942,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1764618551,"Value":15261026526,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6409546464,"JVMOffHeapMemory":130123272,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4514787,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476111843,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":67,"MinorGCTime":928,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5093364,"Executor Run Time":26228,"Executor CPU Time":25719182347,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":359,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":98606155,"Shuffle Write Time":257547577,"Shuffle Records Written":7213446},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1046\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1045\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_616\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_616\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432565356,"Completion Time":1646432591633,"Accumulables":[{"ID":715,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":719,"Name":"filesystem read time (sampled)","Value":"15256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":720,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":721,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"data size","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":948,"Name":"shuffle bytes written","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":949,"Name":"shuffle records written","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":950,"Name":"shuffle write time","Value":"2183597942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":951,"Name":"duration","Value":"217384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":952,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":991,"Name":"internal.metrics.executorDeserializeTime","Value":50,"Internal":true,"Count Failed Values":true},{"ID":992,"Name":"internal.metrics.executorDeserializeCpuTime","Value":43810882,"Internal":true,"Count Failed Values":true},{"ID":993,"Name":"internal.metrics.executorRunTime","Value":218478,"Internal":true,"Count Failed Values":true},{"ID":994,"Name":"internal.metrics.executorCpuTime","Value":214288938424,"Internal":true,"Count Failed Values":true},{"ID":995,"Name":"internal.metrics.resultSize","Value":17027,"Internal":true,"Count Failed Values":true},{"ID":996,"Name":"internal.metrics.jvmGCTime","Value":2215,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.shuffle.write.writeTime","Value":2183597942,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1015,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.input.sampledTimeReadNano","Value":15261026526,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":11,"Completion Time":1646432591634,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":9,"physicalPlanDescription":"== Physical Plan ==\nCollectLimit (93)\n+- AdaptiveSparkPlan (92)\n +- == Final Plan ==\n * Project (43)\n +- * BroadcastHashJoin Inner BuildRight (42)\n :- * Project (28)\n : +- * BroadcastHashJoin Inner BuildRight (27)\n : :- * Project (24)\n : : +- * BroadcastHashJoin Inner BuildRight (23)\n : : :- CustomShuffleReader (5)\n : : : +- ShuffleQueryStage (4)\n : : : +- Exchange (3)\n : : : +- * Filter (2)\n : : : +- Scan csv (1)\n : : +- BroadcastQueryStage (22)\n : : +- BroadcastExchange (21)\n : : +- * Filter (20)\n : : +- * RunningWindowFunction (19)\n : : +- * Sort (18)\n : : +- ShuffleQueryStage (17)\n : : +- Exchange (16)\n : : +- * HashAggregate (15)\n : : +- CustomShuffleReader (14)\n : : +- ShuffleQueryStage (13)\n : : +- Exchange (12)\n : : +- * HashAggregate (11)\n : : +- Union (10)\n : : :- * Project (7)\n : : : +- Scan csv (6)\n : : +- * Project (9)\n : : +- Scan csv (8)\n : +- BroadcastQueryStage (26)\n : +- ReusedExchange (25)\n +- BroadcastQueryStage (41)\n +- BroadcastExchange (40)\n +- * Filter (39)\n +- * RunningWindowFunction (38)\n +- * Sort (37)\n +- ShuffleQueryStage (36)\n +- Exchange (35)\n +- * HashAggregate (34)\n +- CustomShuffleReader (33)\n +- ShuffleQueryStage (32)\n +- Exchange (31)\n +- * HashAggregate (30)\n +- Scan csv (29)\n +- == Initial Plan ==\n Project (91)\n +- SortMergeJoin Inner (90)\n :- Sort (80)\n : +- Exchange (79)\n : +- Project (78)\n : +- SortMergeJoin Inner (77)\n : :- Sort (62)\n : : +- Exchange (61)\n : : +- Project (60)\n : : +- SortMergeJoin Inner (59)\n : : :- Sort (46)\n : : : +- Exchange (45)\n : : : +- Filter (44)\n : : : +- Scan csv (1)\n : : +- Sort (58)\n : : +- Exchange (57)\n : : +- Filter (56)\n : : +- RunningWindowFunction (55)\n : : +- Sort (54)\n : : +- Exchange (53)\n : : +- HashAggregate (52)\n : : +- Exchange (51)\n : : +- HashAggregate (50)\n : : +- Union (49)\n : : :- Project (47)\n : : : +- Scan csv (6)\n : : +- Project (48)\n : : +- Scan csv (8)\n : +- Sort (76)\n : +- Exchange (75)\n : +- Filter (74)\n : +- RunningWindowFunction (73)\n : +- Sort (72)\n : +- Exchange (71)\n : +- HashAggregate (70)\n : +- Exchange (69)\n : +- HashAggregate (68)\n : +- Union (67)\n : :- Project (64)\n : : +- Scan csv (63)\n : +- Project (66)\n : +- Scan csv (65)\n +- Sort (89)\n +- Exchange (88)\n +- Filter (87)\n +- RunningWindowFunction (86)\n +- Sort (85)\n +- Exchange (84)\n +- HashAggregate (83)\n +- Exchange (82)\n +- HashAggregate (81)\n +- Scan csv (29)\n\n\n(1) Scan csv \nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nPushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]\nReadSchema: struct\n\n(2) Filter [codegen id : 1]\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(3) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]\n\n(4) ShuffleQueryStage\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: 0, Statistics(sizeInBytes=5.5 GiB, rowCount=6.16E+7, isRuntime=true)\n\n(5) CustomShuffleReader\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: local\n\n(6) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(7) Project [codegen id : 2]\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(8) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(9) Project [codegen id : 3]\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(10) Union\n\n(11) HashAggregate [codegen id : 4]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(12) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]\n\n(13) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 1, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)\n\n(14) CustomShuffleReader\nInput [1]: [airport#144]\nArguments: coalesced\n\n(15) HashAggregate [codegen id : 11]\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(16) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]\n\n(17) ShuffleQueryStage\nOutput [1]: [airport#144]\nArguments: 7, Statistics(sizeInBytes=8.9 KiB, rowCount=381, isRuntime=true)\n\n(18) Sort [codegen id : 13]\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(19) RunningWindowFunction [codegen id : 13]\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(20) Filter [codegen id : 13]\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(21) BroadcastExchange\nInput [2]: [airport#144, airport_id#149]\nArguments: HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#2105]\n\n(22) BroadcastQueryStage\nOutput [2]: [airport#144, airport_id#149]\nArguments: 10, Statistics(sizeInBytes=4.0 MiB, rowCount=381, isRuntime=true)\n\n(23) BroadcastHashJoin [codegen id : 15]\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(24) Project [codegen id : 15]\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(25) ReusedExchange [Reuses operator id: 21]\nOutput [2]: [airport#178, airport_id#177]\n\n(26) BroadcastQueryStage\nOutput [2]: [airport#178, airport_id#177]\nArguments: 12, Statistics(sizeInBytes=4.0 MiB, rowCount=381, isRuntime=true)\n\n(27) BroadcastHashJoin [codegen id : 15]\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(28) Project [codegen id : 15]\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(29) Scan csv \nOutput [1]: [OP_CARRIER#180]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(30) HashAggregate [codegen id : 8]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(31) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]\n\n(32) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)\n\n(33) CustomShuffleReader\nInput [1]: [OP_CARRIER#180]\nArguments: coalesced\n\n(34) HashAggregate [codegen id : 9]\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(35) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]\n\n(36) ShuffleQueryStage\nOutput [1]: [OP_CARRIER#180]\nArguments: 5, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)\n\n(37) Sort [codegen id : 10]\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(38) RunningWindowFunction [codegen id : 10]\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(39) Filter [codegen id : 10]\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(40) BroadcastExchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1676]\n\n(41) BroadcastQueryStage\nOutput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: 6, Statistics(sizeInBytes=4.0 MiB, rowCount=23, isRuntime=true)\n\n(42) BroadcastHashJoin [codegen id : 15]\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(43) Project [codegen id : 15]\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(44) Filter\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nCondition : ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))\n\n(45) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#665]\n\n(46) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25]\nArguments: [ORIGIN#21 ASC NULLS FIRST], false, 0\n\n(47) Project\nOutput [1]: [ORIGIN#21 AS airport#144]\nInput [1]: [ORIGIN#21]\n\n(48) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(49) Union\n\n(50) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(51) Exchange\nInput [1]: [airport#144]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#656]\n\n(52) HashAggregate\nInput [1]: [airport#144]\nKeys [1]: [airport#144]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#144]\n\n(53) Exchange\nInput [1]: [airport#144]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#659]\n\n(54) Sort\nInput [1]: [airport#144]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(55) RunningWindowFunction\nInput [1]: [airport#144]\nArguments: [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false\n\n(56) Filter\nInput [2]: [airport#144, airport_id#149]\nCondition : isnotnull(airport#144)\n\n(57) Exchange\nInput [2]: [airport#144, airport_id#149]\nArguments: hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#666]\n\n(58) Sort\nInput [2]: [airport#144, airport_id#149]\nArguments: [airport#144 ASC NULLS FIRST], false, 0\n\n(59) SortMergeJoin\nLeft keys [1]: [ORIGIN#21]\nRight keys [1]: [airport#144]\nJoin condition: None\n\n(60) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, ORIGIN#21, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport#144, airport_id#149]\n\n(61) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: hashpartitioning(DEST#22, 200), ENSURE_REQUIREMENTS, [id=#680]\n\n(62) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]\nArguments: [DEST#22 ASC NULLS FIRST], false, 0\n\n(63) Scan csv \nOutput [1]: [ORIGIN#21]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(64) Project\nOutput [1]: [ORIGIN#21 AS airport#178]\nInput [1]: [ORIGIN#21]\n\n(65) Scan csv \nOutput [1]: [DEST#22]\nBatched: false\nLocation: InMemoryFileIndex [s3://job-bank/ml-rec/input-data/airlinedelay/gz]\nReadSchema: struct\n\n(66) Project\nOutput [1]: [DEST#22 AS airport#146]\nInput [1]: [DEST#22]\n\n(67) Union\n\n(68) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(69) Exchange\nInput [1]: [airport#178]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#671]\n\n(70) HashAggregate\nInput [1]: [airport#178]\nKeys [1]: [airport#178]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [airport#178]\n\n(71) Exchange\nInput [1]: [airport#178]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#674]\n\n(72) Sort\nInput [1]: [airport#178]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(73) RunningWindowFunction\nInput [1]: [airport#178]\nArguments: [airport#178, row_number() windowspecdefinition(airport#178 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#177], [airport#178 ASC NULLS FIRST], false\n\n(74) Filter\nInput [2]: [airport#178, airport_id#177]\nCondition : isnotnull(airport#178)\n\n(75) Exchange\nInput [2]: [airport#178, airport_id#177]\nArguments: hashpartitioning(airport#178, 200), ENSURE_REQUIREMENTS, [id=#681]\n\n(76) Sort\nInput [2]: [airport#178, airport_id#177]\nArguments: [airport#178 ASC NULLS FIRST], false, 0\n\n(77) SortMergeJoin\nLeft keys [1]: [DEST#22]\nRight keys [1]: [airport#178]\nJoin condition: None\n\n(78) Project\nOutput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport#178, airport_id#177]\n\n(79) Exchange\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: hashpartitioning(OP_CARRIER#19, 200), ENSURE_REQUIREMENTS, [id=#695]\n\n(80) Sort\nInput [6]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]\nArguments: [OP_CARRIER#19 ASC NULLS FIRST], false, 0\n\n(81) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(82) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#686]\n\n(83) HashAggregate\nInput [1]: [OP_CARRIER#180]\nKeys [1]: [OP_CARRIER#180]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [OP_CARRIER#180]\n\n(84) Exchange\nInput [1]: [OP_CARRIER#180]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [id=#689]\n\n(85) Sort\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(86) RunningWindowFunction\nInput [1]: [OP_CARRIER#180]\nArguments: [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false\n\n(87) Filter\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nCondition : isnotnull(OP_CARRIER#180)\n\n(88) Exchange\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#696]\n\n(89) Sort\nInput [2]: [OP_CARRIER#180, carrier_id#123]\nArguments: [OP_CARRIER#180 ASC NULLS FIRST], false, 0\n\n(90) SortMergeJoin\nLeft keys [1]: [OP_CARRIER#19]\nRight keys [1]: [OP_CARRIER#180]\nJoin condition: AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25)\n\n(91) Project\nOutput [8]: [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]\nInput [8]: [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177, OP_CARRIER#180, carrier_id#123]\n\n(92) AdaptiveSparkPlan\nOutput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: isFinalPlan=true\n\n(93) CollectLimit\nInput [8]: [month#231, day_of_month#232, day_of_week#233, OP_CARRIER#234, ORIGIN#235, DEST#236, CRS_DEP_TIME#237, DEP_DELAY#238]\nArguments: 11\n\n","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 11","children":[{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"WholeStageCodegen (15)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [cast(month(cast(FL_DATE#18 as date)) as string) AS month#231, cast(dayofmonth(cast(FL_DATE#18 as date)) as string) AS day_of_month#232, cast(dayofweek(cast(FL_DATE#18 as date)) as string) AS day_of_week#233, cast(carrier_id#123 as string) AS OP_CARRIER#234, cast(airport_id#149 as string) AS ORIGIN#235, cast(airport_id#177 as string) AS DEST#236, cast(CRS_DEP_TIME#23 as string) AS CRS_DEP_TIME#237, cast(DEP_DELAY#25 as string) AS DEP_DELAY#238]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [OP_CARRIER#19], [OP_CARRIER#180], Inner, BuildRight, AtLeastNNulls(n, month(cast(FL_DATE#18 as date)),dayofmonth(cast(FL_DATE#18 as date)),dayofweek(cast(FL_DATE#18 as date)),carrier_id#123,airport_id#149,airport_id#177,CRS_DEP_TIME#23,DEP_DELAY#25), false","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149, airport_id#177]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [DEST#22], [airport#178], Inner, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [FL_DATE#18, OP_CARRIER#19, DEST#22, CRS_DEP_TIME#23, DEP_DELAY#25, airport_id#149]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [ORIGIN#21], [airport#144], Inner, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0, Statistics(sizeInBytes=5.5 GiB, rowCount=6.16E+7, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(ORIGIN#21, 200), ENSURE_REQUIREMENTS, [id=#1046]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter ((isnotnull(ORIGIN#21) AND isnotnull(DEST#22)) AND isnotnull(OP_CARRIER#19))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [FL_DATE#18,OP_CARRIER#19,ORIGIN#21,DEST#22,CRS_DEP_TIME#23,DEP_DELAY#25] Batched: false, DataFilters: [isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)], ReadSchema: struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[IsNotNull(ORIGIN), IsNotNull(DEST), IsNotNull(OP_CARRIER)]","DataFilters":"[isnotnull(ORIGIN#21), isnotnull(DEST#22), isnotnull(OP_CARRIER#19)]"},"metrics":[{"name":"file sorting by size time","accumulatorId":722,"metricType":"timing"},{"name":"number of files read","accumulatorId":716,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":721,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":720,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":719,"metricType":"timing"},{"name":"metadata time","accumulatorId":717,"metricType":"timing"},{"name":"size of files read","accumulatorId":718,"metricType":"size"},{"name":"number of output rows","accumulatorId":715,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":952,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":951,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":949,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":950,"metricType":"nsTiming"},{"name":"records read","accumulatorId":947,"metricType":"sum"},{"name":"local bytes read","accumulatorId":945,"metricType":"size"},{"name":"fetch wait time","accumulatorId":946,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":943,"metricType":"size"},{"name":"local blocks read","accumulatorId":942,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":941,"metricType":"sum"},{"name":"data size","accumulatorId":940,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":944,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":948,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1274,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10, Statistics(sizeInBytes=4.0 MiB, rowCount=381, isRuntime=true)","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#2105]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#144)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7, Statistics(sizeInBytes=8.9 KiB, rowCount=381, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1191,"metricType":"sum"},{"name":"partition data size","accumulatorId":1192,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1188,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1189,"metricType":"timing"},{"name":"peak memory","accumulatorId":1187,"metricType":"size"},{"name":"number of output rows","accumulatorId":1186,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1190,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1185,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1183,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1184,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1181,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1179,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1180,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1177,"metricType":"size"},{"name":"local blocks read","accumulatorId":1176,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1175,"metricType":"sum"},{"name":"data size","accumulatorId":1174,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1178,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1182,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1232,"metricType":"size"},{"name":"cumulative time","accumulatorId":1233,"metricType":"timing"},{"name":"peak memory","accumulatorId":1231,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":1235,"metricType":"sum"},{"name":"sort time","accumulatorId":1230,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":1236,"metricType":"sum"},{"name":"spill write time","accumulatorId":1234,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1228,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1227,"metricType":"timing"},{"name":"time to build","accumulatorId":1226,"metricType":"timing"},{"name":"time to collect","accumulatorId":1225,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1224,"metricType":"sum"},{"name":"data size","accumulatorId":1223,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1272,"metricType":"sum"},{"name":"time to read broadcast","accumulatorId":1273,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 12, Statistics(sizeInBytes=4.0 MiB, rowCount=381, isRuntime=true)","children":[{"nodeName":"ReusedExchange","simpleString":"ReusedExchange [airport#178, airport_id#177], BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#2105]","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#2105]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(airport#144)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [airport#144, row_number() windowspecdefinition(airport#144 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS airport_id#149], [airport#144 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [airport#144 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7, Statistics(sizeInBytes=8.9 KiB, rowCount=381, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1834]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1, Statistics(sizeInBytes=148.5 KiB, rowCount=6.34E+3, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(airport#144, 200), ENSURE_REQUIREMENTS, [id=#1066]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[airport#144], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Union","simpleString":"Union","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [ORIGIN#21 AS airport#144]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [ORIGIN#21] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":789,"metricType":"timing"},{"name":"number of files read","accumulatorId":783,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":788,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":787,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":786,"metricType":"timing"},{"name":"metadata time","accumulatorId":784,"metricType":"timing"},{"name":"size of files read","accumulatorId":785,"metricType":"size"},{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":971,"metricType":"timing"}]},{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Project","simpleString":"Project [DEST#22 AS airport#146]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [DEST#22] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":797,"metricType":"timing"},{"name":"number of files read","accumulatorId":791,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":796,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":795,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":794,"metricType":"timing"},{"name":"metadata time","accumulatorId":792,"metricType":"timing"},{"name":"size of files read","accumulatorId":793,"metricType":"size"},{"name":"number of output rows","accumulatorId":790,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":972,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":967,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":968,"metricType":"timing"},{"name":"peak memory","accumulatorId":966,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of output rows","accumulatorId":965,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":969,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":964,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":963,"metricType":"nsTiming"},{"name":"records read","accumulatorId":960,"metricType":"sum"},{"name":"local bytes read","accumulatorId":958,"metricType":"size"},{"name":"fetch wait time","accumulatorId":959,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":956,"metricType":"size"},{"name":"local blocks read","accumulatorId":955,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":954,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":957,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":961,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1191,"metricType":"sum"},{"name":"partition data size","accumulatorId":1192,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1188,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1189,"metricType":"timing"},{"name":"peak memory","accumulatorId":1187,"metricType":"size"},{"name":"number of output rows","accumulatorId":1186,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1190,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1185,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1183,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1184,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1181,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1179,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1180,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1177,"metricType":"size"},{"name":"local blocks read","accumulatorId":1176,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1175,"metricType":"sum"},{"name":"data size","accumulatorId":1174,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1178,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1182,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1232,"metricType":"size"},{"name":"cumulative time","accumulatorId":1233,"metricType":"timing"},{"name":"peak memory","accumulatorId":1231,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":1235,"metricType":"sum"},{"name":"sort time","accumulatorId":1230,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":1236,"metricType":"sum"},{"name":"spill write time","accumulatorId":1234,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1228,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1227,"metricType":"timing"},{"name":"time to build","accumulatorId":1226,"metricType":"timing"},{"name":"time to collect","accumulatorId":1225,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1224,"metricType":"sum"},{"name":"data size","accumulatorId":1223,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1270,"metricType":"sum"},{"name":"time to read broadcast","accumulatorId":1271,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6, Statistics(sizeInBytes=4.0 MiB, rowCount=23, isRuntime=true)","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(ArrayBuffer(input[0, string, false]),false), [id=#1676]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(OP_CARRIER#180)","children":[{"nodeName":"RunningWindowFunction","simpleString":"RunningWindowFunction [OP_CARRIER#180, row_number() windowspecdefinition(OP_CARRIER#180 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS carrier_id#123], [OP_CARRIER#180 ASC NULLS FIRST], false","children":[{"nodeName":"Sort","simpleString":"Sort [OP_CARRIER#180 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5, Statistics(sizeInBytes=552.0 B, rowCount=23, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#1458]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"CustomShuffleReader","simpleString":"CustomShuffleReader coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4, Statistics(sizeInBytes=3.6 KiB, rowCount=154, isRuntime=true)","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(OP_CARRIER#180, 200), ENSURE_REQUIREMENTS, [id=#1142]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[OP_CARRIER#180], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan csv ","simpleString":"FileScan csv [OP_CARRIER#180] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz], PartitionFilters: [], PushedFilters: [], ReadSchema: struct","children":[],"metadata":{"Location":"InMemoryFileIndex[s3://job-bank/ml-rec/input-data/airlinedelay/gz]","ReadSchema":"struct","Format":"CSV","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"file sorting by size time","accumulatorId":939,"metricType":"timing"},{"name":"number of files read","accumulatorId":933,"metricType":"sum"},{"name":"filesystem read data size","accumulatorId":938,"metricType":"size"},{"name":"filesystem read data size (sampled)","accumulatorId":937,"metricType":"size"},{"name":"filesystem read time (sampled)","accumulatorId":936,"metricType":"timing"},{"name":"metadata time","accumulatorId":934,"metricType":"timing"},{"name":"size of files read","accumulatorId":935,"metricType":"size"},{"name":"number of output rows","accumulatorId":932,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":987,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":988,"metricType":"timing"},{"name":"peak memory","accumulatorId":986,"metricType":"size"},{"name":"passthrough output rows","accumulatorId":990,"metricType":"sum"},{"name":"number of output rows","accumulatorId":985,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":989,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":984,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":982,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":983,"metricType":"nsTiming"},{"name":"records read","accumulatorId":980,"metricType":"sum"},{"name":"local bytes read","accumulatorId":978,"metricType":"size"},{"name":"fetch wait time","accumulatorId":979,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":976,"metricType":"size"},{"name":"local blocks read","accumulatorId":975,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":974,"metricType":"sum"},{"name":"data size","accumulatorId":973,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":977,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":981,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1098,"metricType":"sum"},{"name":"partition data size","accumulatorId":1099,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1095,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1096,"metricType":"timing"},{"name":"peak memory","accumulatorId":1094,"metricType":"size"},{"name":"number of output rows","accumulatorId":1093,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":1097,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1092,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1090,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1091,"metricType":"nsTiming"},{"name":"records read","accumulatorId":1088,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1086,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1087,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1084,"metricType":"size"},{"name":"local blocks read","accumulatorId":1083,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1082,"metricType":"sum"},{"name":"data size","accumulatorId":1081,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":1085,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1089,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1139,"metricType":"size"},{"name":"cumulative time","accumulatorId":1140,"metricType":"timing"},{"name":"peak memory","accumulatorId":1138,"metricType":"size"},{"name":"num prefix comparisons","accumulatorId":1142,"metricType":"sum"},{"name":"sort time","accumulatorId":1137,"metricType":"timing"},{"name":"num record comparisons","accumulatorId":1143,"metricType":"sum"},{"name":"spill write time","accumulatorId":1141,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1136,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1135,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1134,"metricType":"timing"},{"name":"time to build","accumulatorId":1133,"metricType":"timing"},{"name":"time to collect","accumulatorId":1132,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1131,"metricType":"sum"},{"name":"data size","accumulatorId":1130,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1268,"metricType":"sum"},{"name":"time to read broadcast","accumulatorId":1269,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1267,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":655,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":656,"metricType":"nsTiming"},{"name":"records read","accumulatorId":653,"metricType":"sum"},{"name":"local bytes read","accumulatorId":651,"metricType":"size"},{"name":"fetch wait time","accumulatorId":652,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":649,"metricType":"size"},{"name":"local blocks read","accumulatorId":648,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":647,"metricType":"sum"},{"name":"remote bytes read to disk","accumulatorId":650,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":654,"metricType":"size"}]}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":9,"accumUpdates":[[1274,50]]} +{"Event":"SparkListenerJobStart","Job ID":18,"Submission Time":1646432591756,"Stage Infos":[{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":10,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1046\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_1045\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_616\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_616\",\"name\":\"Scan csv \"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":32,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":72,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2278\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_2261\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[31],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[31,32],"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":32,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":72,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2278\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_2261\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[31],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432591763,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.clusterUsageTags.clusterNoDriverDaemon":"false","spark.databricks.delta.multiClusterWrites.enabled":"true","spark.databricks.clusterUsageTags.cloudProvider":"AWS","spark.databricks.managedCatalog.gcs.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.abfs.impl.disable.cache":"true","spark.databricks.delta.logStore.crossCloud.fatal":"true","spark.databricks.job.id":"703","spark.databricks.clusterUsageTags.hailEnabled":"false","spark.scheduler.pool":"4598586477831989067","spark.databricks.service.dbutils.server.backend":"com.databricks.dbconnect.SparkServerDBUtils","spark.hadoop.fs.fcfs-s3a.impl":"*********(redacted)","spark.hadoop.databricks.s3.create.deleteUnnecessaryFakeDirectories":"false","spark.hadoop.fs.s3a.fast.upload.default":"true","spark.databricks.clusterUsageTags.instanceWorkerEnvNetworkType":"default","spark.sql.warehouse.dir":"/user/hive/warehouse","spark.databricks.clusterUsageTags.attribute_tag_dust_execution_env":"","spark.speculation":"false","spark.hadoop.databricks.dbfs.client.version":"v2","spark.databricks.clusterUsageTags.attribute_tag_budget":"","spark.hadoop.hive.server2.use.SSL":"true","spark.hadoop.fs.fcfs-s3a.impl.disable.cache":"true","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeType":"ebs_volume_type: GENERAL_PURPOSE_SSD\n","spark.driver.host":"10.77.243.103","spark.sql.hive.convertMetastoreParquet":"true","spark.scheduler.listenerbus.eventqueue.capacity":"20000","spark.serializer.objectStreamReset":"100","spark.databricks.clusterUsageTags.instanceBootstrapType":"vm-selfbootstrap","spark.databricks.clusterUsageTags.attribute_tag_service":"","spark.databricks.env":"databricks-python-repl","spark.databricks.session.share":"false","spark.databricks.tahoe.logStore.gcp.class":"com.databricks.tahoe.store.GCPLogStore","spark.databricks.passthrough.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.driverInstancePrivateIp":"10.77.250.60","spark.databricks.clusterUsageTags.clusterNumCustomTags":"0","spark.hadoop.hive.hmshandler.retry.interval":"2000","spark.eventLog.enabled":"false","spark.r.backendConnectionTimeout":"604800","spark.hadoop.fs.s3a.multipart.threshold":"104857600","spark.shuffle.manager":"SORT","spark.hadoop.fs.wasb.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.streaming.driver.writeAheadLog.closeFileAfterWrite":"true","spark.databricks.secret.envVar.keys.toRedact":"*********(redacted)","spark.databricks.clusterUsageTags.clusterEbsVolumeSize":"100","spark.databricks.managedCatalog.s3a.tokenProviderClassName":"*********(redacted)","spark.hadoop.fs.wasbs.impl.disable.cache":"true","spark.cleaner.referenceTracking.blocking":"false","spark.databricks.clusterUsageTags.region":"us-east-1","spark.databricks.clusterUsageTags.ngrokNpipEnabled":"true","spark.driver.maxResultSize":"4g","spark.metrics.conf":"/databricks/spark/conf/metrics.properties","spark.hadoop.fs.s3.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.clusterSku":"STANDARD_SKU","spark.databricks.clusterUsageTags.clusterOwnerOrgId":"656201176161048","spark.hadoop.fs.cpfs-s3.impl":"*********(redacted)","spark.databricks.clusterUsageTags.autoTerminationMinutes":"0","spark.databricks.clusterUsageTags.clusterGeneration":"0","spark.hadoop.fs.adl.impl":"com.databricks.adl.AdlFileSystem","spark.databricks.clusterUsageTags.enableElasticDisk":"false","spark.hadoop.fs.abfss.impl.disable.cache":"true","spark.databricks.clusterUsageTags.enableCredentialPassthrough":"false","spark.ui.port":"40001","spark.job.interruptOnCancel":"true","spark.hadoop.fs.gs.outputstream.upload.chunk.size":"16777216","spark.hadoop.spark.driverproxy.customHeadersToProperties":"*********(redacted)","spark.driver.port":"39021","spark.shuffle.reduceLocality.enabled":"false","spark.shuffle.service.enabled":"true","spark.databricks.acl.provider":"com.databricks.sql.acl.ReflectionBackedAclProvider","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.eventLog.dir":"eventlogs","spark.extraListeners":"com.databricks.backend.daemon.driver.DBCEventLoggingListener","spark.eventLog.buffer.kb":"1000000k","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.managedCatalog.clientClassName":"com.databricks.managedcatalog.ManagedCatalogClientImpl","spark.hadoop.hive.server2.keystore.password":"*********(redacted)","spark.rdd.compress":"true","spark.hadoop.fs.mcfs-abfss.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.hadoop.hive.hmshandler.retry.attempts":"10","spark.databricks.clusterUsageTags.instanceProfileUsed":"true","spark.databricks.service.client.type":"python","spark.hadoop.fs.file.impl":"com.databricks.backend.daemon.driver.WorkspaceLocalFileSystem","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeCount":"3","spark.databricks.clusterUsageTags.userProvidedRemoteVolumeSizeGb":"100","spark.databricks.passthrough.glue.credentialsProviderFactoryClassName":"*********(redacted)","spark.databricks.redactor":"com.databricks.spark.util.DatabricksSparkLogRedactorProxy","spark.databricks.clusterUsageTags.driverInstanceId":"i-01ae3124fd2891da4","spark.databricks.clusterUsageTags.privateLinkEnabled":"false","spark.hadoop.parquet.memory.pool.ratio":"0.5","spark.databricks.clusterUsageTags.clusterName":"job-703-run-31144","spark.repl.class.uri":"spark://10.77.243.103:39021/classes","spark.databricks.clusterUsageTags.sparkEnvVarContainsBacktick":"false","spark.databricks.clusterUsageTags.clusterFirstOnDemand":"1","spark.driver.tempDirectory":"/local_disk0/tmp","spark.databricks.passthrough.s3a.threadPoolExecutor.factory.class":"com.databricks.backend.daemon.driver.aws.S3APassthroughThreadPoolExecutorFactory","spark.hadoop.fs.abfs.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","spark.hadoop.fs.wasbs.impl":"shaded.databricks.org.apache.hadoop.fs.azure.NativeAzureFileSystem","spark.databricks.clusterUsageTags.sparkVersion":"9.1.x-scala2.12","spark.hadoop.fs.fcfs-s3.impl":"*********(redacted)","spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2","spark.databricks.clusterUsageTags.clusterState":"Pending","spark.databricks.clusterUsageTags.clusterUnityCatalogMode":"None","spark.worker.aioaLazyConfig.dbfsReadinessCheckClientClass":"com.databricks.backend.daemon.driver.NephosDbfsReadinessCheckClient","spark.databricks.clusterUsageTags.clusterTargetWorkers":"4","spark.databricks.passthrough.oauth.refresher.impl":"*********(redacted)","spark.repl.class.outputDir":"/local_disk0/tmp/repl/spark-5706921391408253054-afd4d8ba-1528-4503-91ed-cd2eab7d25a4","spark.databricks.driverNfs.pathSuffix":".ephemeral_nfs","spark.databricks.clusterUsageTags.workerEnvironmentId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.app.name":"AirlineDelayLR_gz","spark.databricks.credential.redactor":"*********(redacted)","spark.hadoop.fs.abfss.impl":"shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","spark.sql.hive.metastore.jars":"/databricks/databricks-hive/*","spark.hadoop.hive.warehouse.subdir.inherit.perms":"false","spark.sql.parquet.cacheMetadata":"true","spark.hadoop.fs.cpfs-abfss.impl":"*********(redacted)","spark.hadoop.hive.server2.thrift.http.cookie.auth.enabled":"false","spark.hadoop.fs.s3a.multipart.size":"10485760","spark.storage.blockManagerTimeoutIntervalMs":"300000","spark.databricks.service.dbutils.repl.backend":"com.databricks.dbconnect.ReplDBUtils","spark.hadoop.fs.s3a.connection.maximum":"200","spark.databricks.clusterUsageTags.numPerGlobalInitScriptsV2":"0","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.r.numRBackendThreads":"1","spark.memory.offHeap.enabled":"false","spark.hadoop.parquet.block.size.row.check.min":"10","spark.databricks.clusterUsageTags.clusterEbsVolumeCount":"3","spark.databricks.secret.sparkConf.keys.toRedact":"*********(redacted)","spark.scheduler.mode":"FAIR","spark.storage.memoryFraction":"0.5","spark.databricks.repl.enableClassFileCleanup":"true","spark.databricks.clusterUsageTags.clusterId":"0304-221748-fngq99ev","spark.driver.memory":"1g","spark.databricks.clusterSource":"JOB","spark.databricks.passthrough.adls.tokenProviderClassName":"*********(redacted)","spark.databricks.acl.scim.client":"com.databricks.spark.sql.acl.client.DriverToWebappScimClient","spark.sql.hive.convertCTAS":"true","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.throwsException":"true","spark.databricks.clusterUsageTags.clusterMetastoreAccessType":"RDS_DIRECT","spark.databricks.overrideDefaultCommitProtocol":"org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol","spark.databricks.clusterUsageTags.isIMv2Enabled":"true","spark.files.fetchFailure.unRegisterOutputOnHost":"true","spark.logConf":"true","spark.hadoop.parquet.block.size.row.check.max":"10","spark.databricks.clusterUsageTags.enableGlueCatalogCredentialPassthrough":"*********(redacted)","spark.databricks.clusterUsageTags.clusterCreator":"JobLauncher","spark.databricks.tahoe.logStore.class":"com.databricks.tahoe.store.DelegatingLogStore","spark.databricks.workspace.multipleResults.enabled":"true","spark.executor.tempDirectory":"/local_disk0/tmp","spark.databricks.clusterUsageTags.clusterSpotBidPricePercent":"100","spark.databricks.sparkContextId":"5706921391408253054","spark.worker.cleanup.enabled":"false","spark.jars.packages":"org.apache.hadoop:hadoop-aws:3.2.1","spark.databricks.driverNodeTypeId":"m5.xlarge","spark.hadoop.fs.mcfs-s3n.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsDoubleQuotes":"false","spark.hadoop.hive.server2.idle.operation.timeout":"7200000","spark.speculation.multiplier":"3","spark.task.reaper.enabled":"true","spark.databricks.driverNfs.enabled":"true","spark.databricks.workspace.matplotlibInline.enabled":"true","spark.databricks.clusterUsageTags.containerZoneId":"us-east-1b","spark.hadoop.hive.server2.session.check.interval":"60000","spark.databricks.clusterUsageTags.driverContainerPrivateIp":"10.77.243.103","spark.sql.streaming.checkpointFileManagerClass":"com.databricks.spark.sql.streaming.DatabricksCheckpointFileManager","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.clusterUsageTags.clusterEbsVolumeType":"GENERAL_PURPOSE_SSD","spark.sql.streaming.stopTimeout":"15s","spark.app.startTime":"1646432441343","spark.executor.id":"driver","spark.databricks.job.runId":"31144","spark.hadoop.databricks.s3commit.client.sslTrustAll":"false","spark.hadoop.hive.server2.idle.session.timeout":"900000","spark.hadoop.fs.cpfs-abfss.impl.disable.cache":"true","spark.hadoop.fs.mcfs-s3a.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.clusterUsageTags.sparkEnvVarContainsNewline":"false","spark.databricks.clusterUsageTags.clusterPinned":"false","spark.hadoop.fs.fcfs-s3n.impl":"*********(redacted)","spark.hadoop.fs.mcfs-gs.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.hadoop.hive.server2.thrift.http.port":"10000","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.clusterUsageTags.instanceWorkerEnvId":"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313","spark.databricks.io.directoryCommit.enableLogicalDelete":"false","spark.databricks.clusterUsageTags.enableJobsAutostart":"true","spark.databricks.clusterUsageTags.clusterLogDestination":"s3://job-bank-temp/databricks/9.1.x-scala2.12/AirlineDelayLR_gz/arianna","spark.hadoop.parquet.page.verify-checksum.enabled":"true","spark.databricks.clusterUsageTags.clusterResourceClass":"default","spark.shuffle.service.port":"4048","spark.databricks.passthrough.glue.executorServiceFactoryClassName":"*********(redacted)","spark.hadoop.fs.cpfs-s3n.impl":"*********(redacted)","spark.databricks.acl.client":"com.databricks.spark.sql.acl.client.SparkSqlAclClient","spark.sql.legacy.createHiveTableByDefault":"false","spark.databricks.clusterUsageTags.enableJdbcAutoStart":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsDollarSign":"false","spark.databricks.clusterUsageTags.clusterStateMessage":"Starting Spark","spark.hadoop.spark.sql.parquet.output.committer.class":"org.apache.spark.sql.parquet.DirectParquetOutputCommitter","spark.databricks.preemption.enabled":"true","spark.master":"spark://10.77.243.103:7077","spark.databricks.tahoe.logStore.aws.class":"com.databricks.tahoe.store.MultiClusterLogStore","spark.hadoop.fs.adl.impl.disable.cache":"true","spark.rpc.message.maxSize":"256","spark.hadoop.fs.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem","spark.task.reaper.killTimeout":"60s","spark.hadoop.hive.server2.enable.doAs":"false","spark.hadoop.fs.fcfs-s3n.impl.disable.cache":"true","spark.hadoop.fs.s3n.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.databricks.clusterUsageTags.enableLocalDiskEncryption":"false","spark.databricks.clusterUsageTags.numPerClusterInitScriptsV2":"0","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.hadoop.fs.azure.skip.metrics":"true","spark.databricks.clusterUsageTags.driverContainerId":"8a041ac64b95432ab1643fe44cf52169","spark.files.overwrite":"true","spark.databricks.clusterUsageTags.sparkEnvVarContainsSingleQuotes":"false","spark.hadoop.fs.s3a.threads.max":"136","spark.databricks.clusterUsageTags.clusterNumSshKeys":"0","spark.sparkr.use.daemon":"false","spark.hadoop.fs.mcfs-abfss.impl.disable.cache":"true","spark.streaming.driver.writeAheadLog.allowBatching":"true","spark.databricks.clusterUsageTags.clusterScalingType":"fixed_size","spark.databricks.workerNodeTypeId":"m5.4xlarge","spark.hadoop.fs.fcfs-s3.impl.disable.cache":"true","spark.sql.sources.commitProtocolClass":"com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol","spark.databricks.clusterUsageTags.clusterWorkers":"4","spark.sql.hive.metastore.sharedPrefixes":"org.mariadb.jdbc,com.mysql.jdbc,org.postgresql,com.microsoft.sqlserver,microsoft.sql.DateTimeOffset,microsoft.sql.Types,com.databricks,com.codahale,com.fasterxml.jackson,shaded.databricks","spark.hadoop.fs.AbstractFileSystem.gs.impl":"shaded.databricks.V2_1_4.com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","spark.executor.memory":"47016m","spark.hadoop.databricks.fs.perfMetrics.enable":"true","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.clusterUsageTags.dataPlaneRegion":"us-east-1","spark.home":"/databricks/spark","spark.hadoop.fs.cpfs-s3a.impl":"*********(redacted)","spark.databricks.tahoe.logStore.azure.class":"com.databricks.tahoe.store.AzureLogStore","spark.hadoop.parquet.page.size.check.estimate":"false","spark.hadoop.fs.gs.impl.disable.cache":"true","spark.sql.execution.id":"9","spark.sql.catalogImplementation":"hive","spark.hadoop.fs.cpfs-adl.impl":"*********(redacted)","spark.databricks.passthrough.s3a.tokenProviderClassName":"*********(redacted)","spark.databricks.clusterUsageTags.isSingleUserCluster":"false","spark.databricks.cloudfetch.requesterClassName":"*********(redacted)","spark.databricks.clusterUsageTags.sparkEnvVarContainsEscape":"false","spark.databricks.clusterUsageTags.attribute_tag_dust_suite":"","spark.hadoop.hive.server2.keystore.path":"/databricks/keys/jetty-ssl-driver-keystore.jks","spark.hadoop.fs.wasb.impl.disable.cache":"true","spark.hadoop.hive.server2.transport.mode":"http","spark.hadoop.spark.databricks.io.parquet.verifyChecksumOnWrite.enabled":"true","spark.driver.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableSqlAclsOnly":"false","spark.databricks.clusterUsageTags.clusterNodeType":"m5.4xlarge","spark.databricks.clusterUsageTags.instanceProfileArn":"arn:aws:iam::471881062455:instance-profile/databricks-workspace-stack-access-data-buckets","spark.databricks.clusterUsageTags.containerType":"LXC","spark.databricks.managedCatalog.adls.gen2.tokenProviderClassName":"*********(redacted)","spark.files.useFetchCache":"false","spark.sql.allowMultipleContexts":"false","spark.databricks.clusterUsageTags.enableDfAcls":"false","spark.databricks.clusterUsageTags.clusterPythonVersion":"3","spark.databricks.cloudfetch.hasRegionSupport":"true","spark.databricks.job.type":"python","spark.hadoop.fs.s3a.impl":"shaded.databricks.org.apache.hadoop.fs.s3a.S3AFileSystem","spark.shuffle.memoryFraction":"0.2","spark.hadoop.fs.mcfs-s3.impl":"com.databricks.sql.acl.fs.ManagedCatalogFileSystem","spark.sql.sources.default":"delta","spark.databricks.clusterUsageTags.attribute_tag_dust_maintainer":"","spark.databricks.clusterUsageTags.clusterAvailability":"SPOT_WITH_FALLBACK","spark.sql.parquet.compression.codec":"snappy","spark.hadoop.mapred.output.committer.class":"com.databricks.backend.daemon.data.client.DirectOutputCommitter","spark.hadoop.fs.s3a.fast.upload.active.blocks":"32","spark.databricks.token":"*********(redacted)","spark.hadoop.spark.sql.sources.outputCommitterClass":"com.databricks.backend.daemon.data.client.MapReduceDirectOutputCommitter","spark.hadoop.fs.cpfs-adl.impl.disable.cache":"true","spark.app.id":"app-20220304222044-0000","spark.databricks.clusterUsageTags.clusterOwnerUserId":"6878104766663444","spark.hadoop.fs.s3a.fast.upload":"true","spark.databricks.clusterUsageTags.clusterLogDeliveryEnabled":"true","spark.databricks.clusterUsageTags.driverNodeType":"m5.xlarge","spark.speculation.quantile":"0.9","spark.databricks.cloudProvider":"AWS","spark.akka.frameSize":"256","spark.hadoop.parquet.page.write-checksum.enabled":"true","spark.eventLog.rolling.enabled":"false","spark.databricks.clusterUsageTags.clusterAllTags":"[{\"key\":\"Vendor\",\"value\":\"Databricks\"},{\"key\":\"Creator\",\"value\":\"arianna@synccomputing.com\"},{\"key\":\"ClusterName\",\"value\":\"job-703-run-31144\"},{\"key\":\"ClusterId\",\"value\":\"0304-221748-fngq99ev\"},{\"key\":\"JobId\",\"value\":\"703\"},{\"key\":\"RunName\",\"value\":\"AirlineDelayLR_gz\"},{\"key\":\"Name\",\"value\":\"workerenv-656201176161048-20de29ca-1c16-42b0-a91e-4f6a30f57313-worker\"}]","spark.sql.hive.metastore.version":"0.13.0"}} +{"Event":"SparkListenerTaskStart","Stage ID":32,"Stage Attempt ID":0,"Task Info":{"Task ID":101,"Index":0,"Attempt":0,"Launch Time":1646432591778,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":32,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":101,"Index":0,"Attempt":0,"Launch Time":1646432591778,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432591873,"Failed":false,"Killed":false,"Accumulables":[{"ID":942,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":945,"Name":"local bytes read","Update":"7603473","Value":"7603473","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":946,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":947,"Name":"records read","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1268,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1269,"Name":"time to read broadcast","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1270,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1271,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1272,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1273,"Name":"time to read broadcast","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1275,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1276,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4723681,"Value":4723681,"Internal":true,"Count Failed Values":true},{"ID":1277,"Name":"internal.metrics.executorRunTime","Update":72,"Value":72,"Internal":true,"Count Failed Values":true},{"ID":1278,"Name":"internal.metrics.executorCpuTime","Update":58754610,"Value":58754610,"Internal":true,"Count Failed Values":true},{"ID":1279,"Name":"internal.metrics.resultSize","Update":2044,"Value":2044,"Internal":true,"Count Failed Values":true},{"ID":1284,"Name":"internal.metrics.peakExecutionMemory","Update":201360384,"Value":201360384,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4723681,"Executor Run Time":72,"Executor CPU Time":58754610,"Peak Execution Memory":201360384,"Result Size":2044,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":32,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":72,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2278\",\"name\":\"WholeStageCodegen\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_2261\",\"name\":\"CustomShuffleReader\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[31],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:354)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432591763,"Completion Time":1646432591874,"Accumulables":[{"ID":942,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":945,"Name":"local bytes read","Value":"7603473","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":946,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":947,"Name":"records read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1268,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1269,"Name":"time to read broadcast","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1270,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1271,"Name":"time to read broadcast","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1272,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1273,"Name":"time to read broadcast","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1275,"Name":"internal.metrics.executorDeserializeTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":1276,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4723681,"Internal":true,"Count Failed Values":true},{"ID":1277,"Name":"internal.metrics.executorRunTime","Value":72,"Internal":true,"Count Failed Values":true},{"ID":1278,"Name":"internal.metrics.executorCpuTime","Value":58754610,"Internal":true,"Count Failed Values":true},{"ID":1279,"Name":"internal.metrics.resultSize","Value":2044,"Internal":true,"Count Failed Values":true},{"ID":1284,"Name":"internal.metrics.peakExecutionMemory","Value":201360384,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":18,"Completion Time":1646432591876,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":9,"time":1646432591915,"errorMessage":""} +{"Event":"SparkListenerJobStart","Job ID":19,"Submission Time":1646432592810,"Stage Infos":[{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":10,"RDD Info":[{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2826\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2825\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2371\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2371\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[33],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_2826\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":10,"RDD Info":[{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2826\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2825\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2371\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2371\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432592812,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_2826\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":102,"Index":0,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":103,"Index":1,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":104,"Index":2,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":105,"Index":3,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":106,"Index":4,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":107,"Index":5,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":108,"Index":6,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":109,"Index":7,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":110,"Index":8,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":111,"Index":9,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerJobStart","Job ID":20,"Submission Time":1646432592890,"Stage Infos":[{"Stage ID":34,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":20,"RDD Info":[{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2846\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_2842\",\"name\":\"Union\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[79,82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":80,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2841\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2845\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2838\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[34],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_2846\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":34,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":20,"RDD Info":[{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2846\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_2842\",\"name\":\"Union\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[79,82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":80,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2841\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2845\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2838\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432592892,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_2846\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":112,"Index":0,"Attempt":0,"Launch Time":1646432592911,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":113,"Index":1,"Attempt":0,"Launch Time":1646432592911,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":114,"Index":2,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":115,"Index":3,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":116,"Index":4,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":117,"Index":5,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":118,"Index":6,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":119,"Index":7,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":120,"Index":8,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":121,"Index":9,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":122,"Index":10,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":123,"Index":11,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":124,"Index":12,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":125,"Index":13,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":126,"Index":14,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":127,"Index":15,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":128,"Index":16,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":129,"Index":17,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":130,"Index":18,"Attempt":0,"Launch Time":1646432592914,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":34,"Stage Attempt ID":0,"Task Info":{"Task ID":131,"Index":19,"Attempt":0,"Launch Time":1646432592914,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerJobStart","Job ID":21,"Submission Time":1646432592942,"Stage Infos":[{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":10,"RDD Info":[{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2922\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2921\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[35],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_2922\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":10,"RDD Info":[{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2922\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2921\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432592958,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_2922\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":132,"Index":0,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":133,"Index":1,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":134,"Index":2,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":135,"Index":3,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":136,"Index":4,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":137,"Index":5,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":138,"Index":6,"Attempt":0,"Launch Time":1646432592988,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":139,"Index":7,"Attempt":0,"Launch Time":1646432592988,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":140,"Index":8,"Attempt":0,"Launch Time":1646432592988,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":141,"Index":9,"Attempt":0,"Launch Time":1646432592988,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":139,"Index":7,"Attempt":0,"Launch Time":1646432592988,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432605060,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"12033","Value":"12033","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"12040","Value":"12040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"5819811","Value":"5819811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1329","Value":"1329","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"336","Value":"336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"952","Value":"952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"3907453","Value":"3907453","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2516238,"Value":2516238,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":12045,"Value":12045,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":11927579260,"Value":11927579260,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2132,"Value":2132,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":62,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":952,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":3907453,"Value":3907453,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":180693843,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":5819811,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1329131301,"Value":1329131301,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":180693843,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":659,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2516238,"Executor Run Time":12045,"Executor CPU Time":11927579260,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":3907453,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":141,"Index":9,"Attempt":0,"Launch Time":1646432592988,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432605319,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"12","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"12289","Value":"24322","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"12301","Value":"24341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"5617658","Value":"11437469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1220","Value":"2549","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"356626668","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"175932825","Value":"356626668","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"288","Value":"624","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"816","Value":"1768","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"12","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"3702742","Value":"7610195","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2457291,"Value":4973529,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":12304,"Value":24349,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":12117562761,"Value":24045142021,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2133,"Value":4265,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":70,"Value":132,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":1768,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":3702742,"Value":7610195,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":356626668,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":11437469,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1220679766,"Value":2549811067,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":356626668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":127751616,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":66,"MinorGCTime":585,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2457291,"Executor Run Time":12304,"Executor CPU Time":12117562761,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":70,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":3702742,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":138,"Index":6,"Attempt":0,"Launch Time":1646432592988,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432605576,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"14","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"12554","Value":"36876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"12561","Value":"36902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"5819079","Value":"17256548","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1321","Value":"3870","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"538863533","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"182236865","Value":"538863533","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"336","Value":"960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"952","Value":"2720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"14","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"3880940","Value":"11491135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2610903,"Value":7584432,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":12565,"Value":36914,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":12421970434,"Value":36467112455,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2133,"Value":6398,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":84,"Value":216,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":2720,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":3880940,"Value":11491135,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":538863533,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":17256548,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1321199010,"Value":3871010077,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":538863533,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129470216,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":576,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2610903,"Executor Run Time":12565,"Executor CPU Time":12421970434,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":84,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":3880940,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":130,"Index":18,"Attempt":0,"Launch Time":1646432592914,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432605728,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"320","Value":"320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"12752","Value":"12752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"12770","Value":"12770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"12761","Value":"12761","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"5674621","Value":"5674621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1390","Value":"1390","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"178217644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"178217644","Value":"178217644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7680","Value":"7680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"12025","Value":"12025","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"320","Value":"320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"14039462","Value":"14039462","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4365139,"Value":4365139,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":12785,"Value":12785,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":12657616437,"Value":12657616437,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2562,"Value":2562,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":62,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12025,"Value":12025,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":320,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":14039462,"Value":14039462,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":178217644,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":5674621,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1390945771,"Value":1390945771,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":178217644,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":659,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4365139,"Executor Run Time":12785,"Executor CPU Time":12657616437,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12025,"Shuffle Write Time":14039462,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":117,"Index":5,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432606087,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"312","Value":"632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"13125","Value":"25877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"13137","Value":"25907","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"13129","Value":"13129","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"6096762","Value":"6096762","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1313","Value":"1313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"183939508","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"183939508","Value":"183939508","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7488","Value":"15168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11723","Value":"23748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"312","Value":"632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12731601","Value":"26771063","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4020836,"Value":8385975,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":13150,"Value":25935,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":12974010425,"Value":25631626862,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":5125,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":84,"Value":146,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11723,"Value":23748,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":312,"Value":632,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12731601,"Value":26771063,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":362157152,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":11771383,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1313532169,"Value":2704477940,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":362157152,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129470216,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":576,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4020836,"Executor Run Time":13150,"Executor CPU Time":12974010425,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":84,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11723,"Shuffle Write Time":12731601,"Shuffle Records Written":312},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":131,"Index":19,"Attempt":0,"Launch Time":1646432592914,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432606117,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"310","Value":"942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"13151","Value":"39028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"3007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"13166","Value":"39073","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"13159","Value":"25920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"5617658","Value":"11292279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1244","Value":"2634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"175932825","Value":"354150469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7440","Value":"22608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11887","Value":"35635","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"310","Value":"942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12620800","Value":"39391863","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4150753,"Value":12536728,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":13179,"Value":39114,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":13037404802,"Value":38669031664,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":7688,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":70,"Value":216,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11887,"Value":35635,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":310,"Value":942,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12620800,"Value":39391863,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":538089977,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":17389041,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1244763096,"Value":3949241036,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":538089977,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":127751616,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":66,"MinorGCTime":585,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4150753,"Executor Run Time":13179,"Executor CPU Time":13037404802,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":70,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11887,"Shuffle Write Time":12620800,"Shuffle Records Written":310},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":137,"Index":5,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432606201,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"15","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"13174","Value":"50050","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"4000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"13184","Value":"50086","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"6096762","Value":"23353310","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1293","Value":"5163","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"722803041","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"183939508","Value":"722803041","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"360","Value":"1320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"1020","Value":"3740","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"15","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"3725068","Value":"15216203","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2211161,"Value":9795593,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":13188,"Value":50102,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":13031261812,"Value":49498374267,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2133,"Value":8531,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":70,"Value":286,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1020,"Value":3740,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":15,"Value":55,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":3725068,"Value":15216203,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":722803041,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":23353310,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1293722433,"Value":5164732510,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":722803041,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":127751616,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":66,"MinorGCTime":585,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2211161,"Executor Run Time":13188,"Executor CPU Time":13031261812,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":70,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1020,"Shuffle Write Time":3725068,"Shuffle Records Written":15},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":126,"Index":14,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432606588,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"301","Value":"1243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"13610","Value":"52638","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"4007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"13628","Value":"52701","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"13619","Value":"39539","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"6066650","Value":"17358929","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1436","Value":"4070","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"539186349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"185035880","Value":"539186349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7224","Value":"29832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11463","Value":"47098","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"301","Value":"1243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"15143866","Value":"54535729","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4615023,"Value":17151751,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":13645,"Value":52759,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":13516440409,"Value":52185472073,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2562,"Value":10250,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":278,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11463,"Value":47098,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":301,"Value":1243,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":15143866,"Value":54535729,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":723125857,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":23455691,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1436866458,"Value":5386107494,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":723125857,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":659,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4615023,"Executor Run Time":13645,"Executor CPU Time":13516440409,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11463,"Shuffle Write Time":15143866,"Shuffle Records Written":301},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":115,"Index":3,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432607096,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"296","Value":"1539","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"14132","Value":"66770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"5014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"14147","Value":"66848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"14140","Value":"27269","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"6429338","Value":"12526100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1360","Value":"2673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"379013823","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"195074315","Value":"379013823","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7104","Value":"36936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11482","Value":"58580","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"296","Value":"1539","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"11739136","Value":"66274865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4056113,"Value":21207864,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":14159,"Value":66918,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":14032465894,"Value":66217937967,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":12813,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":70,"Value":348,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11482,"Value":58580,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":1539,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":11739136,"Value":66274865,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":918200172,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":29885029,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1360067026,"Value":6746174520,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":918200172,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":127751616,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":66,"MinorGCTime":585,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4056113,"Executor Run Time":14159,"Executor CPU Time":14032465894,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":70,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11482,"Shuffle Write Time":11739136,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":134,"Index":2,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432607123,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"18","Value":"73","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"14101","Value":"64151","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"14108","Value":"64194","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"6450117","Value":"29803427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1456","Value":"6619","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"919617341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"196814300","Value":"919617341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"432","Value":"1752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"1164","Value":"4904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"18","Value":"73","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"3920725","Value":"19136928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2693201,"Value":12488794,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":14113,"Value":64215,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":13968831541,"Value":63467205808,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2133,"Value":10664,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":84,"Value":370,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1164,"Value":4904,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":73,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":3920725,"Value":19136928,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":919617341,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":29803427,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1456407904,"Value":6621140414,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":919617341,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129493376,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":634,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2693201,"Executor Run Time":14113,"Executor CPU Time":13968831541,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":84,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1164,"Shuffle Write Time":3920725,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":136,"Index":4,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432607216,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"16","Value":"89","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"14191","Value":"78342","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"14198","Value":"78392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"6066650","Value":"35870077","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1351","Value":"7970","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1104653221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"185035880","Value":"1104653221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"384","Value":"2136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"1028","Value":"5932","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"16","Value":"89","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"3957935","Value":"23094863","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2589546,"Value":15078340,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":14203,"Value":78418,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":13692777844,"Value":77159983652,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2133,"Value":12797,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":423,"Value":793,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1028,"Value":5932,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":89,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":3957935,"Value":23094863,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1104653221,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":35870077,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1351384053,"Value":7972524467,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1104653221,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131144960,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":1141,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2589546,"Executor Run Time":14203,"Executor CPU Time":13692777844,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":423,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1028,"Shuffle Write Time":3957935,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":114,"Index":2,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432607671,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"305","Value":"1844","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"14695","Value":"81465","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"14712","Value":"81560","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"14703","Value":"41972","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"6450117","Value":"18976217","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1509","Value":"4182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"575828123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"196814300","Value":"575828123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7320","Value":"44256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11668","Value":"70248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"305","Value":"1844","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"13881848","Value":"80156713","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4551095,"Value":25758959,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":14728,"Value":81646,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":14559490232,"Value":80777428199,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2562,"Value":15375,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":74,"Value":422,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":70248,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":1844,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":13881848,"Value":80156713,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1115014472,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":36335146,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1509798013,"Value":8255972533,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1115014472,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":659,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4551095,"Executor Run Time":14728,"Executor CPU Time":14559490232,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":74,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":13881848,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":113,"Index":1,"Attempt":0,"Launch Time":1646432592911,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432607686,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"320","Value":"2164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"14724","Value":"96189","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"14736","Value":"96296","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"14729","Value":"56701","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"6369482","Value":"25345699","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1524","Value":"5706","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"773466657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"197638534","Value":"773466657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7680","Value":"51936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11962","Value":"82210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"320","Value":"2164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12649378","Value":"92806091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3919754,"Value":29678713,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":14749,"Value":96395,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":14565393806,"Value":95342822005,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":17938,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":84,"Value":506,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11962,"Value":82210,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":2164,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12649378,"Value":92806091,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1312653006,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":42704628,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1524909875,"Value":9780882408,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1312653006,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129493376,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":634,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3919754,"Executor Run Time":14749,"Executor CPU Time":14565393806,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":84,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11962,"Shuffle Write Time":12649378,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":116,"Index":4,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432607766,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"299","Value":"2463","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"14799","Value":"110988","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"14816","Value":"111112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"14809","Value":"71510","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"6066650","Value":"31412349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1701","Value":"7407","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"958502537","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"185035880","Value":"958502537","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7176","Value":"59112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11443","Value":"93653","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"299","Value":"2463","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12622513","Value":"105428604","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4037314,"Value":33716027,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":14831,"Value":111226,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":14079055810,"Value":109421877815,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":20501,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":423,"Value":929,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11443,"Value":93653,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":299,"Value":2463,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12622513,"Value":105428604,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1497688886,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":48771278,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1701125842,"Value":11482008250,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1497688886,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131144960,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":1141,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4037314,"Executor Run Time":14831,"Executor CPU Time":14079055810,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":423,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11443,"Shuffle Write Time":12622513,"Shuffle Records Written":299},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":140,"Index":8,"Attempt":0,"Launch Time":1646432592988,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432607812,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"12","Value":"101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"14787","Value":"93129","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"14794","Value":"93186","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"5674621","Value":"41544698","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1502","Value":"9472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1282870865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"178217644","Value":"1282870865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"288","Value":"2424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"816","Value":"6748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"12","Value":"101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"3419438","Value":"26514301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2682690,"Value":17761030,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":14798,"Value":93216,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":14311633513,"Value":91471617165,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2133,"Value":14930,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":423,"Value":1216,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":6748,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":101,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":3419438,"Value":26514301,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":1282870865,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":41544698,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1502476679,"Value":9475001146,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":1282870865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131144960,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":1141,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2682690,"Executor Run Time":14798,"Executor CPU Time":14311633513,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":423,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":3419438,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":121,"Index":9,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432608032,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"313","Value":"2776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"15060","Value":"126048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"15083","Value":"126195","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"15064","Value":"86574","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"5617658","Value":"37030007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1446","Value":"8853","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"1134435362","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"175932825","Value":"1134435362","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7512","Value":"66624","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11907","Value":"105560","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"313","Value":"2776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"13678636","Value":"119107240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3358008,"Value":37074035,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":15096,"Value":126322,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":14951679618,"Value":124373557433,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":23064,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":84,"Value":1013,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11907,"Value":105560,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":2776,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":13678636,"Value":119107240,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":1673621711,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":54388936,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1446792543,"Value":12928800793,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":1673621711,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129493376,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":634,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3358008,"Executor Run Time":15096,"Executor CPU Time":14951679618,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":84,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11907,"Shuffle Write Time":13678636,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":122,"Index":10,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432608726,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"358","Value":"3134","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"15750","Value":"141798","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"15767","Value":"141962","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"15758","Value":"55297","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"7213446","Value":"24572375","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1714","Value":"5784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"767792969","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"228606620","Value":"767792969","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"8592","Value":"75216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"12847","Value":"118407","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"358","Value":"3134","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"14684397","Value":"133791637","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4446396,"Value":41520431,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":15783,"Value":142105,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":15645989288,"Value":140019546721,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2562,"Value":25626,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":74,"Value":1087,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12847,"Value":118407,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":3134,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":14684397,"Value":133791637,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1902228331,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":61602382,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1714225269,"Value":14643026062,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1902228331,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":659,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4446396,"Executor Run Time":15783,"Executor CPU Time":15645989288,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":74,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12847,"Shuffle Write Time":14684397,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":129,"Index":17,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432609282,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"324","Value":"3458","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"741081088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"16320","Value":"158118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"11021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"16332","Value":"158294","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"16324","Value":"71621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"5819811","Value":"30392186","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1550","Value":"7334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"948486812","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"180693843","Value":"948486812","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7776","Value":"82992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"12053","Value":"130460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"324","Value":"3458","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12446493","Value":"146238130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3921586,"Value":45442017,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":16346,"Value":158451,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":16152247642,"Value":156171794363,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":28189,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":96,"Value":1183,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":741081088,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12053,"Value":130460,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":324,"Value":3458,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12446493,"Value":146238130,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":2082922174,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":67422193,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1550639497,"Value":16193665559,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":2082922174,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129493376,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":634,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3921586,"Executor Run Time":16346,"Executor CPU Time":16152247642,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":96,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12053,"Shuffle Write Time":12446493,"Shuffle Records Written":324},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":120,"Index":8,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432609385,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"320","Value":"3778","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"808452096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"16420","Value":"174538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"12021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"16436","Value":"174730","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"16429","Value":"103003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"5674621","Value":"42704628","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1471","Value":"10324","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1312653006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"178217644","Value":"1312653006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7680","Value":"90672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"12024","Value":"142484","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"320","Value":"3778","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12850991","Value":"159089121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4057268,"Value":49499285,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":16451,"Value":174902,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":15921221402,"Value":172093015765,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":30752,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":423,"Value":1606,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":808452096,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12024,"Value":142484,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":3778,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12850991,"Value":159089121,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":2261139818,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":73096814,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1471427700,"Value":17665093259,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":2261139818,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131144960,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":1141,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4057268,"Executor Run Time":16451,"Executor CPU Time":15921221402,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":423,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12024,"Shuffle Write Time":12850991,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":133,"Index":1,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432609587,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"16","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"16562","Value":"109691","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"16569","Value":"109755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"6369482","Value":"47914180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1525","Value":"10997","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1480509399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"197638534","Value":"1480509399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"384","Value":"2808","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"1088","Value":"7836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"16","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"3821881","Value":"30336182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2202992,"Value":19964022,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":16573,"Value":109789,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":16404946899,"Value":107876564064,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2133,"Value":17063,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":79,"Value":1295,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1088,"Value":7836,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":117,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":3821881,"Value":30336182,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1480509399,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":47914180,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1525236232,"Value":11000237378,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1480509399,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":127751616,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":66,"MinorGCTime":585,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2202992,"Executor Run Time":16573,"Executor CPU Time":16404946899,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":79,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1088,"Shuffle Write Time":3821881,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":127,"Index":15,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432609594,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"313","Value":"4091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"875823104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"16629","Value":"191167","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"13021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"16644","Value":"191374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"16637","Value":"88258","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"6096762","Value":"36488948","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1437","Value":"8771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1132426320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"183939508","Value":"1132426320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7512","Value":"98184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11789","Value":"154273","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"313","Value":"4091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12898689","Value":"171987810","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4134403,"Value":53633688,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":16657,"Value":191559,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":16487493853,"Value":188580509618,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":33315,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":79,"Value":1685,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":875823104,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11789,"Value":154273,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":4091,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12898689,"Value":171987810,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":2445079326,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":79193576,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1437417752,"Value":19102511011,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":2445079326,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":127751616,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":66,"MinorGCTime":585,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4134403,"Executor Run Time":16657,"Executor CPU Time":16487493853,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":79,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11789,"Shuffle Write Time":12898689,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":128,"Index":16,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432609602,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"322","Value":"4413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"943194112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"16637","Value":"207804","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"14021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"16653","Value":"208027","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"16646","Value":"104904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"5819079","Value":"42308027","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1536","Value":"10307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"1314663185","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"182236865","Value":"1314663185","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7728","Value":"105912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"12099","Value":"166372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"322","Value":"4413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12457362","Value":"184445172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4054787,"Value":57688475,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":16668,"Value":208227,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":16126033508,"Value":204706543126,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":35878,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":423,"Value":2108,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":943194112,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12099,"Value":166372,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":4413,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12457362,"Value":184445172,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":2627316191,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":85012655,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1536012312,"Value":20638523323,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":2627316191,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131144960,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":1141,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4054787,"Executor Run Time":16668,"Executor CPU Time":16126033508,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":423,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12099,"Shuffle Write Time":12457362,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":132,"Index":0,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432609798,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"18","Value":"135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"16772","Value":"126463","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"16780","Value":"126535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"7213446","Value":"55127626","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1682","Value":"12679","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1709116019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"228606620","Value":"1709116019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"432","Value":"3240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"1224","Value":"9060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"18","Value":"135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"3845543","Value":"34181725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2731289,"Value":22695311,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":16784,"Value":126573,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":16281663232,"Value":124158227296,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2133,"Value":19196,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":423,"Value":1718,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1224,"Value":9060,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":135,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":3845543,"Value":34181725,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1709116019,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":55127626,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1682189430,"Value":12682426808,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1709116019,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131144960,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":1141,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2731289,"Executor Run Time":16784,"Executor CPU Time":16281663232,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":423,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1224,"Shuffle Write Time":3845543,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":118,"Index":6,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432609869,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"322","Value":"4735","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"1010565120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"16895","Value":"224699","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"15021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"16912","Value":"224939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"16904","Value":"119907","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"5819079","Value":"48523707","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1702","Value":"12026","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"1494889871","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"182236865","Value":"1494889871","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7728","Value":"113640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"12103","Value":"178475","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"322","Value":"4735","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12791815","Value":"197236987","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":58,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4464530,"Value":62153005,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":16927,"Value":225154,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":16780651032,"Value":221487194158,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2562,"Value":38440,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":74,"Value":2182,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1010565120,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12103,"Value":178475,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":4735,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12791815,"Value":197236987,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":2809553056,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":90831734,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1702493641,"Value":22341016964,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":2809553056,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":659,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4464530,"Executor Run Time":16927,"Executor CPU Time":16780651032,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":74,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12103,"Shuffle Write Time":12791815,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":119,"Index":7,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432610019,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"325","Value":"5060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"1077936128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"17053","Value":"241752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"16028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"17069","Value":"242008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"17062","Value":"136969","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"5819811","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1683","Value":"13709","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"180693843","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7800","Value":"121440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"12061","Value":"190536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"325","Value":"5060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"12480950","Value":"209717937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":61,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3288521,"Value":65441526,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":17083,"Value":242237,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":16701959196,"Value":238189153354,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":41003,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":79,"Value":2261,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1077936128,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12061,"Value":190536,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":325,"Value":5060,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":12480950,"Value":209717937,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":2990246899,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":96651545,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1683884505,"Value":24024901469,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":2990246899,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":127751616,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":66,"MinorGCTime":585,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3288521,"Executor Run Time":17083,"Executor CPU Time":16701959196,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":79,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12061,"Shuffle Write Time":12480950,"Shuffle Records Written":325},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":125,"Index":13,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432610840,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"296","Value":"5356","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"1145307136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"17880","Value":"259632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"17035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"17892","Value":"259900","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"17885","Value":"122789","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"6429338","Value":"48737365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1602","Value":"11909","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1509737500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"195074315","Value":"1509737500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7104","Value":"128544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11481","Value":"202017","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"296","Value":"5356","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"11398174","Value":"221116111","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":64,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3836379,"Value":69277905,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":17904,"Value":260141,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":17692940076,"Value":255882093430,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":43566,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":96,"Value":2357,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1145307136,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11481,"Value":202017,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":5356,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":11398174,"Value":221116111,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":3185321214,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":103080883,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1602695284,"Value":25627596753,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":3185321214,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129493376,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":634,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3836379,"Executor Run Time":17904,"Executor CPU Time":17692940076,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":96,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11481,"Shuffle Write Time":11398174,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":135,"Index":3,"Attempt":0,"Launch Time":1646432592987,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432610903,"Failed":false,"Killed":false,"Accumulables":[{"ID":1392,"Name":"number of output rows","Update":"19","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Update":"17875","Value":"144338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Update":"17883","Value":"144418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Update":"6429338","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Update":"1778","Value":"14457","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Update":"195074315","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Update":"456","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Update":"1232","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Update":"19","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Update":"4094858","Value":"38276583","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2273364,"Value":24968675,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Update":17888,"Value":144461,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Update":17711701697,"Value":141869928993,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Update":2132,"Value":21328,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Update":74,"Value":1792,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1232,"Value":10292,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":19,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Update":4094858,"Value":38276583,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1778952528,"Value":14461379336,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":659,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2273364,"Executor Run Time":17888,"Executor CPU Time":17711701697,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":74,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1232,"Shuffle Write Time":4094858,"Shuffle Records Written":19},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":10,"RDD Info":[{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2922\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2921\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432592958,"Completion Time":1646432610904,"Accumulables":[{"ID":1392,"Name":"number of output rows","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"peak memory","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"time in aggregation build","Value":"144338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1396,"Name":"avg hash probe bucket list iters","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1428,"Name":"duration","Value":"144418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1429,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1433,"Name":"filesystem read time (sampled)","Value":"14457","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1434,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1435,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1437,"Name":"data size","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1445,"Name":"shuffle bytes written","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1446,"Name":"shuffle records written","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1447,"Name":"shuffle write time","Value":"38276583","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1448,"Name":"internal.metrics.executorDeserializeTime","Value":24,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.executorDeserializeCpuTime","Value":24968675,"Internal":true,"Count Failed Values":true},{"ID":1450,"Name":"internal.metrics.executorRunTime","Value":144461,"Internal":true,"Count Failed Values":true},{"ID":1451,"Name":"internal.metrics.executorCpuTime","Value":141869928993,"Internal":true,"Count Failed Values":true},{"ID":1452,"Name":"internal.metrics.resultSize","Value":21328,"Internal":true,"Count Failed Values":true},{"ID":1453,"Name":"internal.metrics.jvmGCTime","Value":1792,"Internal":true,"Count Failed Values":true},{"ID":1457,"Name":"internal.metrics.peakExecutionMemory","Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":1468,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":10292,"Internal":true,"Count Failed Values":true},{"ID":1469,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":154,"Internal":true,"Count Failed Values":true},{"ID":1470,"Name":"internal.metrics.shuffle.write.writeTime","Value":38276583,"Internal":true,"Count Failed Values":true},{"ID":1471,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1472,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1473,"Name":"internal.metrics.input.sampledTimeReadNano","Value":14461379336,"Internal":true,"Count Failed Values":true},{"ID":1474,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":21,"Completion Time":1646432610906,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":22,"Submission Time":1646432610994,"Stage Infos":[{"Stage ID":37,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":1,"RDD Info":[{"RDD ID":92,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3267\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_3261\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3266\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[36],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":36,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":10,"RDD Info":[{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2922\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2921\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[37,36],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_3267\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":37,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":1,"RDD Info":[{"RDD ID":92,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3267\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_3261\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3266\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[36],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432610995,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_3267\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":37,"Stage Attempt ID":0,"Task Info":{"Task ID":142,"Index":0,"Attempt":0,"Launch Time":1646432611043,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":37,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":142,"Index":0,"Attempt":0,"Launch Time":1646432611043,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432611087,"Failed":false,"Killed":false,"Accumulables":[{"ID":1438,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1439,"Name":"local blocks read","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1440,"Name":"remote bytes read","Update":"7224","Value":"7224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1442,"Name":"local bytes read","Update":"3068","Value":"3068","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1443,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1444,"Name":"records read","Update":"154","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1478,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1479,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1481,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1482,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1483,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1486,"Name":"data size","Update":"552","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1494,"Name":"shuffle bytes written","Update":"259","Value":"259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1495,"Name":"shuffle records written","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1496,"Name":"shuffle write time","Update":"301224","Value":"301224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1497,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1498,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2743331,"Value":2743331,"Internal":true,"Count Failed Values":true},{"ID":1499,"Name":"internal.metrics.executorRunTime","Update":29,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":1500,"Name":"internal.metrics.executorCpuTime","Update":24835402,"Value":24835402,"Internal":true,"Count Failed Values":true},{"ID":1501,"Name":"internal.metrics.resultSize","Update":2738,"Value":2738,"Internal":true,"Count Failed Values":true},{"ID":1506,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1510,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":1511,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":1512,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":7224,"Value":7224,"Internal":true,"Count Failed Values":true},{"ID":1513,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1514,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":3068,"Value":3068,"Internal":true,"Count Failed Values":true},{"ID":1515,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1516,"Name":"internal.metrics.shuffle.read.recordsRead","Update":154,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":1517,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":259,"Value":259,"Internal":true,"Count Failed Values":true},{"ID":1518,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":1519,"Name":"internal.metrics.shuffle.write.writeTime","Update":301224,"Value":301224,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2743331,"Executor Run Time":29,"Executor CPU Time":24835402,"Peak Execution Memory":67371008,"Result Size":2738,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":3,"Fetch Wait Time":0,"Remote Bytes Read":7224,"Remote Bytes Read To Disk":0,"Local Bytes Read":3068,"Total Records Read":154},"Shuffle Write Metrics":{"Shuffle Bytes Written":259,"Shuffle Write Time":301224,"Shuffle Records Written":23},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":37,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":1,"RDD Info":[{"RDD ID":92,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3267\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_3261\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3266\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[36],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432610995,"Completion Time":1646432611087,"Accumulables":[{"ID":1438,"Name":"remote blocks read","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1439,"Name":"local blocks read","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1440,"Name":"remote bytes read","Value":"7224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1442,"Name":"local bytes read","Value":"3068","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1443,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1444,"Name":"records read","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1478,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1479,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1481,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1482,"Name":"avg hash probe bucket list iters","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1483,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1486,"Name":"data size","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1494,"Name":"shuffle bytes written","Value":"259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1495,"Name":"shuffle records written","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1496,"Name":"shuffle write time","Value":"301224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1497,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1498,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2743331,"Internal":true,"Count Failed Values":true},{"ID":1499,"Name":"internal.metrics.executorRunTime","Value":29,"Internal":true,"Count Failed Values":true},{"ID":1500,"Name":"internal.metrics.executorCpuTime","Value":24835402,"Internal":true,"Count Failed Values":true},{"ID":1501,"Name":"internal.metrics.resultSize","Value":2738,"Internal":true,"Count Failed Values":true},{"ID":1506,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1510,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":7,"Internal":true,"Count Failed Values":true},{"ID":1511,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":3,"Internal":true,"Count Failed Values":true},{"ID":1512,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":7224,"Internal":true,"Count Failed Values":true},{"ID":1513,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1514,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":3068,"Internal":true,"Count Failed Values":true},{"ID":1515,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1516,"Name":"internal.metrics.shuffle.read.recordsRead","Value":154,"Internal":true,"Count Failed Values":true},{"ID":1517,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":259,"Internal":true,"Count Failed Values":true},{"ID":1518,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23,"Internal":true,"Count Failed Values":true},{"ID":1519,"Name":"internal.metrics.shuffle.write.writeTime","Value":301224,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":22,"Completion Time":1646432611093,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":23,"Submission Time":1646432611151,"Stage Infos":[{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":10,"RDD Info":[{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2922\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2396\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2921\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":39,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":1,"RDD Info":[{"RDD ID":92,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3267\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_3261\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3266\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[38],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":94,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3513\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[38,39,40],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_6\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId ada07415-a687-4ed6-994d-864762ba6741)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"ada07415-a687-4ed6-994d-864762ba6741","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":94,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3513\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432611151,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_6\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId ada07415-a687-4ed6-994d-864762ba6741)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"ada07415-a687-4ed6-994d-864762ba6741","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":40,"Stage Attempt ID":0,"Task Info":{"Task ID":143,"Index":0,"Attempt":0,"Launch Time":1646432611167,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":40,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":143,"Index":0,"Attempt":0,"Launch Time":1646432611167,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432611189,"Failed":false,"Killed":false,"Accumulables":[{"ID":1488,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1491,"Name":"local bytes read","Update":"259","Value":"259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1492,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1493,"Name":"records read","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1527,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1528,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1529,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1530,"Name":"cumulative time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1531,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1532,"Name":"num prefix comparisons","Update":"77","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1534,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1535,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1536,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1537,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2778383,"Value":2778383,"Internal":true,"Count Failed Values":true},{"ID":1538,"Name":"internal.metrics.executorRunTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":1539,"Name":"internal.metrics.executorCpuTime","Update":5590643,"Value":5590643,"Internal":true,"Count Failed Values":true},{"ID":1540,"Name":"internal.metrics.resultSize","Update":3792,"Value":3792,"Internal":true,"Count Failed Values":true},{"ID":1545,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1549,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1550,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1551,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1552,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1553,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":259,"Value":259,"Internal":true,"Count Failed Values":true},{"ID":1554,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1555,"Name":"internal.metrics.shuffle.read.recordsRead","Update":23,"Value":23,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2778383,"Executor Run Time":6,"Executor CPU Time":5590643,"Peak Execution Memory":67174400,"Result Size":3792,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":259,"Total Records Read":23},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":94,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3513\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432611151,"Completion Time":1646432611190,"Accumulables":[{"ID":1488,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1491,"Name":"local bytes read","Value":"259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1492,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1493,"Name":"records read","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1527,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1528,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1529,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1530,"Name":"cumulative time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1531,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1532,"Name":"num prefix comparisons","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1534,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1535,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1536,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1537,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2778383,"Internal":true,"Count Failed Values":true},{"ID":1538,"Name":"internal.metrics.executorRunTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":1539,"Name":"internal.metrics.executorCpuTime","Value":5590643,"Internal":true,"Count Failed Values":true},{"ID":1540,"Name":"internal.metrics.resultSize","Value":3792,"Internal":true,"Count Failed Values":true},{"ID":1545,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1549,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1550,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1551,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1552,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1553,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":259,"Internal":true,"Count Failed Values":true},{"ID":1554,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1555,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":23,"Completion Time":1646432611191,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":108,"Index":6,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432611273,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"18343","Value":"18343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1315","Value":"1315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"558631584","Value":"558631584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"77834569","Value":"77834569","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"180001432","Value":"180001432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2256456,"Value":2256456,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":18432,"Value":18432,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":18278365766,"Value":18278365766,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1703,"Value":1703,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":88,"Value":88,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":77834569,"Value":77834569,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819079,"Value":5819079,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":180001432,"Value":180001432,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":5819079,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1315618243,"Value":1315618243,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":127751616,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":66,"MinorGCTime":585,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2256456,"Executor Run Time":18432,"Executor CPU Time":18278365766,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":88,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":77834569,"Shuffle Write Time":180001432,"Shuffle Records Written":5819079},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":123,"Index":11,"Attempt":0,"Launch Time":1646432592912,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432611531,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"318","Value":"5674","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"1212678144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"18568","Value":"278200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"18035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"18582","Value":"278482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"18575","Value":"141364","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"6369482","Value":"55106847","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1586","Value":"13495","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1707376034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"197638534","Value":"1707376034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7632","Value":"136176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11947","Value":"213964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"318","Value":"5674","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"10758844","Value":"231874955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":67,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3495571,"Value":72773476,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":18594,"Value":278735,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":18413704300,"Value":274295797730,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":46129,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":88,"Value":2445,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1212678144,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11947,"Value":213964,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":318,"Value":5674,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":10758844,"Value":231874955,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":3382959748,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":109450365,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1586070171,"Value":27213666924,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":3382959748,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":127751616,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460803,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":66,"MinorGCTime":585,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3495571,"Executor Run Time":18594,"Executor CPU Time":18413704300,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":88,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11947,"Shuffle Write Time":10758844,"Shuffle Records Written":318},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":112,"Index":0,"Attempt":0,"Launch Time":1646432592911,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432611539,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"358","Value":"6032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"1280049152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"18572","Value":"296772","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"19035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"18590","Value":"297072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Update":"18581","Value":"155550","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Update":"1800","Value":"15509","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"8592","Value":"144768","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"12849","Value":"226813","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"358","Value":"6032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"13330255","Value":"245205210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":71,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3962463,"Value":76735939,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":18605,"Value":297340,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":18067335198,"Value":292363132928,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":48692,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":432,"Value":2877,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1280049152,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12849,"Value":226813,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":6032,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":13330255,"Value":245205210,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":3611566368,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":116663811,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1800390563,"Value":29014057487,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":3611566368,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131335688,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":74,"MinorGCTime":1360,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3962463,"Executor Run Time":18605,"Executor CPU Time":18067335198,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":432,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12849,"Shuffle Write Time":13330255,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":34,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":124,"Index":12,"Attempt":0,"Launch Time":1646432592913,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432611703,"Failed":false,"Killed":false,"Accumulables":[{"ID":1326,"Name":"number of output rows","Update":"305","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Update":"67371008","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Update":"18740","Value":"315512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Update":"18757","Value":"315829","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Update":"18749","Value":"160113","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Update":"6450117","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Update":"1654","Value":"15149","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Update":"196814300","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"7320","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Update":"11668","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Update":"305","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Update":"11462491","Value":"256667701","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":74,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3417804,"Value":80153743,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Update":18769,"Value":316109,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Update":18228279066,"Value":310591411994,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Update":2563,"Value":51255,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Update":432,"Value":3309,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":238481,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Update":11462491,"Value":256667701,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1654822513,"Value":30668880000,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":3808380668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131335688,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":74,"MinorGCTime":1360,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3417804,"Executor Run Time":18769,"Executor CPU Time":18228279066,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":432,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":11462491,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":34,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":20,"RDD Info":[{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2846\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_2842\",\"name\":\"Union\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[79,82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":80,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2841\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2845\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2838\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432592892,"Completion Time":1646432611703,"Accumulables":[{"ID":1326,"Name":"number of output rows","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"peak memory","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1329,"Name":"time in aggregation build","Value":"315512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"avg hash probe bucket list iters","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1362,"Name":"duration","Value":"315829","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1363,"Name":"duration","Value":"155550","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1364,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1368,"Name":"filesystem read time (sampled)","Value":"15509","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1369,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1370,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1372,"Name":"duration","Value":"160113","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1373,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1377,"Name":"filesystem read time (sampled)","Value":"15149","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1378,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1379,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"shuffle bytes written","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1390,"Name":"shuffle records written","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"shuffle write time","Value":"256667701","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1398,"Name":"internal.metrics.executorDeserializeTime","Value":74,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.executorDeserializeCpuTime","Value":80153743,"Internal":true,"Count Failed Values":true},{"ID":1400,"Name":"internal.metrics.executorRunTime","Value":316109,"Internal":true,"Count Failed Values":true},{"ID":1401,"Name":"internal.metrics.executorCpuTime","Value":310591411994,"Internal":true,"Count Failed Values":true},{"ID":1402,"Name":"internal.metrics.resultSize","Value":51255,"Internal":true,"Count Failed Values":true},{"ID":1403,"Name":"internal.metrics.jvmGCTime","Value":3309,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.peakExecutionMemory","Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":238481,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.write.writeTime","Value":256667701,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.input.bytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.input.recordsRead","Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.input.sampledTimeReadNano","Value":30668880000,"Internal":true,"Count Failed Values":true},{"ID":1424,"Name":"internal.metrics.input.sampledBytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":20,"Completion Time":1646432611706,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":24,"Submission Time":1646432611748,"Stage Infos":[{"Stage ID":42,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":1,"RDD Info":[{"RDD ID":97,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3891\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":95,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_3885\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3890\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[41],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":41,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":20,"RDD Info":[{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2846\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_2842\",\"name\":\"Union\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[79,82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":80,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2841\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2845\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2838\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[42,41],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_3891\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":42,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":1,"RDD Info":[{"RDD ID":97,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3891\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":95,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_3885\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3890\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[41],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432611749,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_3891\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":42,"Stage Attempt ID":0,"Task Info":{"Task ID":144,"Index":0,"Attempt":0,"Launch Time":1646432611766,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":42,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":144,"Index":0,"Attempt":0,"Launch Time":1646432611766,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432611834,"Failed":false,"Killed":false,"Accumulables":[{"ID":1382,"Name":"remote blocks read","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1383,"Name":"local blocks read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1384,"Name":"remote bytes read","Update":"179315","Value":"179315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1386,"Name":"local bytes read","Update":"59166","Value":"59166","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1387,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1388,"Name":"records read","Update":"6337","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1571,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1572,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1574,"Name":"time in aggregation build","Update":"17","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1575,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1576,"Name":"duration","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1579,"Name":"data size","Update":"9144","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1587,"Name":"shuffle bytes written","Update":"2243","Value":"2243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1588,"Name":"shuffle records written","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1589,"Name":"shuffle write time","Update":"324065","Value":"324065","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1590,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1591,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2703166,"Value":2703166,"Internal":true,"Count Failed Values":true},{"ID":1592,"Name":"internal.metrics.executorRunTime","Update":52,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":1593,"Name":"internal.metrics.executorCpuTime","Update":44625641,"Value":44625641,"Internal":true,"Count Failed Values":true},{"ID":1594,"Name":"internal.metrics.resultSize","Update":3168,"Value":3168,"Internal":true,"Count Failed Values":true},{"ID":1599,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1603,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1604,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":1605,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":179315,"Value":179315,"Internal":true,"Count Failed Values":true},{"ID":1606,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1607,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":59166,"Value":59166,"Internal":true,"Count Failed Values":true},{"ID":1608,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1609,"Name":"internal.metrics.shuffle.read.recordsRead","Update":6337,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1610,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2243,"Value":2243,"Internal":true,"Count Failed Values":true},{"ID":1611,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":381,"Value":381,"Internal":true,"Count Failed Values":true},{"ID":1612,"Name":"internal.metrics.shuffle.write.writeTime","Update":324065,"Value":324065,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2703166,"Executor Run Time":52,"Executor CPU Time":44625641,"Peak Execution Memory":67371008,"Result Size":3168,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":15,"Local Blocks Fetched":5,"Fetch Wait Time":0,"Remote Bytes Read":179315,"Remote Bytes Read To Disk":0,"Local Bytes Read":59166,"Total Records Read":6337},"Shuffle Write Metrics":{"Shuffle Bytes Written":2243,"Shuffle Write Time":324065,"Shuffle Records Written":381},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":42,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":1,"RDD Info":[{"RDD ID":97,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3891\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":95,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_3885\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3890\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[41],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432611749,"Completion Time":1646432611834,"Accumulables":[{"ID":1382,"Name":"remote blocks read","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1383,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1384,"Name":"remote bytes read","Value":"179315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1386,"Name":"local bytes read","Value":"59166","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1387,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1388,"Name":"records read","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1571,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1572,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1574,"Name":"time in aggregation build","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1575,"Name":"avg hash probe bucket list iters","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1576,"Name":"duration","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1579,"Name":"data size","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1587,"Name":"shuffle bytes written","Value":"2243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1588,"Name":"shuffle records written","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1589,"Name":"shuffle write time","Value":"324065","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1590,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1591,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2703166,"Internal":true,"Count Failed Values":true},{"ID":1592,"Name":"internal.metrics.executorRunTime","Value":52,"Internal":true,"Count Failed Values":true},{"ID":1593,"Name":"internal.metrics.executorCpuTime","Value":44625641,"Internal":true,"Count Failed Values":true},{"ID":1594,"Name":"internal.metrics.resultSize","Value":3168,"Internal":true,"Count Failed Values":true},{"ID":1599,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1603,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":15,"Internal":true,"Count Failed Values":true},{"ID":1604,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":1605,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":179315,"Internal":true,"Count Failed Values":true},{"ID":1606,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1607,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":59166,"Internal":true,"Count Failed Values":true},{"ID":1608,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1609,"Name":"internal.metrics.shuffle.read.recordsRead","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1610,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2243,"Internal":true,"Count Failed Values":true},{"ID":1611,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":381,"Internal":true,"Count Failed Values":true},{"ID":1612,"Name":"internal.metrics.shuffle.write.writeTime","Value":324065,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":24,"Completion Time":1646432611835,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":25,"Submission Time":1646432611879,"Stage Infos":[{"Stage ID":45,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":99,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4087\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[98],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":98,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[44],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":43,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":20,"RDD Info":[{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2846\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_2842\",\"name\":\"Union\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[79,82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":80,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2841\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2845\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2374\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2838\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2376\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":44,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":1,"RDD Info":[{"RDD ID":97,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3891\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":95,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_3885\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_3890\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[43],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[45,43,44],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_10\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 3f840c3c-efeb-4212-8442-47d43ca847b3)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"3f840c3c-efeb-4212-8442-47d43ca847b3","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":45,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":99,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4087\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[98],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":98,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[44],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432611883,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_10\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 3f840c3c-efeb-4212-8442-47d43ca847b3)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"3f840c3c-efeb-4212-8442-47d43ca847b3","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":45,"Stage Attempt ID":0,"Task Info":{"Task ID":145,"Index":0,"Attempt":0,"Launch Time":1646432611894,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":45,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":145,"Index":0,"Attempt":0,"Launch Time":1646432611894,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432611946,"Failed":false,"Killed":false,"Accumulables":[{"ID":1580,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1582,"Name":"remote bytes read","Update":"2243","Value":"2243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1585,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1586,"Name":"records read","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1620,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1621,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1622,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1623,"Name":"cumulative time","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1624,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1625,"Name":"num prefix comparisons","Update":"2765","Value":"2765","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1627,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1628,"Name":"duration","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1629,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1630,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2727118,"Value":2727118,"Internal":true,"Count Failed Values":true},{"ID":1631,"Name":"internal.metrics.executorRunTime","Update":34,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":1632,"Name":"internal.metrics.executorCpuTime","Update":29177081,"Value":29177081,"Internal":true,"Count Failed Values":true},{"ID":1633,"Name":"internal.metrics.resultSize","Update":8045,"Value":8045,"Internal":true,"Count Failed Values":true},{"ID":1638,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1642,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1643,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1644,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":2243,"Value":2243,"Internal":true,"Count Failed Values":true},{"ID":1645,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1646,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1647,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1648,"Name":"internal.metrics.shuffle.read.recordsRead","Update":381,"Value":381,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2727118,"Executor Run Time":34,"Executor CPU Time":29177081,"Peak Execution Memory":67174400,"Result Size":8045,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":2243,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":381},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":45,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":99,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4087\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[98],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":98,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[44],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432611883,"Completion Time":1646432611946,"Accumulables":[{"ID":1580,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1582,"Name":"remote bytes read","Value":"2243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1585,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1586,"Name":"records read","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1620,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1621,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1622,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1623,"Name":"cumulative time","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1624,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1625,"Name":"num prefix comparisons","Value":"2765","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1627,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1628,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1629,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1630,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2727118,"Internal":true,"Count Failed Values":true},{"ID":1631,"Name":"internal.metrics.executorRunTime","Value":34,"Internal":true,"Count Failed Values":true},{"ID":1632,"Name":"internal.metrics.executorCpuTime","Value":29177081,"Internal":true,"Count Failed Values":true},{"ID":1633,"Name":"internal.metrics.resultSize","Value":8045,"Internal":true,"Count Failed Values":true},{"ID":1638,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1642,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1643,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1644,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":2243,"Internal":true,"Count Failed Values":true},{"ID":1645,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1646,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1647,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1648,"Name":"internal.metrics.shuffle.read.recordsRead","Value":381,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":25,"Completion Time":1646432611946,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":111,"Index":9,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432612273,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"5617658","Value":"11436737","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"19337","Value":"37680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"5617658","Value":"11436737","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1590","Value":"2905","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"358169690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"175932825","Value":"358169690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"539295168","Value":"1097926752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"74785065","Value":"152619634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"5617658","Value":"11436737","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"230231540","Value":"410232972","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2912235,"Value":5168691,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":19430,"Value":37862,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":19078763449,"Value":37357129215,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1702,"Value":3405,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":86,"Value":174,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":74785065,"Value":152619634,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5617658,"Value":11436737,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":230231540,"Value":410232972,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":358169690,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":11436737,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1590816899,"Value":2906435142,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":358169690,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":65,"MinorGCTime":659,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2912235,"Executor Run Time":19430,"Executor CPU Time":19078763449,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":86,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":74785065,"Shuffle Write Time":230231540,"Shuffle Records Written":5617658},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":106,"Index":4,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432612519,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"6066650","Value":"17503387","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"19587","Value":"57267","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"6066650","Value":"17503387","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1370","Value":"4275","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"543205570","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"185035880","Value":"543205570","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"582398400","Value":"1680325152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"79904590","Value":"232524224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"6066650","Value":"17503387","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"187030217","Value":"597263189","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2041924,"Value":7210615,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":19678,"Value":57540,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":19469033858,"Value":56826163073,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1703,"Value":5108,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":106,"Value":280,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79904590,"Value":232524224,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6066650,"Value":17503387,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":187030217,"Value":597263189,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":543205570,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":17503387,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1370013444,"Value":4276448586,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":543205570,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129493376,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":634,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2041924,"Executor Run Time":19678,"Executor CPU Time":19469033858,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":106,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":79904590,"Shuffle Write Time":187030217,"Shuffle Records Written":6066650},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":105,"Index":3,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432614176,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"6429338","Value":"23932725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"21241","Value":"78508","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"6429338","Value":"23932725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1462","Value":"5737","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"738279885","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"195074315","Value":"738279885","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"617216448","Value":"2297541600","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"83388371","Value":"315912595","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"6429338","Value":"23932725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"221831067","Value":"819094256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2366049,"Value":9576664,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":21335,"Value":78875,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":20804201414,"Value":77630364487,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1703,"Value":6811,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":432,"Value":712,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83388371,"Value":315912595,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6429338,"Value":23932725,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":221831067,"Value":819094256,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":738279885,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":23932725,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1462762009,"Value":5739210595,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":738279885,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131335688,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":74,"MinorGCTime":1360,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2366049,"Executor Run Time":21335,"Executor CPU Time":20804201414,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":432,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83388371,"Shuffle Write Time":221831067,"Shuffle Records Written":6429338},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":110,"Index":8,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432614593,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"5674621","Value":"29607346","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"21664","Value":"100172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"5674621","Value":"29607346","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1432","Value":"7169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"916497529","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"178217644","Value":"916497529","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"544763616","Value":"2842305216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"75751912","Value":"391664507","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"5674621","Value":"29607346","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"189245409","Value":"1008339665","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2132636,"Value":11709300,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":21750,"Value":100625,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":21547388621,"Value":99177753108,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1703,"Value":8514,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":106,"Value":818,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":75751912,"Value":391664507,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5674621,"Value":29607346,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":189245409,"Value":1008339665,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":916497529,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":29607346,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1432967435,"Value":7172178030,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":916497529,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129493376,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":634,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2132636,"Executor Run Time":21750,"Executor CPU Time":21547388621,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":106,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":75751912,"Shuffle Write Time":189245409,"Shuffle Records Written":5674621},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":109,"Index":7,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432615702,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"5819811","Value":"35427157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"22775","Value":"122947","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"5819811","Value":"35427157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1470","Value":"8639","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"1097191372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"180693843","Value":"1097191372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"558701856","Value":"3401007072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"78151025","Value":"469815532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"5819811","Value":"35427157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"212697944","Value":"1221037609","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2318633,"Value":14027933,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":22862,"Value":123487,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":22365265850,"Value":121543018958,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1703,"Value":10217,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":432,"Value":1250,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":78151025,"Value":469815532,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819811,"Value":35427157,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":212697944,"Value":1221037609,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":1097191372,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":35427157,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1470842659,"Value":8643020689,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":1097191372,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11439903816,"JVMOffHeapMemory":131335688,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139113952,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":678082016,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1449245,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":74,"MinorGCTime":1360,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2318633,"Executor Run Time":22862,"Executor CPU Time":22365265850,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":432,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":78151025,"Shuffle Write Time":212697944,"Shuffle Records Written":5819811},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":102,"Index":0,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432616118,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"7213446","Value":"42640603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"23173","Value":"146120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"7213446","Value":"42640603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1721","Value":"10360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1325797992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"228606620","Value":"1325797992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"692490816","Value":"4093497888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"98606155","Value":"568421687","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"7213446","Value":"42640603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"228097012","Value":"1449134621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2113790,"Value":16141723,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":23277,"Value":146764,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":23086499013,"Value":144629517971,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1703,"Value":11920,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":106,"Value":1356,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":98606155,"Value":568421687,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7213446,"Value":42640603,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":228097012,"Value":1449134621,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1325797992,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":42640603,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1721437460,"Value":10364458149,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1325797992,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7815347784,"JVMOffHeapMemory":129493376,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":634,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2113790,"Executor Run Time":23277,"Executor CPU Time":23086499013,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":106,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":98606155,"Shuffle Write Time":228097012,"Shuffle Records Written":7213446},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":104,"Index":2,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432617451,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"6450117","Value":"49090720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"24521","Value":"170641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"6450117","Value":"49090720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1572","Value":"11932","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1522612292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"196814300","Value":"1522612292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"619211232","Value":"4712709120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"84724622","Value":"653146309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"6450117","Value":"49090720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"199836904","Value":"1648971525","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2481384,"Value":18623107,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":24610,"Value":171374,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":24411498898,"Value":169041016869,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1703,"Value":13623,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":88,"Value":1444,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":84724622,"Value":653146309,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6450117,"Value":49090720,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":199836904,"Value":1648971525,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1522612292,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":49090720,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1572550543,"Value":11937008692,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1522612292,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11038657552,"JVMOffHeapMemory":128612472,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838884,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1460963,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":69,"MinorGCTime":613,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2481384,"Executor Run Time":24610,"Executor CPU Time":24411498898,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":88,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":84724622,"Shuffle Write Time":199836904,"Shuffle Records Written":6450117},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":107,"Index":5,"Attempt":0,"Launch Time":1646432592823,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432618014,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"6096762","Value":"55187482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"25072","Value":"195713","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"6096762","Value":"55187482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1638","Value":"13570","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1706551800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"183939508","Value":"1706551800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"585289152","Value":"5297998272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"80392000","Value":"733538309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"6096762","Value":"55187482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"257955853","Value":"1906927378","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2957959,"Value":21581066,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":25170,"Value":196544,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":25001212202,"Value":194042229071,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1702,"Value":15325,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":86,"Value":1530,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":80392000,"Value":733538309,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6096762,"Value":55187482,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":257955853,"Value":1906927378,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":1706551800,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":55187482,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1638488304,"Value":13575496996,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":1706551800,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":68,"MinorGCTime":695,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2957959,"Executor Run Time":25170,"Executor CPU Time":25001212202,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":86,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":80392000,"Shuffle Write Time":257955853,"Shuffle Records Written":6096762},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":103,"Index":1,"Attempt":0,"Launch Time":1646432592822,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432618670,"Failed":false,"Killed":false,"Accumulables":[{"ID":1305,"Name":"number of output rows","Update":"6369482","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Update":"25727","Value":"221440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Update":"6369482","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Update":"1829","Value":"15399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Update":"197638534","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Update":"611470272","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Update":"85250127","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Update":"6369482","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Update":"263009926","Value":"2169937304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2871318,"Value":24452384,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Update":25826,"Value":222370,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Update":25627485539,"Value":219669714610,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Update":1702,"Value":17027,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Update":86,"Value":1616,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85250127,"Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6369482,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Update":263009926,"Value":2169937304,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1829956221,"Value":15405453217,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11160530416,"JVMOffHeapMemory":128110736,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870820,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467876,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":68,"MinorGCTime":695,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2871318,"Executor Run Time":25826,"Executor CPU Time":25627485539,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":86,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85250127,"Shuffle Write Time":263009926,"Shuffle Records Written":6369482},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"rdd at Instrumentation.scala:79","Number of Tasks":10,"RDD Info":[{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2826\",\"name\":\"Exchange\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2825\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_2371\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_2371\",\"name\":\"Scan csv \"}","Callsite":"rdd at Instrumentation.scala:79","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.util.Instrumentation.$anonfun$logDataset$1(Instrumentation.scala:79)\norg.apache.spark.internal.Logging.logInfo(Logging.scala:57)\norg.apache.spark.internal.Logging.logInfo$(Logging.scala:56)\norg.apache.spark.ml.util.Instrumentation.logDataset(Instrumentation.scala:79)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:331)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432592812,"Completion Time":1646432618670,"Accumulables":[{"ID":1305,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1306,"Name":"duration","Value":"221440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1307,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1311,"Name":"filesystem read time (sampled)","Value":"15399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1312,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1313,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1315,"Name":"data size","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"shuffle bytes written","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"shuffle records written","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"shuffle write time","Value":"2169937304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1332,"Name":"internal.metrics.executorDeserializeTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":1333,"Name":"internal.metrics.executorDeserializeCpuTime","Value":24452384,"Internal":true,"Count Failed Values":true},{"ID":1334,"Name":"internal.metrics.executorRunTime","Value":222370,"Internal":true,"Count Failed Values":true},{"ID":1335,"Name":"internal.metrics.executorCpuTime","Value":219669714610,"Internal":true,"Count Failed Values":true},{"ID":1336,"Name":"internal.metrics.resultSize","Value":17027,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.jvmGCTime","Value":1616,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.write.writeTime","Value":2169937304,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.input.sampledTimeReadNano","Value":15405453217,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":19,"Completion Time":1646432618671,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":26,"Submission Time":1646432619046,"Stage Infos":[{"Stage ID":46,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":10,"RDD Info":[{"RDD ID":108,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4886\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[107],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4424\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":105,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4424\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4885\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[46],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_4886\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":46,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":10,"RDD Info":[{"RDD ID":108,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4886\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[107],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4424\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":105,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4424\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4885\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432619047,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_4886\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":146,"Index":0,"Attempt":0,"Launch Time":1646432619060,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":147,"Index":1,"Attempt":0,"Launch Time":1646432619060,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":148,"Index":2,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":149,"Index":3,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":150,"Index":4,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":151,"Index":5,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":152,"Index":6,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":153,"Index":7,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":154,"Index":8,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":155,"Index":9,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerJobStart","Job ID":27,"Submission Time":1646432619128,"Stage Infos":[{"Stage ID":47,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":20,"RDD Info":[{"RDD ID":117,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4906\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[116],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4905\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4898\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[110],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4901\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[113],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_4902\",\"name\":\"Union\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[111,114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[47],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_4906\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":47,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":20,"RDD Info":[{"RDD ID":117,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4906\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[116],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4905\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4898\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[110],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4901\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[113],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_4902\",\"name\":\"Union\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[111,114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432619134,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_4906\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":156,"Index":0,"Attempt":0,"Launch Time":1646432619159,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":157,"Index":1,"Attempt":0,"Launch Time":1646432619159,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":158,"Index":2,"Attempt":0,"Launch Time":1646432619159,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":159,"Index":3,"Attempt":0,"Launch Time":1646432619159,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":160,"Index":4,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":161,"Index":5,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":162,"Index":6,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":163,"Index":7,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":164,"Index":8,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":165,"Index":9,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":166,"Index":10,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":167,"Index":11,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":168,"Index":12,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":169,"Index":13,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":170,"Index":14,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":171,"Index":15,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":172,"Index":16,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":173,"Index":17,"Attempt":0,"Launch Time":1646432619162,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":174,"Index":18,"Attempt":0,"Launch Time":1646432619163,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":175,"Index":19,"Attempt":0,"Launch Time":1646432619163,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerJobStart","Job ID":28,"Submission Time":1646432619201,"Stage Infos":[{"Stage ID":48,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":10,"RDD Info":[{"RDD ID":121,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4982\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[120],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":120,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4981\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[119],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":119,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[118],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":118,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[48],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_4982\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":48,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":10,"RDD Info":[{"RDD ID":121,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4982\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[120],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":120,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4981\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[119],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":119,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[118],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":118,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432619203,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_4982\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":176,"Index":0,"Attempt":0,"Launch Time":1646432619212,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":177,"Index":1,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":178,"Index":2,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":179,"Index":3,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":180,"Index":4,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":181,"Index":5,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":182,"Index":6,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":183,"Index":7,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":184,"Index":8,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":185,"Index":9,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":165,"Index":9,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432631302,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"313","Value":"313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"12091","Value":"12091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"12102","Value":"12102","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"12095","Value":"12095","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"5617658","Value":"5617658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1292","Value":"1292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7512","Value":"7512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11907","Value":"11907","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"313","Value":"313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"13104744","Value":"13104744","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3197249,"Value":3197249,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":12117,"Value":12117,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":11978058085,"Value":11978058085,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":2563,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":38,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11907,"Value":11907,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":313,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":13104744,"Value":13104744,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":5617658,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1292723196,"Value":1292723196,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3193854080,"JVMOffHeapMemory":131072776,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":75,"MinorGCTime":686,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3197249,"Executor Run Time":12117,"Executor CPU Time":11978058085,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":38,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11907,"Shuffle Write Time":13104744,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":172,"Index":16,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432631709,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"322","Value":"635","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"12493","Value":"24584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"12506","Value":"24608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"12497","Value":"12497","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1326","Value":"1326","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7728","Value":"15240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"12099","Value":"24006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"322","Value":"635","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"13710116","Value":"26814860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3566368,"Value":6763617,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":12521,"Value":24638,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":12390363648,"Value":24368421733,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2562,"Value":5125,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":40,"Value":78,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12099,"Value":24006,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":635,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":13710116,"Value":26814860,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":358169690,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":11436737,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1326805187,"Value":2619528383,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":358169690,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3883227104,"JVMOffHeapMemory":129796704,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":716,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3566368,"Executor Run Time":12521,"Executor CPU Time":12390363648,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":40,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12099,"Shuffle Write Time":13710116,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":175,"Index":19,"Attempt":0,"Launch Time":1646432619163,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432631865,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"310","Value":"945","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"12652","Value":"37236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"3007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"12663","Value":"37271","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"12656","Value":"25153","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"5617658","Value":"11436737","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1251","Value":"2577","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"358169690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"175932825","Value":"358169690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7440","Value":"22680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11887","Value":"35893","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"310","Value":"945","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"13600687","Value":"40415547","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3613147,"Value":10376764,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":12679,"Value":37317,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":12394225892,"Value":36762647625,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":7688,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":186,"Value":264,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11887,"Value":35893,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":310,"Value":945,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":13600687,"Value":40415547,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":534102515,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":17054395,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1251834065,"Value":3871362448,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":534102515,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3613147,"Executor Run Time":12679,"Executor CPU Time":12394225892,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":186,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11887,"Shuffle Write Time":13600687,"Shuffle Records Written":310},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":183,"Index":7,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432631867,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"12542","Value":"12542","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"12549","Value":"12549","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"5819811","Value":"5819811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"1348","Value":"1348","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"336","Value":"336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"952","Value":"952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"3692977","Value":"3692977","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2683324,"Value":2683324,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":12554,"Value":12554,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":12428267152,"Value":12428267152,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2132,"Value":2132,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":40,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":952,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":3692977,"Value":3692977,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":180693843,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":5819811,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1348723879,"Value":1348723879,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":180693843,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3883227104,"JVMOffHeapMemory":129796704,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":716,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2683324,"Executor Run Time":12554,"Executor CPU Time":12428267152,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":40,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":3692977,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":157,"Index":1,"Attempt":0,"Launch Time":1646432619159,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432632586,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"320","Value":"1265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"13374","Value":"50610","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"4007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"13386","Value":"50657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"13379","Value":"25474","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"6369482","Value":"11987140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1435","Value":"2727","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"373571359","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"197638534","Value":"373571359","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7680","Value":"30360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11962","Value":"47855","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"320","Value":"1265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"12971359","Value":"53386906","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3363101,"Value":13739865,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":13400,"Value":50717,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":13243942027,"Value":50006589652,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":10251,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":48,"Value":312,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11962,"Value":47855,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":1265,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":12971359,"Value":53386906,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":731741049,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":23423877,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1435685815,"Value":5307048263,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":731741049,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3193854080,"JVMOffHeapMemory":131072776,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":75,"MinorGCTime":686,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3363101,"Executor Run Time":13400,"Executor CPU Time":13243942027,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":48,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11962,"Shuffle Write Time":12971359,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":161,"Index":5,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432632604,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"312","Value":"1577","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"13393","Value":"64003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"13406","Value":"64063","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"13397","Value":"38871","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"6096762","Value":"18083902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1424","Value":"4151","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"557510867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"183939508","Value":"557510867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7488","Value":"37848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11723","Value":"59578","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"312","Value":"1577","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"13378864","Value":"66765770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3496094,"Value":17235959,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":13418,"Value":64135,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":13205984017,"Value":63212573669,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":12814,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":48,"Value":360,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11723,"Value":59578,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":312,"Value":1577,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":13378864,"Value":66765770,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":915680557,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":29520639,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1424565137,"Value":6731613400,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":915680557,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3193854080,"JVMOffHeapMemory":131072776,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":75,"MinorGCTime":686,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3496094,"Executor Run Time":13418,"Executor CPU Time":13205984017,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":48,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11723,"Shuffle Write Time":13378864,"Shuffle Records Written":312},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":171,"Index":15,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432632776,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"313","Value":"1890","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"13559","Value":"77562","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"13572","Value":"77635","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"13564","Value":"38717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"6096762","Value":"17533499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1334","Value":"3911","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"542109198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"183939508","Value":"542109198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7512","Value":"45360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11789","Value":"71367","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"313","Value":"1890","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"15657533","Value":"82423303","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4040033,"Value":21275992,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":13589,"Value":77724,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":13281069233,"Value":76493642902,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":15377,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":207,"Value":567,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11789,"Value":71367,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":1890,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":15657533,"Value":82423303,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":1099620065,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":35617401,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1334116978,"Value":8065730378,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":1099620065,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4040033,"Executor Run Time":13589,"Executor CPU Time":13281069233,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":207,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11789,"Shuffle Write Time":15657533,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":179,"Index":3,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432632891,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"19","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"13565","Value":"26107","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"13573","Value":"26122","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"6429338","Value":"12249149","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"1433","Value":"2781","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"375768158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"195074315","Value":"375768158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"456","Value":"792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"1232","Value":"2184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"19","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"3899505","Value":"7592482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2670872,"Value":5354196,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":13578,"Value":26132,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":13457586162,"Value":25885853314,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2132,"Value":4264,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":40,"Value":80,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1232,"Value":2184,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":19,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":3899505,"Value":7592482,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":375768158,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":12249149,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1433427557,"Value":2782151436,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":375768158,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3883227104,"JVMOffHeapMemory":129796704,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":716,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2670872,"Executor Run Time":13578,"Executor CPU Time":13457586162,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":40,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1232,"Shuffle Write Time":3899505,"Shuffle Records Written":19},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":162,"Index":6,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432633028,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"322","Value":"2212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"13815","Value":"91377","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"13827","Value":"91462","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"13819","Value":"52690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"5819079","Value":"23902981","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1356","Value":"5507","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"739747732","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"182236865","Value":"739747732","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7728","Value":"53088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"12103","Value":"83470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"322","Value":"2212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"14339979","Value":"96763282","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3285435,"Value":24561427,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":13843,"Value":91567,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":13672557203,"Value":90166200105,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":17940,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":73,"Value":640,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12103,"Value":83470,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":2212,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":14339979,"Value":96763282,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":1281856930,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":41436480,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1356778129,"Value":9422508507,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":1281856930,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2197191992,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592035,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":72,"MinorGCTime":650,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3285435,"Executor Run Time":13843,"Executor CPU Time":13672557203,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":73,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12103,"Shuffle Write Time":14339979,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":160,"Index":4,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432633119,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"299","Value":"2511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"13900","Value":"105277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"13912","Value":"105374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"13903","Value":"66593","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"6066650","Value":"29969631","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1437","Value":"6944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"924783612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"185035880","Value":"924783612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7176","Value":"60264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11443","Value":"94913","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"299","Value":"2511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"15847353","Value":"112610635","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4443012,"Value":29004439,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":13930,"Value":105497,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":13782261543,"Value":103948461648,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2562,"Value":20502,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":689,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11443,"Value":94913,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":299,"Value":2511,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":15847353,"Value":112610635,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1466892810,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":47503130,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1437399148,"Value":10859907655,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1466892810,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3883227104,"JVMOffHeapMemory":129796704,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":70,"MinorGCTime":716,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4443012,"Executor Run Time":13930,"Executor CPU Time":13782261543,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":49,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11443,"Shuffle Write Time":15847353,"Shuffle Records Written":299},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":173,"Index":17,"Attempt":0,"Launch Time":1646432619162,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432633169,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"324","Value":"2835","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"13958","Value":"119235","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"9014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"13969","Value":"119343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"13962","Value":"52679","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"5819811","Value":"23353310","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1436","Value":"5347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"722803041","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"180693843","Value":"722803041","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7776","Value":"68040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"12053","Value":"106966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"324","Value":"2835","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"11736672","Value":"124347307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3423049,"Value":32427488,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":13982,"Value":119479,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":13826832684,"Value":117775294332,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":23065,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":48,"Value":737,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12053,"Value":106966,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":324,"Value":2835,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":11736672,"Value":124347307,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":1647586653,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":53322941,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1436945845,"Value":12296853500,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":1647586653,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3193854080,"JVMOffHeapMemory":131072776,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":75,"MinorGCTime":686,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3423049,"Executor Run Time":13982,"Executor CPU Time":13826832684,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":48,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12053,"Shuffle Write Time":11736672,"Shuffle Records Written":324},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":185,"Index":9,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432633665,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"12","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"14347","Value":"40454","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"14353","Value":"40475","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"5617658","Value":"17866807","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"1377","Value":"4158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"551700983","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"175932825","Value":"551700983","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"288","Value":"1080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"816","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"12","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"3634378","Value":"11226860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2126756,"Value":7480952,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":14357,"Value":40489,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":14041979644,"Value":39927832958,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2133,"Value":6397,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":207,"Value":287,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":3000,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":3634378,"Value":11226860,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":551700983,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":17866807,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1377658334,"Value":4159809770,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":551700983,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2126756,"Executor Run Time":14357,"Executor CPU Time":14041979644,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":207,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":3634378,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":178,"Index":2,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432633821,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"18","Value":"63","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"14499","Value":"54953","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"4000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"14505","Value":"54980","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"6450117","Value":"24316924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"1517","Value":"5675","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"748515283","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"196814300","Value":"748515283","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"432","Value":"1512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"1164","Value":"4164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"18","Value":"63","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"3611058","Value":"14837918","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2626022,"Value":10106974,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":14509,"Value":54998,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":14402370962,"Value":54330203920,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2133,"Value":8530,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":48,"Value":335,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1164,"Value":4164,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":63,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":3611058,"Value":14837918,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":748515283,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":24316924,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1517637222,"Value":5677446992,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":748515283,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3193854080,"JVMOffHeapMemory":131072776,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":75,"MinorGCTime":686,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2626022,"Executor Run Time":14509,"Executor CPU Time":14402370962,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":48,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1164,"Shuffle Write Time":3611058,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":158,"Index":2,"Attempt":0,"Launch Time":1646432619159,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432634726,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"305","Value":"3140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"15508","Value":"134743","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"15521","Value":"134864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"15512","Value":"82105","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"6450117","Value":"36419748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1457","Value":"8401","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1121597912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"196814300","Value":"1121597912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7320","Value":"75360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11668","Value":"118634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"305","Value":"3140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"17690253","Value":"142037560","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3218023,"Value":35645511,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":15540,"Value":135019,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":15360053260,"Value":133135347592,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":25628,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":85,"Value":822,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":118634,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":3140,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":17690253,"Value":142037560,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1844400953,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":59773058,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1457723276,"Value":13754576776,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1844400953,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2197191992,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592035,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":72,"MinorGCTime":650,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3218023,"Executor Run Time":15540,"Executor CPU Time":15360053260,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":85,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":17690253,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":156,"Index":0,"Attempt":0,"Launch Time":1646432619159,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432634831,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"358","Value":"3498","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"741081088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"15616","Value":"150359","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"11014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"15627","Value":"150491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"15619","Value":"97724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"7213446","Value":"43633194","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1663","Value":"10064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1350204532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"228606620","Value":"1350204532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"8592","Value":"83952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"12849","Value":"131483","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"358","Value":"3498","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"13120784","Value":"155158344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4409445,"Value":40054956,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":15642,"Value":150661,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":15491919389,"Value":148627266981,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2562,"Value":28190,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":871,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":741081088,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12849,"Value":131483,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":3498,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":13120784,"Value":155158344,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":2073007573,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":66986504,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1663402073,"Value":15417978849,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":2073007573,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3883227104,"JVMOffHeapMemory":129865272,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":73,"MinorGCTime":744,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4409445,"Executor Run Time":15642,"Executor CPU Time":15491919389,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":49,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12849,"Shuffle Write Time":13120784,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":181,"Index":5,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432634909,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"15","Value":"78","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"15583","Value":"70536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"15590","Value":"70570","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"6096762","Value":"30413686","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"1452","Value":"7127","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"932454791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"183939508","Value":"932454791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"360","Value":"1872","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"1020","Value":"5184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"15","Value":"78","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"3745097","Value":"18583015","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2571321,"Value":12678295,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":15594,"Value":70592,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":15288360098,"Value":69618564018,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2133,"Value":10663,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":207,"Value":542,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1020,"Value":5184,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":15,"Value":78,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":3745097,"Value":18583015,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":932454791,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":30413686,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1452114334,"Value":7129561326,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":932454791,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2571321,"Executor Run Time":15594,"Executor CPU Time":15288360098,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":207,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1020,"Shuffle Write Time":3745097,"Shuffle Records Written":15},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":182,"Index":6,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432634979,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"14","Value":"92","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"15657","Value":"86193","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"15663","Value":"86233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"5819079","Value":"36232765","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"1547","Value":"8674","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"1114691656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"182236865","Value":"1114691656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"336","Value":"2208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"952","Value":"6136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"14","Value":"92","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"3316721","Value":"21899736","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2588215,"Value":15266510,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":15667,"Value":86259,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":15536851140,"Value":85155415158,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2133,"Value":12796,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":48,"Value":590,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":6136,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":92,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":3316721,"Value":21899736,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":1114691656,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":36232765,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1547395446,"Value":8676956772,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":1114691656,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3193854080,"JVMOffHeapMemory":131072776,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":75,"MinorGCTime":686,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2588215,"Executor Run Time":15667,"Executor CPU Time":15536851140,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":48,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":3316721,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":184,"Index":8,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432635323,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"12","Value":"104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"16005","Value":"102198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"16011","Value":"102244","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"5674621","Value":"41907386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"2269","Value":"10943","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1292909300","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"178217644","Value":"1292909300","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"288","Value":"2496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"816","Value":"6952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"12","Value":"104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"3419614","Value":"25319350","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2530945,"Value":17797455,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":16015,"Value":102274,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":15023113935,"Value":100178529093,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2133,"Value":14929,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":85,"Value":675,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":6952,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":104,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":3419614,"Value":25319350,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":1292909300,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":41907386,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":2269805956,"Value":10946762728,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":1292909300,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2807472864,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":76,"MinorGCTime":698,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2530945,"Executor Run Time":16015,"Executor CPU Time":15023113935,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":85,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":3419614,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":169,"Index":13,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432635551,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"296","Value":"3794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"808452096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"16328","Value":"166687","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"12021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"16344","Value":"166835","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"16331","Value":"69010","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"6429338","Value":"29782648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1602","Value":"6949","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"917877356","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"195074315","Value":"917877356","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7104","Value":"91056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11481","Value":"142964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"296","Value":"3794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"21427275","Value":"176585619","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3236150,"Value":43291106,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":16365,"Value":167026,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":16172448072,"Value":164799715053,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":30753,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":57,"Value":928,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":808452096,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11481,"Value":142964,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":3794,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":21427275,"Value":176585619,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":2268081888,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":73415842,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1602520871,"Value":17020499720,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":2268081888,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3193854080,"JVMOffHeapMemory":131072776,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":75,"MinorGCTime":686,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3236150,"Executor Run Time":16365,"Executor CPU Time":16172448072,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":57,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11481,"Shuffle Write Time":21427275,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":170,"Index":14,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432635651,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"301","Value":"4095","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"875823104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"16438","Value":"183125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"13021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"16449","Value":"183284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"16442","Value":"85452","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"6066650","Value":"35849298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1496","Value":"8445","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1102913236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"185035880","Value":"1102913236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7224","Value":"98280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11463","Value":"154427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"301","Value":"4095","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"14454571","Value":"191040190","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3476961,"Value":46768067,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":16464,"Value":183490,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":16283160414,"Value":181082875467,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":33316,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":85,"Value":1013,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":875823104,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11463,"Value":154427,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":301,"Value":4095,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":14454571,"Value":191040190,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":2453117768,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":79482492,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1496324802,"Value":18516824522,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":2453117768,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2807472864,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":76,"MinorGCTime":698,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3476961,"Executor Run Time":16464,"Executor CPU Time":16283160414,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":85,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11463,"Shuffle Write Time":14454571,"Shuffle Records Written":301},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":174,"Index":18,"Attempt":0,"Launch Time":1646432619163,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432635659,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"320","Value":"4415","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"943194112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"16444","Value":"199569","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"14021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"16459","Value":"199743","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"16448","Value":"101900","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"5674621","Value":"41523919","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1872","Value":"10317","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1281130880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"178217644","Value":"1281130880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7680","Value":"105960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"12025","Value":"166452","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"320","Value":"4415","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"14900301","Value":"205940491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":51,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3476832,"Value":50244899,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":16474,"Value":199964,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":15885153693,"Value":196968029160,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":35879,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":85,"Value":1098,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":943194112,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12025,"Value":166452,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":4415,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":14900301,"Value":205940491,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":2631335412,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":85157113,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1872647028,"Value":20389471550,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":2631335412,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2807472864,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":76,"MinorGCTime":698,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3476832,"Executor Run Time":16474,"Executor CPU Time":15885153693,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":85,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12025,"Shuffle Write Time":14900301,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":168,"Index":12,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432636126,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"305","Value":"4720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"1010565120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"16908","Value":"216477","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"15021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"16924","Value":"216667","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"16916","Value":"118816","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"6450117","Value":"47974036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1657","Value":"11974","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"196814300","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7320","Value":"113280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11668","Value":"178120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"305","Value":"4720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"12258011","Value":"218198502","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":56,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4126792,"Value":54371691,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":16937,"Value":216901,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":16778800173,"Value":213746829333,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2562,"Value":38441,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":1147,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1010565120,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":178120,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":4720,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":12258011,"Value":218198502,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":2828149712,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":91607230,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1657211497,"Value":22046683047,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":2828149712,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3883227104,"JVMOffHeapMemory":129865272,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":73,"MinorGCTime":744,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4126792,"Executor Run Time":16937,"Executor CPU Time":16778800173,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":49,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":12258011,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":164,"Index":8,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432636432,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"320","Value":"5040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"1077936128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"17214","Value":"233691","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"16021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"17229","Value":"233896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"17220","Value":"114944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"5674621","Value":"49307815","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1628","Value":"11692","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1528422176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"178217644","Value":"1528422176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7680","Value":"120960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"12024","Value":"190144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"320","Value":"5040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"12968960","Value":"231167462","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":61,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4578107,"Value":58949798,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":17243,"Value":234144,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":17086386587,"Value":230833215920,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2562,"Value":41003,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":1196,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1077936128,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12024,"Value":190144,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":5040,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":12968960,"Value":231167462,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":3006367356,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":97281851,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1628923076,"Value":23675606123,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":3006367356,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3883227104,"JVMOffHeapMemory":129865272,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":73,"MinorGCTime":744,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4578107,"Executor Run Time":17243,"Executor CPU Time":17086386587,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":49,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12024,"Shuffle Write Time":12968960,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":152,"Index":6,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432637064,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"17897","Value":"17897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1330","Value":"1330","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"558631584","Value":"558631584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"77834569","Value":"77834569","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"180962717","Value":"180962717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2088915,"Value":2088915,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":17984,"Value":17984,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":17854964335,"Value":17854964335,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1703,"Value":1703,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":77834569,"Value":77834569,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819079,"Value":5819079,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":180962717,"Value":180962717,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":5819079,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1330322542,"Value":1330322542,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3193854080,"JVMOffHeapMemory":131078160,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":713,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2088915,"Executor Run Time":17984,"Executor CPU Time":17854964335,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":57,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":77834569,"Shuffle Write Time":180962717,"Shuffle Records Written":5819079},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":163,"Index":7,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432637337,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"325","Value":"5365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"1145307136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"18126","Value":"251817","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"17028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"18137","Value":"252033","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"18129","Value":"133073","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"5819811","Value":"55127626","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1842","Value":"13534","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"1709116019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"180693843","Value":"1709116019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7800","Value":"128760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"12061","Value":"202205","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"325","Value":"5365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"12623930","Value":"243791392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":65,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3588013,"Value":62537811,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":18150,"Value":252294,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":17608707250,"Value":248441923170,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":43566,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":212,"Value":1408,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1145307136,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12061,"Value":202205,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":325,"Value":5365,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":12623930,"Value":243791392,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":3187061199,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":103101662,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1842272580,"Value":25517878703,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":3187061199,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3588013,"Executor Run Time":18150,"Executor CPU Time":17608707250,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":212,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12061,"Shuffle Write Time":12623930,"Shuffle Records Written":325},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":176,"Index":0,"Attempt":0,"Launch Time":1646432619212,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432637342,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"18","Value":"122","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"18023","Value":"120221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"18030","Value":"120274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"7213446","Value":"49120832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"1748","Value":"12691","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1521515920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"228606620","Value":"1521515920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"432","Value":"2928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"1224","Value":"8176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"18","Value":"122","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"3837926","Value":"29157276","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2276302,"Value":20073757,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":18035,"Value":120309,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":17885616539,"Value":118064145632,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2133,"Value":17062,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":85,"Value":760,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1224,"Value":8176,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":122,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":3837926,"Value":29157276,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1521515920,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":49120832,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1748422785,"Value":12695185513,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1521515920,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2807472864,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":76,"MinorGCTime":698,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2276302,"Executor Run Time":18035,"Executor CPU Time":17885616539,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":85,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1224,"Shuffle Write Time":3837926,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":180,"Index":4,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432637872,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"16","Value":"138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"18552","Value":"138773","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"18560","Value":"138834","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"6066650","Value":"55187482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"1643","Value":"14334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1706551800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"185035880","Value":"1706551800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"384","Value":"3312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"1028","Value":"9204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"16","Value":"138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"4294009","Value":"33451285","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2491678,"Value":22565435,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":18565,"Value":138874,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":18366127570,"Value":136430273202,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2133,"Value":19195,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":85,"Value":845,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1028,"Value":9204,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":138,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":4294009,"Value":33451285,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1706551800,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":55187482,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1643488266,"Value":14338673779,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1706551800,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2807472864,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":76,"MinorGCTime":698,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2491678,"Executor Run Time":18565,"Executor CPU Time":18366127570,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":85,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1028,"Shuffle Write Time":4294009,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":177,"Index":1,"Attempt":0,"Launch Time":1646432619213,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432637881,"Failed":false,"Killed":false,"Accumulables":[{"ID":1767,"Name":"number of output rows","Update":"16","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Update":"18562","Value":"157335","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Update":"18568","Value":"157402","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Update":"6369482","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Update":"1648","Value":"15982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Update":"197638534","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Update":"384","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Update":"1088","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Update":"16","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Update":"4100086","Value":"37551371","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2560832,"Value":25126267,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Update":18573,"Value":157447,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Update":18265105646,"Value":154695378848,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Update":2133,"Value":21328,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Update":212,"Value":1057,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1088,"Value":10292,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Update":4100086,"Value":37551371,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1648339490,"Value":15987013269,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2560832,"Executor Run Time":18573,"Executor CPU Time":18265105646,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":212,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1088,"Shuffle Write Time":4100086,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":48,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":10,"RDD Info":[{"RDD ID":121,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4982\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[120],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":120,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4981\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[119],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":119,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[118],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":118,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432619203,"Completion Time":1646432637881,"Accumulables":[{"ID":1767,"Name":"number of output rows","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1768,"Name":"peak memory","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1770,"Name":"time in aggregation build","Value":"157335","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1771,"Name":"avg hash probe bucket list iters","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"duration","Value":"157402","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1808,"Name":"filesystem read time (sampled)","Value":"15982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1809,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1810,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1812,"Name":"data size","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1820,"Name":"shuffle bytes written","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1821,"Name":"shuffle records written","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1822,"Name":"shuffle write time","Value":"37551371","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1823,"Name":"internal.metrics.executorDeserializeTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1824,"Name":"internal.metrics.executorDeserializeCpuTime","Value":25126267,"Internal":true,"Count Failed Values":true},{"ID":1825,"Name":"internal.metrics.executorRunTime","Value":157447,"Internal":true,"Count Failed Values":true},{"ID":1826,"Name":"internal.metrics.executorCpuTime","Value":154695378848,"Internal":true,"Count Failed Values":true},{"ID":1827,"Name":"internal.metrics.resultSize","Value":21328,"Internal":true,"Count Failed Values":true},{"ID":1828,"Name":"internal.metrics.jvmGCTime","Value":1057,"Internal":true,"Count Failed Values":true},{"ID":1832,"Name":"internal.metrics.peakExecutionMemory","Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":10292,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":154,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.shuffle.write.writeTime","Value":37551371,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1847,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1848,"Name":"internal.metrics.input.sampledTimeReadNano","Value":15987013269,"Internal":true,"Count Failed Values":true},{"ID":1849,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":28,"Completion Time":1646432637882,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":29,"Submission Time":1646432637923,"Stage Infos":[{"Stage ID":49,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":10,"RDD Info":[{"RDD ID":121,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4982\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[120],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":120,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4981\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[119],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":119,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[118],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":118,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":50,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":1,"RDD Info":[{"RDD ID":124,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5334\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[123],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":123,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5333\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[122],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":122,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5328\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[121],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[49],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[49,50],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_5334\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":50,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":1,"RDD Info":[{"RDD ID":124,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5334\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[123],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":123,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5333\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[122],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":122,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5328\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[121],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[49],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432637924,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_5334\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":50,"Stage Attempt ID":0,"Task Info":{"Task ID":186,"Index":0,"Attempt":0,"Launch Time":1646432637936,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":50,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":186,"Index":0,"Attempt":0,"Launch Time":1646432637936,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432638002,"Failed":false,"Killed":false,"Accumulables":[{"ID":1813,"Name":"remote blocks read","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1814,"Name":"local blocks read","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1815,"Name":"remote bytes read","Update":"8108","Value":"8108","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1817,"Name":"local bytes read","Update":"2184","Value":"2184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1818,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1819,"Name":"records read","Update":"154","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1853,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1854,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1856,"Name":"time in aggregation build","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1857,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1858,"Name":"duration","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1861,"Name":"data size","Update":"552","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1869,"Name":"shuffle bytes written","Update":"265","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1870,"Name":"shuffle records written","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1871,"Name":"shuffle write time","Update":"276897","Value":"276897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1872,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1873,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2713782,"Value":2713782,"Internal":true,"Count Failed Values":true},{"ID":1874,"Name":"internal.metrics.executorRunTime","Update":49,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":1875,"Name":"internal.metrics.executorCpuTime","Update":29884319,"Value":29884319,"Internal":true,"Count Failed Values":true},{"ID":1876,"Name":"internal.metrics.resultSize","Update":2737,"Value":2737,"Internal":true,"Count Failed Values":true},{"ID":1881,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1885,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":1886,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1887,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":8108,"Value":8108,"Internal":true,"Count Failed Values":true},{"ID":1888,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1889,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2184,"Value":2184,"Internal":true,"Count Failed Values":true},{"ID":1890,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1891,"Name":"internal.metrics.shuffle.read.recordsRead","Update":154,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":1892,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":265,"Value":265,"Internal":true,"Count Failed Values":true},{"ID":1893,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":1894,"Name":"internal.metrics.shuffle.write.writeTime","Update":276897,"Value":276897,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2713782,"Executor Run Time":49,"Executor CPU Time":29884319,"Peak Execution Memory":67371008,"Result Size":2737,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":8108,"Remote Bytes Read To Disk":0,"Local Bytes Read":2184,"Total Records Read":154},"Shuffle Write Metrics":{"Shuffle Bytes Written":265,"Shuffle Write Time":276897,"Shuffle Records Written":23},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":50,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":1,"RDD Info":[{"RDD ID":124,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5334\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[123],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":123,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5333\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[122],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":122,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5328\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[121],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[49],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432637924,"Completion Time":1646432638003,"Accumulables":[{"ID":1813,"Name":"remote blocks read","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1814,"Name":"local blocks read","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1815,"Name":"remote bytes read","Value":"8108","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1817,"Name":"local bytes read","Value":"2184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1818,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1819,"Name":"records read","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1853,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1854,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1856,"Name":"time in aggregation build","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1857,"Name":"avg hash probe bucket list iters","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1858,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1861,"Name":"data size","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1869,"Name":"shuffle bytes written","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1870,"Name":"shuffle records written","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1871,"Name":"shuffle write time","Value":"276897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1872,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1873,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2713782,"Internal":true,"Count Failed Values":true},{"ID":1874,"Name":"internal.metrics.executorRunTime","Value":49,"Internal":true,"Count Failed Values":true},{"ID":1875,"Name":"internal.metrics.executorCpuTime","Value":29884319,"Internal":true,"Count Failed Values":true},{"ID":1876,"Name":"internal.metrics.resultSize","Value":2737,"Internal":true,"Count Failed Values":true},{"ID":1881,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1885,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":8,"Internal":true,"Count Failed Values":true},{"ID":1886,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1887,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":8108,"Internal":true,"Count Failed Values":true},{"ID":1888,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1889,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2184,"Internal":true,"Count Failed Values":true},{"ID":1890,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1891,"Name":"internal.metrics.shuffle.read.recordsRead","Value":154,"Internal":true,"Count Failed Values":true},{"ID":1892,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":265,"Internal":true,"Count Failed Values":true},{"ID":1893,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23,"Internal":true,"Count Failed Values":true},{"ID":1894,"Name":"internal.metrics.shuffle.write.writeTime","Value":276897,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":29,"Completion Time":1646432638003,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":30,"Submission Time":1646432638064,"Stage Infos":[{"Stage ID":51,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":10,"RDD Info":[{"RDD ID":121,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4982\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[120],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":120,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4981\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[119],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":119,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[118],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":118,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4449\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":52,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":1,"RDD Info":[{"RDD ID":124,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5334\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[123],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":123,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5333\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[122],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":122,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5328\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[121],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[51],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":53,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":126,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5587\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[125],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":125,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[124],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[52],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[51,52,53],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_6\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 81915923-1eb4-4bd5-aa61-8c071950f546)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"81915923-1eb4-4bd5-aa61-8c071950f546","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":53,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":126,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5587\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[125],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":125,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[124],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[52],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432638065,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_6\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 81915923-1eb4-4bd5-aa61-8c071950f546)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"81915923-1eb4-4bd5-aa61-8c071950f546","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":167,"Index":11,"Attempt":0,"Launch Time":1646432619161,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432638074,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"318","Value":"5683","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"1212678144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"18862","Value":"270679","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"18028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"18874","Value":"270907","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"18866","Value":"137682","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"6369482","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1680","Value":"13654","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7632","Value":"136392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11947","Value":"214152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"318","Value":"5683","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"12540660","Value":"256332052","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":68,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3321644,"Value":65859455,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":18887,"Value":271181,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":18587751944,"Value":267029675114,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":46129,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":212,"Value":1620,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1212678144,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11947,"Value":214152,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":318,"Value":5683,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":12540660,"Value":256332052,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":3384699733,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":109471144,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1680490101,"Value":27198368804,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":3384699733,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3321644,"Executor Run Time":18887,"Executor CPU Time":18587751944,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":212,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11947,"Shuffle Write Time":12540660,"Shuffle Records Written":318},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":53,"Stage Attempt ID":0,"Task Info":{"Task ID":187,"Index":0,"Attempt":0,"Launch Time":1646432638075,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":53,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":187,"Index":0,"Attempt":0,"Launch Time":1646432638075,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432638099,"Failed":false,"Killed":false,"Accumulables":[{"ID":1862,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1864,"Name":"remote bytes read","Update":"265","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1867,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1868,"Name":"records read","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1902,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1903,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1904,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1905,"Name":"cumulative time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1906,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1907,"Name":"num prefix comparisons","Update":"77","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1909,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1910,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1911,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1912,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2987193,"Value":2987193,"Internal":true,"Count Failed Values":true},{"ID":1913,"Name":"internal.metrics.executorRunTime","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":1914,"Name":"internal.metrics.executorCpuTime","Update":5822246,"Value":5822246,"Internal":true,"Count Failed Values":true},{"ID":1915,"Name":"internal.metrics.resultSize","Update":3792,"Value":3792,"Internal":true,"Count Failed Values":true},{"ID":1920,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1924,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1925,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1926,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":265,"Value":265,"Internal":true,"Count Failed Values":true},{"ID":1927,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1928,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1929,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1930,"Name":"internal.metrics.shuffle.read.recordsRead","Update":23,"Value":23,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2987193,"Executor Run Time":7,"Executor CPU Time":5822246,"Peak Execution Memory":67174400,"Result Size":3792,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":265,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":23},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":53,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":126,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5587\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[125],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":125,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[124],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[52],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432638065,"Completion Time":1646432638099,"Accumulables":[{"ID":1862,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1864,"Name":"remote bytes read","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1867,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1868,"Name":"records read","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1902,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1903,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1904,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1905,"Name":"cumulative time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1906,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1907,"Name":"num prefix comparisons","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1909,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1910,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1911,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1912,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2987193,"Internal":true,"Count Failed Values":true},{"ID":1913,"Name":"internal.metrics.executorRunTime","Value":7,"Internal":true,"Count Failed Values":true},{"ID":1914,"Name":"internal.metrics.executorCpuTime","Value":5822246,"Internal":true,"Count Failed Values":true},{"ID":1915,"Name":"internal.metrics.resultSize","Value":3792,"Internal":true,"Count Failed Values":true},{"ID":1920,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":1924,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1925,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1926,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":265,"Internal":true,"Count Failed Values":true},{"ID":1927,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1928,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1929,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1930,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":30,"Completion Time":1646432638100,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":150,"Index":4,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432638186,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"6066650","Value":"11885729","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"19016","Value":"36913","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"6066650","Value":"11885729","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1365","Value":"2695","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"367272745","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"185035880","Value":"367272745","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"582398400","Value":"1141029984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"79904590","Value":"157739159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"6066650","Value":"11885729","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"185136509","Value":"366099226","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1857319,"Value":3946234,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":19104,"Value":37088,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":18900173363,"Value":36755137698,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1703,"Value":3406,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":97,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79904590,"Value":157739159,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6066650,"Value":11885729,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":185136509,"Value":366099226,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":367272745,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":11885729,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1365614379,"Value":2695936921,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":367272745,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2807472864,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":76,"MinorGCTime":698,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1857319,"Executor Run Time":19104,"Executor CPU Time":18900173363,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":97,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":79904590,"Shuffle Write Time":185136509,"Shuffle Records Written":6066650},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":148,"Index":2,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432638563,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"6450117","Value":"18335846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"19394","Value":"56307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"6450117","Value":"18335846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1370","Value":"4065","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"564087045","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"196814300","Value":"564087045","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"619211232","Value":"1760241216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"84724622","Value":"242463781","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"6450117","Value":"18335846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"191664188","Value":"557763414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2065580,"Value":6011814,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":19483,"Value":56571,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":19357434841,"Value":56112572539,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1703,"Value":5109,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":57,"Value":211,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":84724622,"Value":242463781,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6450117,"Value":18335846,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":191664188,"Value":557763414,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":564087045,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":18335846,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1370978470,"Value":4066915391,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":564087045,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3193854080,"JVMOffHeapMemory":131078160,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1464157,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":713,"MajorGCCount":5,"MajorGCTime":290},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2065580,"Executor Run Time":19483,"Executor CPU Time":19357434841,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":57,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":84724622,"Shuffle Write Time":191664188,"Shuffle Records Written":6450117},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":159,"Index":3,"Attempt":0,"Launch Time":1646432619159,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432638599,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"296","Value":"5979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"1280049152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"19387","Value":"290066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"19035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"19400","Value":"290307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Update":"19393","Value":"152466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Update":"6429338","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Update":"1717","Value":"15251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Update":"195074315","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"7104","Value":"143496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"11482","Value":"225634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"296","Value":"5979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"11513762","Value":"267845814","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":72,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3759026,"Value":69618481,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":19413,"Value":290594,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":19095491598,"Value":286125166712,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":48692,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":217,"Value":1837,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1280049152,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11482,"Value":225634,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":5979,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":11513762,"Value":267845814,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":3579774048,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":115900482,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1717375273,"Value":28915744077,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":3579774048,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3759026,"Executor Run Time":19413,"Executor CPU Time":19095491598,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":217,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11482,"Shuffle Write Time":11513762,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":153,"Index":7,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432638799,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"5819811","Value":"24155657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"19623","Value":"75930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"5819811","Value":"24155657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1413","Value":"5478","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"744780888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"180693843","Value":"744780888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"558701856","Value":"2318943072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"78151025","Value":"320614806","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"5819811","Value":"24155657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"230712638","Value":"788476052","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2546921,"Value":8558735,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":19716,"Value":76287,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":19590143551,"Value":75702716090,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1702,"Value":6811,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":260,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":78151025,"Value":320614806,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819811,"Value":24155657,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":230712638,"Value":788476052,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":744780888,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":24155657,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1413043503,"Value":5479958894,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":744780888,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3883227104,"JVMOffHeapMemory":129865272,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":73,"MinorGCTime":744,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2546921,"Executor Run Time":19716,"Executor CPU Time":19590143551,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":49,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":78151025,"Shuffle Write Time":230712638,"Shuffle Records Written":5819811},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":151,"Index":5,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432639305,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"6096762","Value":"30252419","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"20130","Value":"96060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"6096762","Value":"30252419","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1392","Value":"6870","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"928720396","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"183939508","Value":"928720396","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"585289152","Value":"2904232224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"80392000","Value":"401006806","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"6096762","Value":"30252419","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"218733879","Value":"1007209931","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1963603,"Value":10522338,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":20224,"Value":96511,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":19898852580,"Value":95601568670,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1703,"Value":8514,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":217,"Value":477,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":80392000,"Value":401006806,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6096762,"Value":30252419,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":218733879,"Value":1007209931,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":928720396,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":30252419,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1392821953,"Value":6872780847,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":928720396,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1963603,"Executor Run Time":20224,"Executor CPU Time":19898852580,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":217,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":80392000,"Shuffle Write Time":218733879,"Shuffle Records Written":6096762},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":149,"Index":3,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432640208,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"6429338","Value":"36681757","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"21027","Value":"117087","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"6429338","Value":"36681757","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1600","Value":"8470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1123794711","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"195074315","Value":"1123794711","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"617216448","Value":"3521448672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"83388371","Value":"484395177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"6429338","Value":"36681757","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"247326761","Value":"1254536692","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2301059,"Value":12823397,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":21125,"Value":117636,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":20986889429,"Value":116588458099,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1702,"Value":10216,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":526,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83388371,"Value":484395177,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6429338,"Value":36681757,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":247326761,"Value":1254536692,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1123794711,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":36681757,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1600185222,"Value":8472966069,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1123794711,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3883227104,"JVMOffHeapMemory":129865272,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":476467914,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1400848,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":73,"MinorGCTime":744,"MajorGCCount":5,"MajorGCTime":326},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2301059,"Executor Run Time":21125,"Executor CPU Time":20986889429,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":49,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83388371,"Shuffle Write Time":247326761,"Shuffle Records Written":6429338},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":155,"Index":9,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432640245,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"5617658","Value":"42299415","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"21076","Value":"138163","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"5617658","Value":"42299415","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1426","Value":"9896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"1299727536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"175932825","Value":"1299727536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"539295168","Value":"4060743840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"74785065","Value":"559180242","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"5617658","Value":"42299415","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"206057457","Value":"1460594149","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1997117,"Value":14820514,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":21165,"Value":138801,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":20877342094,"Value":137465800193,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1703,"Value":11919,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":217,"Value":743,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":74785065,"Value":559180242,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5617658,"Value":42299415,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":206057457,"Value":1460594149,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":1299727536,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":42299415,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1426422925,"Value":9899388994,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":1299727536,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133286656,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":78,"MinorGCTime":1546,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1997117,"Executor Run Time":21165,"Executor CPU Time":20877342094,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":217,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":74785065,"Shuffle Write Time":206057457,"Shuffle Records Written":5617658},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":166,"Index":10,"Attempt":0,"Launch Time":1646432619160,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432640758,"Failed":false,"Killed":false,"Accumulables":[{"ID":1701,"Name":"number of output rows","Update":"358","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Update":"67371008","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Update":"21548","Value":"311614","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Update":"21560","Value":"311867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Update":"21552","Value":"159234","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Update":"1906","Value":"15560","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Update":"8592","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Update":"12847","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Update":"358","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Update":"11364900","Value":"279210714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":76,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3402977,"Value":73021458,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Update":21572,"Value":312166,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Update":21367584091,"Value":307492750803,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Update":2563,"Value":51255,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Update":97,"Value":1934,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12847,"Value":238481,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Update":11364900,"Value":279210714,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1906317187,"Value":30822061264,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":3808380668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2807472864,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":76,"MinorGCTime":698,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3402977,"Executor Run Time":21572,"Executor CPU Time":21367584091,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":97,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12847,"Shuffle Write Time":11364900,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":47,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":20,"RDD Info":[{"RDD ID":117,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4906\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[116],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4905\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4898\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[110],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4901\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[113],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_4902\",\"name\":\"Union\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[111,114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432619134,"Completion Time":1646432640759,"Accumulables":[{"ID":1701,"Name":"number of output rows","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"peak memory","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1704,"Name":"time in aggregation build","Value":"311614","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1705,"Name":"avg hash probe bucket list iters","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1737,"Name":"duration","Value":"311867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1738,"Name":"duration","Value":"152466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1739,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1743,"Name":"filesystem read time (sampled)","Value":"15251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1744,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1745,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1747,"Name":"duration","Value":"159234","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1748,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1752,"Name":"filesystem read time (sampled)","Value":"15560","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1753,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1754,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1756,"Name":"data size","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"shuffle bytes written","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"shuffle records written","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"shuffle write time","Value":"279210714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1773,"Name":"internal.metrics.executorDeserializeTime","Value":76,"Internal":true,"Count Failed Values":true},{"ID":1774,"Name":"internal.metrics.executorDeserializeCpuTime","Value":73021458,"Internal":true,"Count Failed Values":true},{"ID":1775,"Name":"internal.metrics.executorRunTime","Value":312166,"Internal":true,"Count Failed Values":true},{"ID":1776,"Name":"internal.metrics.executorCpuTime","Value":307492750803,"Internal":true,"Count Failed Values":true},{"ID":1777,"Name":"internal.metrics.resultSize","Value":51255,"Internal":true,"Count Failed Values":true},{"ID":1778,"Name":"internal.metrics.jvmGCTime","Value":1934,"Internal":true,"Count Failed Values":true},{"ID":1782,"Name":"internal.metrics.peakExecutionMemory","Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":1793,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":238481,"Internal":true,"Count Failed Values":true},{"ID":1794,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1795,"Name":"internal.metrics.shuffle.write.writeTime","Value":279210714,"Internal":true,"Count Failed Values":true},{"ID":1796,"Name":"internal.metrics.input.bytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":1797,"Name":"internal.metrics.input.recordsRead","Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":1798,"Name":"internal.metrics.input.sampledTimeReadNano","Value":30822061264,"Internal":true,"Count Failed Values":true},{"ID":1799,"Name":"internal.metrics.input.sampledBytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":27,"Completion Time":1646432640762,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":31,"Submission Time":1646432640802,"Stage Infos":[{"Stage ID":54,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":20,"RDD Info":[{"RDD ID":117,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4906\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[116],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4905\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4898\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[110],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4901\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[113],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_4902\",\"name\":\"Union\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[111,114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":55,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":1,"RDD Info":[{"RDD ID":129,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5978\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[128],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":127,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5972\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[117],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":128,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5977\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[127],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[54],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[54,55],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_5978\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":55,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":1,"RDD Info":[{"RDD ID":129,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5978\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[128],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":127,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5972\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[117],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":128,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5977\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[127],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[54],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432640806,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_5978\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":55,"Stage Attempt ID":0,"Task Info":{"Task ID":188,"Index":0,"Attempt":0,"Launch Time":1646432640814,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":55,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":188,"Index":0,"Attempt":0,"Launch Time":1646432640814,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432640852,"Failed":false,"Killed":false,"Accumulables":[{"ID":1757,"Name":"remote blocks read","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1758,"Name":"local blocks read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1759,"Name":"remote bytes read","Update":"178375","Value":"178375","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1761,"Name":"local bytes read","Update":"60106","Value":"60106","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1762,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1763,"Name":"records read","Update":"6337","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1946,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1947,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1949,"Name":"time in aggregation build","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1950,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1951,"Name":"duration","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1954,"Name":"data size","Update":"9144","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1962,"Name":"shuffle bytes written","Update":"2249","Value":"2249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1963,"Name":"shuffle records written","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1964,"Name":"shuffle write time","Update":"223621","Value":"223621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1965,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1966,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2440460,"Value":2440460,"Internal":true,"Count Failed Values":true},{"ID":1967,"Name":"internal.metrics.executorRunTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":1968,"Name":"internal.metrics.executorCpuTime","Update":20062948,"Value":20062948,"Internal":true,"Count Failed Values":true},{"ID":1969,"Name":"internal.metrics.resultSize","Update":3168,"Value":3168,"Internal":true,"Count Failed Values":true},{"ID":1974,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1978,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1979,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":1980,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":178375,"Value":178375,"Internal":true,"Count Failed Values":true},{"ID":1981,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1982,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":60106,"Value":60106,"Internal":true,"Count Failed Values":true},{"ID":1983,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1984,"Name":"internal.metrics.shuffle.read.recordsRead","Update":6337,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1985,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2249,"Value":2249,"Internal":true,"Count Failed Values":true},{"ID":1986,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":381,"Value":381,"Internal":true,"Count Failed Values":true},{"ID":1987,"Name":"internal.metrics.shuffle.write.writeTime","Update":223621,"Value":223621,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2440460,"Executor Run Time":22,"Executor CPU Time":20062948,"Peak Execution Memory":67371008,"Result Size":3168,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":15,"Local Blocks Fetched":5,"Fetch Wait Time":0,"Remote Bytes Read":178375,"Remote Bytes Read To Disk":0,"Local Bytes Read":60106,"Total Records Read":6337},"Shuffle Write Metrics":{"Shuffle Bytes Written":2249,"Shuffle Write Time":223621,"Shuffle Records Written":381},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":55,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":1,"RDD Info":[{"RDD ID":129,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5978\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[128],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":127,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5972\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[117],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":128,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5977\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[127],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[54],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432640806,"Completion Time":1646432640852,"Accumulables":[{"ID":1757,"Name":"remote blocks read","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1758,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1759,"Name":"remote bytes read","Value":"178375","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1761,"Name":"local bytes read","Value":"60106","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1762,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1763,"Name":"records read","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1946,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1947,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1949,"Name":"time in aggregation build","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1950,"Name":"avg hash probe bucket list iters","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1951,"Name":"duration","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1954,"Name":"data size","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1962,"Name":"shuffle bytes written","Value":"2249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1963,"Name":"shuffle records written","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1964,"Name":"shuffle write time","Value":"223621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1965,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1966,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2440460,"Internal":true,"Count Failed Values":true},{"ID":1967,"Name":"internal.metrics.executorRunTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":1968,"Name":"internal.metrics.executorCpuTime","Value":20062948,"Internal":true,"Count Failed Values":true},{"ID":1969,"Name":"internal.metrics.resultSize","Value":3168,"Internal":true,"Count Failed Values":true},{"ID":1974,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":1978,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":15,"Internal":true,"Count Failed Values":true},{"ID":1979,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":1980,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":178375,"Internal":true,"Count Failed Values":true},{"ID":1981,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1982,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":60106,"Internal":true,"Count Failed Values":true},{"ID":1983,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1984,"Name":"internal.metrics.shuffle.read.recordsRead","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":1985,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2249,"Internal":true,"Count Failed Values":true},{"ID":1986,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":381,"Internal":true,"Count Failed Values":true},{"ID":1987,"Name":"internal.metrics.shuffle.write.writeTime","Value":223621,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":31,"Completion Time":1646432640852,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":32,"Submission Time":1646432640912,"Stage Infos":[{"Stage ID":56,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":20,"RDD Info":[{"RDD ID":117,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4906\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[116],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4905\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4898\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[110],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4427\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4901\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[113],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4429\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_4902\",\"name\":\"Union\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[111,114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":57,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":1,"RDD Info":[{"RDD ID":129,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5978\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[128],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":127,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5972\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[117],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":128,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_5977\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[127],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[56],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":58,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":131,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6181\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[130],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":130,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[129],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[57],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[56,57,58],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_10\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId f597a19f-c48b-44d1-80ef-1675f4f02155)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"f597a19f-c48b-44d1-80ef-1675f4f02155","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":58,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":131,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6181\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[130],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":130,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[129],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[57],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432640913,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_10\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId f597a19f-c48b-44d1-80ef-1675f4f02155)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"f597a19f-c48b-44d1-80ef-1675f4f02155","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":58,"Stage Attempt ID":0,"Task Info":{"Task ID":189,"Index":0,"Attempt":0,"Launch Time":1646432640923,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":58,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":189,"Index":0,"Attempt":0,"Launch Time":1646432640923,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432640954,"Failed":false,"Killed":false,"Accumulables":[{"ID":1955,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1957,"Name":"remote bytes read","Update":"2249","Value":"2249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1960,"Name":"fetch wait time","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1961,"Name":"records read","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1995,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1996,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1997,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1998,"Name":"cumulative time","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1999,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2000,"Name":"num prefix comparisons","Update":"2773","Value":"2773","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2002,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2003,"Name":"duration","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2004,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2005,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2680370,"Value":2680370,"Internal":true,"Count Failed Values":true},{"ID":2006,"Name":"internal.metrics.executorRunTime","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2007,"Name":"internal.metrics.executorCpuTime","Update":7500488,"Value":7500488,"Internal":true,"Count Failed Values":true},{"ID":2008,"Name":"internal.metrics.resultSize","Update":8045,"Value":8045,"Internal":true,"Count Failed Values":true},{"ID":2013,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2017,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2018,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2019,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":2249,"Value":2249,"Internal":true,"Count Failed Values":true},{"ID":2020,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2021,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2022,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2023,"Name":"internal.metrics.shuffle.read.recordsRead","Update":381,"Value":381,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2680370,"Executor Run Time":10,"Executor CPU Time":7500488,"Peak Execution Memory":67174400,"Result Size":8045,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":1,"Remote Bytes Read":2249,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":381},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":58,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":131,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6181\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[130],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":130,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[129],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[57],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432640913,"Completion Time":1646432640954,"Accumulables":[{"ID":1955,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1957,"Name":"remote bytes read","Value":"2249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1960,"Name":"fetch wait time","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1961,"Name":"records read","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1995,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1996,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1997,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1998,"Name":"cumulative time","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1999,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2000,"Name":"num prefix comparisons","Value":"2773","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2002,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2003,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2004,"Name":"internal.metrics.executorDeserializeTime","Value":3,"Internal":true,"Count Failed Values":true},{"ID":2005,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2680370,"Internal":true,"Count Failed Values":true},{"ID":2006,"Name":"internal.metrics.executorRunTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":2007,"Name":"internal.metrics.executorCpuTime","Value":7500488,"Internal":true,"Count Failed Values":true},{"ID":2008,"Name":"internal.metrics.resultSize","Value":8045,"Internal":true,"Count Failed Values":true},{"ID":2013,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2017,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2018,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2019,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":2249,"Internal":true,"Count Failed Values":true},{"ID":2020,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2021,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2022,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2023,"Name":"internal.metrics.shuffle.read.recordsRead","Value":381,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":32,"Completion Time":1646432640954,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":154,"Index":8,"Attempt":0,"Launch Time":1646432619061,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432642525,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"5674621","Value":"47974036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"23360","Value":"161523","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"5674621","Value":"47974036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1719","Value":"11615","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"178217644","Value":"1477945180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"544763616","Value":"4605507456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"75751912","Value":"634932154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"5674621","Value":"47974036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"187684142","Value":"1648278291","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1760142,"Value":16580656,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":23444,"Value":162245,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":23032791126,"Value":160498591319,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1703,"Value":13622,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":97,"Value":840,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":75751912,"Value":634932154,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5674621,"Value":47974036,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":187684142,"Value":1648278291,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":1477945180,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":47974036,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1719735408,"Value":11619124402,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":1477945180,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":2807472864,"JVMOffHeapMemory":130210920,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4870858,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":76,"MinorGCTime":698,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1760142,"Executor Run Time":23444,"Executor CPU Time":23032791126,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":97,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":75751912,"Shuffle Write Time":187684142,"Shuffle Records Written":5674621},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":147,"Index":1,"Attempt":0,"Launch Time":1646432619060,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432643321,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"6369482","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"24149","Value":"185672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"6369482","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1623","Value":"13238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"197638534","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"611470272","Value":"5216977728","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"85250127","Value":"720182281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"6369482","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"233007693","Value":"1881285984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1959065,"Value":18539721,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":24241,"Value":186486,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":23948961777,"Value":184447553096,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1703,"Value":15325,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":217,"Value":1057,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85250127,"Value":720182281,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6369482,"Value":54343518,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":233007693,"Value":1881285984,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1675583714,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":54343518,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1623171954,"Value":13242296356,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1675583714,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12529158344,"JVMOffHeapMemory":133456136,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4947212,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1582812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":81,"MinorGCTime":1577,"MajorGCCount":5,"MajorGCTime":380},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1959065,"Executor Run Time":24241,"Executor CPU Time":23948961777,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":217,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85250127,"Shuffle Write Time":233007693,"Shuffle Records Written":6369482},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":146,"Index":0,"Attempt":0,"Launch Time":1646432619060,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432646010,"Failed":false,"Killed":false,"Accumulables":[{"ID":1680,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Update":"26824","Value":"212496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Update":"1749","Value":"14987","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Update":"692490816","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Update":"98606155","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Update":"232654747","Value":"2113940731","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1954006,"Value":20493727,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Update":26929,"Value":213415,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Update":26757636560,"Value":211205189656,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Update":1703,"Value":17028,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Update":97,"Value":1154,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":98606155,"Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7213446,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Update":232654747,"Value":2113940731,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1749081107,"Value":14991377463,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12306207584,"JVMOffHeapMemory":130308112,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":4943610,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":543838922,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1592812,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":77,"MinorGCTime":710,"MajorGCCount":5,"MajorGCTime":287},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1954006,"Executor Run Time":26929,"Executor CPU Time":26757636560,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":97,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":98606155,"Shuffle Write Time":232654747,"Shuffle Records Written":7213446},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":46,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":10,"RDD Info":[{"RDD ID":108,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4886\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[107],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4424\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":105,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4424\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4885\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Submission Time":1646432619047,"Completion Time":1646432646010,"Accumulables":[{"ID":1680,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"duration","Value":"212496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1686,"Name":"filesystem read time (sampled)","Value":"14987","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1690,"Name":"data size","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1698,"Name":"shuffle bytes written","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1699,"Name":"shuffle records written","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle write time","Value":"2113940731","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1707,"Name":"internal.metrics.executorDeserializeTime","Value":17,"Internal":true,"Count Failed Values":true},{"ID":1708,"Name":"internal.metrics.executorDeserializeCpuTime","Value":20493727,"Internal":true,"Count Failed Values":true},{"ID":1709,"Name":"internal.metrics.executorRunTime","Value":213415,"Internal":true,"Count Failed Values":true},{"ID":1710,"Name":"internal.metrics.executorCpuTime","Value":211205189656,"Internal":true,"Count Failed Values":true},{"ID":1711,"Name":"internal.metrics.resultSize","Value":17028,"Internal":true,"Count Failed Values":true},{"ID":1712,"Name":"internal.metrics.jvmGCTime","Value":1154,"Internal":true,"Count Failed Values":true},{"ID":1727,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":1728,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1729,"Name":"internal.metrics.shuffle.write.writeTime","Value":2113940731,"Internal":true,"Count Failed Values":true},{"ID":1730,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":1731,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":1732,"Name":"internal.metrics.input.sampledTimeReadNano","Value":14991377463,"Internal":true,"Count Failed Values":true},{"ID":1733,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":26,"Completion Time":1646432646012,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":33,"Submission Time":1646432646365,"Stage Infos":[{"Stage ID":60,"Stage Attempt ID":0,"Stage Name":"treeAggregate at WeightedLeastSquares.scala:107","Number of Tasks":50,"RDD Info":[{"RDD ID":139,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[138],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":136,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"215\",\"name\":\"mapPartitions\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[135],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":138,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[137],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":134,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6438\",\"name\":\"DeserializeToObject\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[133],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":133,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6437\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[132],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":137,"Name":"training instances","Scope":"{\"id\":\"216\",\"name\":\"map\"}","Callsite":"map at Predictor.scala:81","Parent IDs":[136],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":135,"Name":"SQLExecutionRDD","Callsite":"rdd at Predictor.scala:81","Parent IDs":[134],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":132,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_6411\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[108],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[59],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.ml.optim.WeightedLeastSquares.fit(WeightedLeastSquares.scala:107)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:453)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":61,"Stage Attempt ID":0,"Stage Name":"treeAggregate at WeightedLeastSquares.scala:107","Number of Tasks":6,"RDD Info":[{"RDD ID":141,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[140],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":140,"Name":"ShuffledRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[139],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[60],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.ml.optim.WeightedLeastSquares.fit(WeightedLeastSquares.scala:107)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:453)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":59,"Stage Attempt ID":0,"Stage Name":"rdd at Predictor.scala:81","Number of Tasks":10,"RDD Info":[{"RDD ID":108,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4886\",\"name\":\"Exchange\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[107],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4424\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":105,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_4424\",\"name\":\"Scan csv \"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_4885\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.PredictorParams.extractInstances(Predictor.scala:81)\norg.apache.spark.ml.PredictorParams.extractInstances$(Predictor.scala:71)\norg.apache.spark.ml.Predictor.extractInstances(Predictor.scala:115)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:451)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[60,61,59],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":60,"Stage Attempt ID":0,"Stage Name":"treeAggregate at WeightedLeastSquares.scala:107","Number of Tasks":50,"RDD Info":[{"RDD ID":139,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[138],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":136,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"215\",\"name\":\"mapPartitions\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[135],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":138,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[137],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":134,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6438\",\"name\":\"DeserializeToObject\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[133],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":133,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6437\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[132],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":137,"Name":"training instances","Scope":"{\"id\":\"216\",\"name\":\"map\"}","Callsite":"map at Predictor.scala:81","Parent IDs":[136],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":135,"Name":"SQLExecutionRDD","Callsite":"rdd at Predictor.scala:81","Parent IDs":[134],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":132,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_6411\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[108],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[59],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.ml.optim.WeightedLeastSquares.fit(WeightedLeastSquares.scala:107)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:453)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)","Submission Time":1646432646375,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":190,"Index":15,"Attempt":0,"Launch Time":1646432646390,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":191,"Index":10,"Attempt":0,"Launch Time":1646432646390,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":192,"Index":5,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":193,"Index":0,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":194,"Index":16,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":195,"Index":11,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":196,"Index":6,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":197,"Index":1,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":198,"Index":17,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":199,"Index":12,"Attempt":0,"Launch Time":1646432646392,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":200,"Index":7,"Attempt":0,"Launch Time":1646432646392,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":201,"Index":2,"Attempt":0,"Launch Time":1646432646392,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":202,"Index":18,"Attempt":0,"Launch Time":1646432646393,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":203,"Index":13,"Attempt":0,"Launch Time":1646432646394,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":204,"Index":8,"Attempt":0,"Launch Time":1646432646394,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":205,"Index":3,"Attempt":0,"Launch Time":1646432646394,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":206,"Index":19,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":207,"Index":14,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":208,"Index":9,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":209,"Index":4,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":210,"Index":35,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":211,"Index":30,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":212,"Index":25,"Attempt":0,"Launch Time":1646432646396,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":213,"Index":20,"Attempt":0,"Launch Time":1646432646396,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":214,"Index":36,"Attempt":0,"Launch Time":1646432646399,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":215,"Index":31,"Attempt":0,"Launch Time":1646432646400,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":216,"Index":26,"Attempt":0,"Launch Time":1646432646401,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":217,"Index":21,"Attempt":0,"Launch Time":1646432646402,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":218,"Index":37,"Attempt":0,"Launch Time":1646432646402,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":219,"Index":32,"Attempt":0,"Launch Time":1646432646403,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":220,"Index":27,"Attempt":0,"Launch Time":1646432646403,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":221,"Index":22,"Attempt":0,"Launch Time":1646432646404,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":222,"Index":38,"Attempt":0,"Launch Time":1646432646404,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":223,"Index":33,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":224,"Index":28,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":225,"Index":23,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":226,"Index":39,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":227,"Index":34,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":228,"Index":29,"Attempt":0,"Launch Time":1646432646407,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":229,"Index":24,"Attempt":0,"Launch Time":1646432646407,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":230,"Index":45,"Attempt":0,"Launch Time":1646432646407,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":231,"Index":40,"Attempt":0,"Launch Time":1646432646408,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":232,"Index":46,"Attempt":0,"Launch Time":1646432646408,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":233,"Index":41,"Attempt":0,"Launch Time":1646432646409,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":234,"Index":47,"Attempt":0,"Launch Time":1646432646409,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":235,"Index":42,"Attempt":0,"Launch Time":1646432646409,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":236,"Index":48,"Attempt":0,"Launch Time":1646432646410,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":237,"Index":43,"Attempt":0,"Launch Time":1646432646410,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":238,"Index":49,"Attempt":0,"Launch Time":1646432646411,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":239,"Index":44,"Attempt":0,"Launch Time":1646432646411,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":191,"Index":10,"Attempt":0,"Launch Time":1646432646390,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432649572,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"5569222","Value":"5569222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"436341","Value":"436341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"436341","Value":"436341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"436341","Value":"436341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"426737","Value":"426737","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"274","Value":"274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"150994944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"2109","Value":"2109","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"2806335","Value":"2806335","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"2785368","Value":"2785368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"299008","Value":"299008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"2755","Value":"2755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":263,"Value":263,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":94409897,"Value":94409897,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":2872,"Value":2872,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":2580246571,"Value":2580246571,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":8189,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":67,"Value":67,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":352357408,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5569222,"Value":5569222,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":436341,"Value":436341,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":736,"Value":736,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":1360754,"Value":1360754,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":263,"Executor Deserialize CPU Time":94409897,"Executor Run Time":2872,"Executor CPU Time":2580246571,"Peak Execution Memory":352357408,"Result Size":8189,"JVM GC Time":67,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5569222,"Total Records Read":436341},"Shuffle Write Metrics":{"Shuffle Bytes Written":736,"Shuffle Write Time":1360754,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":210,"Index":35,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432649985,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"5007038","Value":"10576260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"385946","Value":"822287","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"385946","Value":"822287","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"385946","Value":"822287","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"376002","Value":"802739","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"7","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"189","Value":"463","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"301989888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"2488","Value":"4597","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"2450583","Value":"5256918","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"2429287","Value":"5214655","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"263017","Value":"562025","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"3014","Value":"5769","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":390,"Value":653,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":116346521,"Value":210756418,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":3157,"Value":6029,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":2703249638,"Value":5283496209,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":16377,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":132,"Value":199,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":704714816,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5007038,"Value":10576260,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":385946,"Value":822287,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":735,"Value":1471,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":1650594,"Value":3011348,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":390,"Executor Deserialize CPU Time":116346521,"Executor Run Time":3157,"Executor CPU Time":2703249638,"Peak Execution Memory":352357408,"Result Size":8188,"JVM GC Time":132,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5007038,"Total Records Read":385946},"Shuffle Write Metrics":{"Shuffle Bytes Written":735,"Shuffle Write Time":1650594,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":190,"Index":15,"Attempt":0,"Launch Time":1646432646390,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432650115,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"5549390","Value":"16125650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"443853","Value":"1266140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"443853","Value":"1266140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"443853","Value":"1266140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"436895","Value":"1239634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"500","Value":"963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"452984832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"2420","Value":"7017","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"2885625","Value":"8142543","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"2866006","Value":"8080661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"306336","Value":"868361","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"3186","Value":"8955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":388,"Value":1041,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":117449870,"Value":328206288,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":3292,"Value":9321,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":2929330943,"Value":8212827152,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":24565,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":132,"Value":331,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":1057072224,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5549390,"Value":16125650,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":443853,"Value":1266140,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":735,"Value":2206,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":140473,"Value":3151821,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":388,"Executor Deserialize CPU Time":117449870,"Executor Run Time":3292,"Executor CPU Time":2929330943,"Peak Execution Memory":352357408,"Result Size":8188,"JVM GC Time":132,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5549390,"Total Records Read":443853},"Shuffle Write Metrics":{"Shuffle Bytes Written":735,"Shuffle Write Time":140473,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":211,"Index":30,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432650265,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"5197262","Value":"21322912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"403027","Value":"1669167","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"403027","Value":"1669167","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"403027","Value":"1669167","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"395697","Value":"1635331","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"332","Value":"1295","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"603979776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"2730","Value":"9747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"2671694","Value":"10814237","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"2648851","Value":"10729512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"277153","Value":"1145514","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"3481","Value":"12436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":264,"Value":1305,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":108812322,"Value":437018610,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":3565,"Value":12886,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":2966269988,"Value":11179097140,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":32754,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":390,"Value":721,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":1409429632,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5197262,"Value":21322912,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":403027,"Value":1669167,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":728,"Value":2934,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":220814,"Value":3372635,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":264,"Executor Deserialize CPU Time":108812322,"Executor Run Time":3565,"Executor CPU Time":2966269988,"Peak Execution Memory":352357408,"Result Size":8189,"JVM GC Time":390,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5197262,"Total Records Read":403027},"Shuffle Write Metrics":{"Shuffle Bytes Written":728,"Shuffle Write Time":220814,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":231,"Index":40,"Attempt":0,"Launch Time":1646432646408,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432651042,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"5138578","Value":"26461490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"398664","Value":"2067831","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"398664","Value":"2067831","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"398664","Value":"2067831","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"392344","Value":"2027675","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"397","Value":"1692","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"754974720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"3020","Value":"12767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"2612328","Value":"13426565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"2589790","Value":"13319302","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"275006","Value":"1420520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"4126","Value":"16562","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":352,"Value":1657,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":68077563,"Value":505096173,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":4237,"Value":17123,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":3330328938,"Value":14509426078,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":40943,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":470,"Value":1191,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":1761787040,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5138578,"Value":26461490,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":398664,"Value":2067831,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":735,"Value":3669,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":1258885,"Value":4631520,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":352,"Executor Deserialize CPU Time":68077563,"Executor Run Time":4237,"Executor CPU Time":3330328938,"Peak Execution Memory":352357408,"Result Size":8189,"JVM GC Time":470,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5138578,"Total Records Read":398664},"Shuffle Write Metrics":{"Shuffle Bytes Written":735,"Shuffle Write Time":1258885,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":213,"Index":20,"Attempt":0,"Launch Time":1646432646396,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432651047,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"5077427","Value":"31538917","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"397065","Value":"2464896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"397065","Value":"2464896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"397065","Value":"2464896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"388533","Value":"2416208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"455","Value":"2147","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"905969664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"3023","Value":"15790","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"2526318","Value":"15952883","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"2507081","Value":"15826383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"272002","Value":"1692522","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"4142","Value":"20704","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":355,"Value":2012,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":92084252,"Value":597180425,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":4239,"Value":21362,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":3262505869,"Value":17771931947,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":49132,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":470,"Value":1661,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":2114144448,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5077427,"Value":31538917,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":397065,"Value":2464896,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":732,"Value":4401,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":110843,"Value":4742363,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":355,"Executor Deserialize CPU Time":92084252,"Executor Run Time":4239,"Executor CPU Time":3262505869,"Peak Execution Memory":352357408,"Result Size":8189,"JVM GC Time":470,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5077427,"Total Records Read":397065},"Shuffle Write Metrics":{"Shuffle Bytes Written":732,"Shuffle Write Time":110843,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":230,"Index":45,"Attempt":0,"Launch Time":1646432646407,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432651087,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"4974479","Value":"36513396","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"387227","Value":"2852123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"387227","Value":"2852123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"6","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"387227","Value":"2852123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"381516","Value":"2797724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"47","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"847","Value":"2994","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"1056964608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"3454","Value":"19244","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"2458158","Value":"18411041","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"2436258","Value":"18262641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"267201","Value":"1959723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"4192","Value":"24896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":333,"Value":2345,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":63369816,"Value":660550241,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":4315,"Value":25677,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":3410235628,"Value":21182167575,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":57321,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":473,"Value":2134,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":2466501856,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":4974479,"Value":36513396,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":387227,"Value":2852123,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":734,"Value":5135,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":1365015,"Value":6107378,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":333,"Executor Deserialize CPU Time":63369816,"Executor Run Time":4315,"Executor CPU Time":3410235628,"Peak Execution Memory":352357408,"Result Size":8189,"JVM GC Time":473,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":4974479,"Total Records Read":387227},"Shuffle Write Metrics":{"Shuffle Bytes Written":734,"Shuffle Write Time":1365015,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":192,"Index":5,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432651126,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"5506047","Value":"42019443","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"425591","Value":"3277714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"425591","Value":"3277714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"5","Value":"65","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"425591","Value":"3277714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"418065","Value":"3215789","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"53","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"398","Value":"3392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"1207959552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"3611","Value":"22855","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"2784193","Value":"21195234","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"2761418","Value":"21024059","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"292397","Value":"2252120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"4268","Value":"29164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":334,"Value":2679,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":68475915,"Value":729026156,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":4353,"Value":30030,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":3357290920,"Value":24539458495,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":65510,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":473,"Value":2607,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":2818859264,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5506047,"Value":42019443,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":425591,"Value":3277714,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":736,"Value":5871,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":108692,"Value":6216070,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":334,"Executor Deserialize CPU Time":68475915,"Executor Run Time":4353,"Executor CPU Time":3357290920,"Peak Execution Memory":352357408,"Result Size":8189,"JVM GC Time":473,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5506047,"Total Records Read":425591},"Shuffle Write Metrics":{"Shuffle Bytes Written":736,"Shuffle Write Time":108692,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":212,"Index":25,"Attempt":0,"Launch Time":1646432646396,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432651126,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"5062619","Value":"47082062","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"397041","Value":"3674755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"397041","Value":"3674755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"5","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"397041","Value":"3674755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"390517","Value":"3606306","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"451","Value":"3843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"1358954496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"3510","Value":"26365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"2548570","Value":"23743804","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"2526559","Value":"23550618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"273437","Value":"2525557","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"4268","Value":"33432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":333,"Value":3012,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":82807850,"Value":811834006,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":4355,"Value":34385,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":3271695818,"Value":27811154313,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":73699,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":473,"Value":3080,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":3171216672,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5062619,"Value":47082062,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":397041,"Value":3674755,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":730,"Value":6601,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":130339,"Value":6346409,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":333,"Executor Deserialize CPU Time":82807850,"Executor Run Time":4355,"Executor CPU Time":3271695818,"Peak Execution Memory":352357408,"Result Size":8189,"JVM GC Time":473,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5062619,"Total Records Read":397041},"Shuffle Write Metrics":{"Shuffle Bytes Written":730,"Shuffle Write Time":130339,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":193,"Index":0,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432651667,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"7603473","Value":"54685535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"566756","Value":"4241511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"566756","Value":"4241511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"566756","Value":"4241511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"555159","Value":"4161465","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"68","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"487","Value":"4330","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"150994944","Value":"1509949440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"4059","Value":"30424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"3939232","Value":"27683036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"3914157","Value":"27464775","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"388770","Value":"2914327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"4801","Value":"38233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":354,"Value":3366,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":77270187,"Value":889104193,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":4877,"Value":39262,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":4071718808,"Value":31882873121,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":81888,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":470,"Value":3550,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":352357408,"Value":3523574080,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":7603473,"Value":54685535,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":566756,"Value":4241511,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":735,"Value":7336,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":111755,"Value":6458164,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":354,"Executor Deserialize CPU Time":77270187,"Executor Run Time":4877,"Executor CPU Time":4071718808,"Peak Execution Memory":352357408,"Result Size":8189,"JVM GC Time":470,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":7603473,"Total Records Read":566756},"Shuffle Write Metrics":{"Shuffle Bytes Written":735,"Shuffle Write Time":111755,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":215,"Index":31,"Attempt":0,"Launch Time":1646432646400,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432652553,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"14179385","Value":"68864920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1069599","Value":"5311110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1069599","Value":"5311110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"87","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1069599","Value":"5311110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1050288","Value":"5211753","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"73","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"719","Value":"5049","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"1811939328","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"4946","Value":"35370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7084874","Value":"34767910","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7053239","Value":"34518014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"734965","Value":"3649292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"5770","Value":"44003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":265,"Value":3631,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":92967454,"Value":982071647,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":5853,"Value":45115,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":5167360679,"Value":37050233800,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":90077,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":506,"Value":4056,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":4026926432,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14179385,"Value":68864920,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1069599,"Value":5311110,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":741,"Value":8077,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":62135,"Value":6520299,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":265,"Executor Deserialize CPU Time":92967454,"Executor Run Time":5853,"Executor CPU Time":5167360679,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":506,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14179385,"Total Records Read":1069599},"Shuffle Write Metrics":{"Shuffle Bytes Written":741,"Shuffle Write Time":62135,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":219,"Index":32,"Attempt":0,"Launch Time":1646432646403,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432652661,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"16283262","Value":"85148182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1222489","Value":"6533599","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1222489","Value":"6533599","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"95","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1222489","Value":"6533599","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1207733","Value":"6419486","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"4","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"801","Value":"5850","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"2113929216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"4933","Value":"40303","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7750719","Value":"42518629","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7719792","Value":"42237806","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"844923","Value":"4494215","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"5878","Value":"49881","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":263,"Value":3894,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":79412662,"Value":1061484309,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":5962,"Value":51077,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":5288911565,"Value":42339145365,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":98266,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":506,"Value":4562,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":4530278784,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16283262,"Value":85148182,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1222489,"Value":6533599,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":748,"Value":8825,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":64119,"Value":6584418,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":263,"Executor Deserialize CPU Time":79412662,"Executor Run Time":5962,"Executor CPU Time":5288911565,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":506,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16283262,"Total Records Read":1222489},"Shuffle Write Metrics":{"Shuffle Bytes Written":748,"Shuffle Write Time":64119,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":195,"Index":11,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432652879,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"15063284","Value":"100211466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1163966","Value":"7697565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1163966","Value":"7697565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"103","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1163966","Value":"7697565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1143328","Value":"7562814","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"4","Value":"81","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"637","Value":"6487","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"2415919104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"5240","Value":"45543","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7298870","Value":"49817499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7269686","Value":"49507492","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"800303","Value":"5294518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"6097","Value":"55978","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":264,"Value":4158,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":85100186,"Value":1146584495,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":6181,"Value":57258,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":5447334286,"Value":47786479651,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":106455,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":506,"Value":5068,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":5033631136,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":15063284,"Value":100211466,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1163966,"Value":7697565,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":742,"Value":9567,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":78059,"Value":6662477,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":264,"Executor Deserialize CPU Time":85100186,"Executor Run Time":6181,"Executor CPU Time":5447334286,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":506,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":15063284,"Total Records Read":1163966},"Shuffle Write Metrics":{"Shuffle Bytes Written":742,"Shuffle Write Time":78059,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":199,"Index":12,"Attempt":0,"Launch Time":1646432646392,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432653018,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"17852704","Value":"118064170","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1354716","Value":"9052281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1354716","Value":"9052281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"111","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1354716","Value":"9052281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1335669","Value":"8898483","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"4","Value":"85","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"738","Value":"7225","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"2717908992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"5250","Value":"50793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"8042908","Value":"57860407","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"8012151","Value":"57519643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"934847","Value":"6229365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"6234","Value":"62212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":262,"Value":4420,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":82763061,"Value":1229347556,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":6320,"Value":63578,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":5609051453,"Value":53395531104,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":114644,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":506,"Value":5574,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":5536983488,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17852704,"Value":118064170,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1354716,"Value":9052281,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":739,"Value":10306,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":72695,"Value":6735172,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":262,"Executor Deserialize CPU Time":82763061,"Executor Run Time":6320,"Executor CPU Time":5609051453,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":506,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17852704,"Total Records Read":1354716},"Shuffle Write Metrics":{"Shuffle Bytes Written":739,"Shuffle Write Time":72695,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":218,"Index":37,"Attempt":0,"Launch Time":1646432646402,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432653157,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"16812061","Value":"134876231","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1252080","Value":"10304361","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1252080","Value":"10304361","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1252080","Value":"10304361","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1230187","Value":"10128670","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"91","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"708","Value":"7933","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"3019898880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"5070","Value":"55863","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7918049","Value":"65778456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7886574","Value":"65406217","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"861213","Value":"7090578","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"6228","Value":"68440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":390,"Value":4810,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":137732905,"Value":1367080461,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":6330,"Value":69908,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":5392012253,"Value":58787543357,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":122832,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":622,"Value":6196,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":6040335840,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16812061,"Value":134876231,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1252080,"Value":10304361,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":744,"Value":11050,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":89766,"Value":6824938,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":390,"Executor Deserialize CPU Time":137732905,"Executor Run Time":6330,"Executor CPU Time":5392012253,"Peak Execution Memory":503352352,"Result Size":8188,"JVM GC Time":622,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16812061,"Total Records Read":1252080},"Shuffle Write Metrics":{"Shuffle Bytes Written":744,"Shuffle Write Time":89766,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":214,"Index":36,"Attempt":0,"Launch Time":1646432646399,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432653172,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"14286711","Value":"149162942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1075303","Value":"11379664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1075303","Value":"11379664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"131","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1075303","Value":"11379664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1050388","Value":"11179058","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"97","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"559","Value":"8492","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"3321888768","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"5201","Value":"61064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7028804","Value":"72807260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"6997233","Value":"72403450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"735414","Value":"7825992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"6245","Value":"74685","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":388,"Value":5198,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":133900222,"Value":1500980683,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":6350,"Value":76258,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":5380597509,"Value":64168140866,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":131020,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":622,"Value":6818,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":6543688192,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14286711,"Value":149162942,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1075303,"Value":11379664,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":743,"Value":11793,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":83133,"Value":6908071,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":388,"Executor Deserialize CPU Time":133900222,"Executor Run Time":6350,"Executor CPU Time":5380597509,"Peak Execution Memory":503352352,"Result Size":8188,"JVM GC Time":622,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14286711,"Total Records Read":1075303},"Shuffle Write Metrics":{"Shuffle Bytes Written":743,"Shuffle Write Time":83133,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":194,"Index":16,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432653538,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"15206714","Value":"164369656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1185888","Value":"12565552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1185888","Value":"12565552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"141","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1185888","Value":"12565552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1167901","Value":"12346959","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"103","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"632","Value":"9124","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"3623878656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"5488","Value":"66552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7540656","Value":"80347916","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7513451","Value":"79916901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"817587","Value":"8643579","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"6610","Value":"81295","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":387,"Value":5585,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":114550200,"Value":1615530883,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":6716,"Value":82974,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":5794697169,"Value":69962838035,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":139208,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":622,"Value":7440,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":7047040544,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":15206714,"Value":164369656,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1185888,"Value":12565552,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":742,"Value":12535,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":111227,"Value":7019298,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":387,"Executor Deserialize CPU Time":114550200,"Executor Run Time":6716,"Executor CPU Time":5794697169,"Peak Execution Memory":503352352,"Result Size":8188,"JVM GC Time":622,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":15206714,"Total Records Read":1185888},"Shuffle Write Metrics":{"Shuffle Bytes Written":742,"Shuffle Write Time":111227,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":198,"Index":17,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432653629,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"17279386","Value":"181649042","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1325325","Value":"13890877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1325325","Value":"13890877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"151","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1325325","Value":"13890877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1311802","Value":"13658761","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"7","Value":"110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"653","Value":"9777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"3925868544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"5461","Value":"72013","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"8008126","Value":"88356042","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7980301","Value":"87897202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"919517","Value":"9563096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"6701","Value":"87996","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":388,"Value":5973,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":138751235,"Value":1754282118,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":6806,"Value":89780,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":5868791468,"Value":75831629503,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":147396,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":622,"Value":8062,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":7550392896,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17279386,"Value":181649042,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1325325,"Value":13890877,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":740,"Value":13275,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":91187,"Value":7110485,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":388,"Executor Deserialize CPU Time":138751235,"Executor Run Time":6806,"Executor CPU Time":5868791468,"Peak Execution Memory":503352352,"Result Size":8188,"JVM GC Time":622,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17279386,"Total Records Read":1325325},"Shuffle Write Metrics":{"Shuffle Bytes Written":740,"Shuffle Write Time":91187,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":223,"Index":33,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654118,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"21849278","Value":"203498320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1616064","Value":"15506941","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1616064","Value":"15506941","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1616064","Value":"15506941","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1590300","Value":"15249061","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"4","Value":"114","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"849","Value":"10626","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"4328521728","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6174","Value":"78187","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"11739245","Value":"100095287","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"11710528","Value":"99607730","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1113704","Value":"10676800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7336","Value":"95332","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":263,"Value":6236,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":89591298,"Value":1843873416,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":7420,"Value":97200,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":6704134039,"Value":82535763542,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":155585,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":506,"Value":8568,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":8154408544,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21849278,"Value":203498320,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1616064,"Value":15506941,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":744,"Value":14019,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":61044,"Value":7171529,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":263,"Executor Deserialize CPU Time":89591298,"Executor Run Time":7420,"Executor CPU Time":6704134039,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":506,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21849278,"Total Records Read":1616064},"Shuffle Write Metrics":{"Shuffle Bytes Written":744,"Shuffle Write Time":61044,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":222,"Index":38,"Attempt":0,"Launch Time":1646432646404,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654353,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"21789512","Value":"225287832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1603400","Value":"17110341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1603400","Value":"17110341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1603400","Value":"17110341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1564114","Value":"16813175","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"7","Value":"121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"938","Value":"11564","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"4731174912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6132","Value":"84319","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"11511301","Value":"111606588","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"11482858","Value":"111090588","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1096067","Value":"11772867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7424","Value":"102756","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":387,"Value":6623,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":117439518,"Value":1961312934,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":7529,"Value":104729,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":6599684293,"Value":89135447835,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":163773,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":622,"Value":9190,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":8758424192,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21789512,"Value":225287832,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1603400,"Value":17110341,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":744,"Value":14763,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":79610,"Value":7251139,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7426825400,"JVMOffHeapMemory":140009040,"OnHeapExecutionMemory":1677721600,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1816984671,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1407631,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":77,"MinorGCTime":1115,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":387,"Executor Deserialize CPU Time":117439518,"Executor Run Time":7529,"Executor CPU Time":6599684293,"Peak Execution Memory":604015648,"Result Size":8188,"JVM GC Time":622,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21789512,"Total Records Read":1603400},"Shuffle Write Metrics":{"Shuffle Bytes Written":744,"Shuffle Write Time":79610,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":227,"Index":34,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654362,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"20325382","Value":"245613214","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1507900","Value":"18618241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1507900","Value":"18618241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1507900","Value":"18618241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1488908","Value":"18302083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"4","Value":"125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"761","Value":"12325","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"5133828096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6469","Value":"90788","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"10001165","Value":"121607753","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"9964616","Value":"121055204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1042165","Value":"12815032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7580","Value":"110336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":265,"Value":6888,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":101560793,"Value":2062873727,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":7663,"Value":112392,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":6938168971,"Value":96073616806,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":171962,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":506,"Value":9696,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":9362439840,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20325382,"Value":245613214,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1507900,"Value":18618241,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":744,"Value":15507,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":107053,"Value":7358192,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":265,"Executor Deserialize CPU Time":101560793,"Executor Run Time":7663,"Executor CPU Time":6938168971,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":506,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20325382,"Total Records Read":1507900},"Shuffle Write Metrics":{"Shuffle Bytes Written":744,"Shuffle Write Time":107053,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":226,"Index":39,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654415,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"20255703","Value":"265868917","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1503082","Value":"20121323","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1503082","Value":"20121323","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"187","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1503082","Value":"20121323","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1476378","Value":"19778461","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"131","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"872","Value":"13197","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"5536481280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6232","Value":"97020","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"9929413","Value":"131537166","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"9892652","Value":"130947856","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1033506","Value":"13848538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7486","Value":"117822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":391,"Value":7279,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":99470016,"Value":2162343743,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":7589,"Value":119981,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":6657823515,"Value":102731440321,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":180150,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":622,"Value":10318,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":9966455488,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20255703,"Value":265868917,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1503082,"Value":20121323,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":745,"Value":16252,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":100491,"Value":7458683,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7426825400,"JVMOffHeapMemory":140009040,"OnHeapExecutionMemory":1677721600,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1816984671,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1407631,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":77,"MinorGCTime":1115,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":391,"Executor Deserialize CPU Time":99470016,"Executor Run Time":7589,"Executor CPU Time":6657823515,"Peak Execution Memory":604015648,"Result Size":8188,"JVM GC Time":622,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20255703,"Total Records Read":1503082},"Shuffle Write Metrics":{"Shuffle Bytes Written":745,"Shuffle Write Time":100491,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":203,"Index":13,"Attempt":0,"Launch Time":1646432646394,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654475,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"23718187","Value":"289587104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1791015","Value":"21912338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1791015","Value":"21912338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"195","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1791015","Value":"21912338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1758874","Value":"21537335","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"4","Value":"135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"773","Value":"13970","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"469762048","Value":"6006243328","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6436","Value":"103456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"11937551","Value":"143474717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"11907003","Value":"142854859","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1231326","Value":"15079864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7692","Value":"125514","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":263,"Value":7542,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":77693344,"Value":2240037087,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":7777,"Value":127758,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":7060344754,"Value":109791785075,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":188339,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":506,"Value":10824,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":671124512,"Value":10637580000,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":23718187,"Value":289587104,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1791015,"Value":21912338,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":745,"Value":16997,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":86213,"Value":7544896,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":263,"Executor Deserialize CPU Time":77693344,"Executor Run Time":7777,"Executor CPU Time":7060344754,"Peak Execution Memory":671124512,"Result Size":8189,"JVM GC Time":506,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":23718187,"Total Records Read":1791015},"Shuffle Write Metrics":{"Shuffle Bytes Written":745,"Shuffle Write Time":86213,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":207,"Index":14,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654658,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"22521225","Value":"312108329","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1704079","Value":"23616417","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1704079","Value":"23616417","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"8","Value":"203","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1704079","Value":"23616417","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1676732","Value":"23214067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"827","Value":"14797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"6408896512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6660","Value":"110116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"10756664","Value":"154231381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"10719584","Value":"153574443","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1174256","Value":"16254120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7876","Value":"133390","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":265,"Value":7807,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":129569548,"Value":2369606635,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":7959,"Value":135717,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":7244824197,"Value":117036609272,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":196528,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":506,"Value":11330,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":11241595648,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22521225,"Value":312108329,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1704079,"Value":23616417,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":745,"Value":17742,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":84546,"Value":7629442,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":265,"Executor Deserialize CPU Time":129569548,"Executor Run Time":7959,"Executor CPU Time":7244824197,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":506,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22521225,"Total Records Read":1704079},"Shuffle Write Metrics":{"Shuffle Bytes Written":745,"Shuffle Write Time":84546,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":197,"Index":1,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654699,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"18420350","Value":"330528679","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1358796","Value":"24975213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1358796","Value":"24975213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1358796","Value":"24975213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1331797","Value":"24545864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"973","Value":"15770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"6710886400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6864","Value":"116980","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"9392292","Value":"163623673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"9357092","Value":"162931535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"932727","Value":"17186847","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7832","Value":"141222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":348,"Value":8155,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":88229249,"Value":2457835884,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":7911,"Value":143628,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":6645573821,"Value":123682183093,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":204717,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":884,"Value":12214,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":11744948000,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":18420350,"Value":330528679,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1358796,"Value":24975213,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":741,"Value":18483,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":110237,"Value":7739679,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":348,"Executor Deserialize CPU Time":88229249,"Executor Run Time":7911,"Executor CPU Time":6645573821,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":884,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":18420350,"Total Records Read":1358796},"Shuffle Write Metrics":{"Shuffle Bytes Written":741,"Shuffle Write Time":110237,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":217,"Index":21,"Attempt":0,"Launch Time":1646432646402,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654761,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"14140207","Value":"344668886","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1091142","Value":"26066355","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1091142","Value":"26066355","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1091142","Value":"26066355","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1068266","Value":"25614130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"1139","Value":"16909","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"7012876288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6614","Value":"123594","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"6841900","Value":"170465573","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"6814278","Value":"169745813","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"747499","Value":"17934346","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7895","Value":"149117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":349,"Value":8504,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":63958543,"Value":2521794427,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":7973,"Value":151601,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":6734390936,"Value":130416574029,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":212906,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":884,"Value":13098,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":12248300352,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14140207,"Value":344668886,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1091142,"Value":26066355,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":740,"Value":19223,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":75610,"Value":7815289,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":349,"Executor Deserialize CPU Time":63958543,"Executor Run Time":7973,"Executor CPU Time":6734390936,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":884,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14140207,"Total Records Read":1091142},"Shuffle Write Metrics":{"Shuffle Bytes Written":740,"Shuffle Write Time":75610,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":233,"Index":41,"Attempt":0,"Launch Time":1646432646409,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654837,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"13740133","Value":"358409019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1035151","Value":"27101506","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1035151","Value":"27101506","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1035151","Value":"27101506","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1016060","Value":"26630190","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"170","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"1090","Value":"17999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"234881024","Value":"7247757312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6754","Value":"130348","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7046687","Value":"177512260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7013519","Value":"176759332","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"711445","Value":"18645791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7971","Value":"157088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":352,"Value":8856,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":84288510,"Value":2606082937,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":8047,"Value":159648,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":6844371369,"Value":137260945398,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":221095,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":884,"Value":13982,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":436243488,"Value":12684543840,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":13740133,"Value":358409019,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1035151,"Value":27101506,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":742,"Value":19965,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":78698,"Value":7893987,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":352,"Executor Deserialize CPU Time":84288510,"Executor Run Time":8047,"Executor CPU Time":6844371369,"Peak Execution Memory":436243488,"Result Size":8189,"JVM GC Time":884,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":13740133,"Total Records Read":1035151},"Shuffle Write Metrics":{"Shuffle Bytes Written":742,"Shuffle Write Time":78698,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":206,"Index":19,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432654897,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"22027093","Value":"380436112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1697367","Value":"28798873","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1697367","Value":"28798873","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1697367","Value":"28798873","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1677876","Value":"28308066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"991","Value":"18990","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"7650410496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6600","Value":"136948","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"10836844","Value":"188349104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"10801236","Value":"187560568","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1175348","Value":"19821139","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"7969","Value":"165057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":391,"Value":9247,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":111832616,"Value":2717915553,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":8071,"Value":167719,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":7123801576,"Value":144384746974,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":229283,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":622,"Value":14604,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":13288559488,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22027093,"Value":380436112,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1697367,"Value":28798873,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":744,"Value":20709,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":85607,"Value":7979594,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7426825400,"JVMOffHeapMemory":140009040,"OnHeapExecutionMemory":1677721600,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1816984671,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1407631,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":77,"MinorGCTime":1115,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":391,"Executor Deserialize CPU Time":111832616,"Executor Run Time":8071,"Executor CPU Time":7123801576,"Peak Execution Memory":604015648,"Result Size":8188,"JVM GC Time":622,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22027093,"Total Records Read":1697367},"Shuffle Write Metrics":{"Shuffle Bytes Written":744,"Shuffle Write Time":85607,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":216,"Index":26,"Attempt":0,"Launch Time":1646432646401,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432655062,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"14350655","Value":"394786767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1106404","Value":"29905277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1106404","Value":"29905277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"6","Value":"246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1106404","Value":"29905277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1090201","Value":"29398267","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"181","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"941","Value":"19931","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"7952400384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"7042","Value":"143990","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7013679","Value":"195362783","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"6983027","Value":"194543595","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"762998","Value":"20584137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"8206","Value":"173263","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":331,"Value":9578,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":73260187,"Value":2791175740,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":8295,"Value":176014,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":7298268504,"Value":151683015478,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":237472,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":539,"Value":15143,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":13791911840,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14350655,"Value":394786767,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1106404,"Value":29905277,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":741,"Value":21450,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":76950,"Value":8056544,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":142985088,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1612345,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":1781,"MajorGCCount":6,"MajorGCTime":649},"Task Metrics":{"Executor Deserialize Time":331,"Executor Deserialize CPU Time":73260187,"Executor Run Time":8295,"Executor CPU Time":7298268504,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":539,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14350655,"Total Records Read":1106404},"Shuffle Write Metrics":{"Shuffle Bytes Written":741,"Shuffle Write Time":76950,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":202,"Index":18,"Attempt":0,"Launch Time":1646432646393,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432655307,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"30","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"23325788","Value":"418112555","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1776905","Value":"31682182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1776905","Value":"31682182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"10","Value":"256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1776905","Value":"31682182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1751997","Value":"31150264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"7","Value":"188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"840","Value":"20771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"469762048","Value":"8422162432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"6994","Value":"150984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"11893096","Value":"207255879","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"11865777","Value":"206409372","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1226396","Value":"21810533","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"8379","Value":"181642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":391,"Value":9969,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":96360202,"Value":2887535942,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":8481,"Value":184495,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":7556323712,"Value":159239339190,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8188,"Value":245660,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":622,"Value":15765,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":671124512,"Value":14463036352,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":23325788,"Value":418112555,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1776905,"Value":31682182,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":744,"Value":22194,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":114527,"Value":8171071,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7426825400,"JVMOffHeapMemory":140009040,"OnHeapExecutionMemory":1677721600,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1816984671,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1407631,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":77,"MinorGCTime":1115,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":391,"Executor Deserialize CPU Time":96360202,"Executor Run Time":8481,"Executor CPU Time":7556323712,"Peak Execution Memory":671124512,"Result Size":8188,"JVM GC Time":622,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":23325788,"Total Records Read":1776905},"Shuffle Write Metrics":{"Shuffle Bytes Written":744,"Shuffle Write Time":114527,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":235,"Index":42,"Attempt":0,"Launch Time":1646432646409,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432655454,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"31","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"16633555","Value":"434746110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1252125","Value":"32934307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1252125","Value":"32934307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1252125","Value":"32934307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1237479","Value":"32387743","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"1237","Value":"22008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"8724152320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"7258","Value":"158242","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7978094","Value":"215233973","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7948557","Value":"214357929","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"866570","Value":"22677103","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"8587","Value":"190229","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":349,"Value":10318,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":69420801,"Value":2956956743,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":8666,"Value":193161,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":7298890735,"Value":166538229925,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":253849,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1007,"Value":16772,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":14966388704,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":31,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16633555,"Value":434746110,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1252125,"Value":32934307,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":745,"Value":22939,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":31,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":80088,"Value":8251159,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17755499528,"JVMOffHeapMemory":141156128,"OnHeapExecutionMemory":3489660928,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3628923999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1619695,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1221,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":349,"Executor Deserialize CPU Time":69420801,"Executor Run Time":8666,"Executor CPU Time":7298890735,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":1007,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16633555,"Total Records Read":1252125},"Shuffle Write Metrics":{"Shuffle Bytes Written":745,"Shuffle Write Time":80088,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":239,"Index":44,"Attempt":0,"Launch Time":1646432646411,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432655521,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"20467942","Value":"455214052","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1520355","Value":"34454662","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1520355","Value":"34454662","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1520355","Value":"34454662","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1500662","Value":"33888405","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"709","Value":"22717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"9126805504","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"7442","Value":"165684","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"10159321","Value":"225393294","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"10122495","Value":"224480424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1049813","Value":"23726916","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"8654","Value":"198883","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":352,"Value":10670,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":98471042,"Value":3055427785,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":8730,"Value":201891,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":7352909230,"Value":173891139155,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":262038,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1007,"Value":17779,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":15570404352,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20467942,"Value":455214052,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1520355,"Value":34454662,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":745,"Value":23684,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":56802,"Value":8307961,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17755499528,"JVMOffHeapMemory":141156128,"OnHeapExecutionMemory":3489660928,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3628923999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1619695,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1221,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":352,"Executor Deserialize CPU Time":98471042,"Executor Run Time":8730,"Executor CPU Time":7352909230,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1007,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20467942,"Total Records Read":1520355},"Shuffle Write Metrics":{"Shuffle Bytes Written":745,"Shuffle Write Time":56802,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":221,"Index":22,"Attempt":0,"Launch Time":1646432646404,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432655809,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"17648290","Value":"472862342","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1330648","Value":"35785310","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1330648","Value":"35785310","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"283","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1330648","Value":"35785310","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1310953","Value":"35199358","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"218","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"944","Value":"23661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"9428795392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"7730","Value":"173414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"8269956","Value":"233663250","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"8241301","Value":"232721725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"917559","Value":"24644475","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"8943","Value":"207826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":353,"Value":11023,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":74578895,"Value":3130006680,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":9018,"Value":210909,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":7688745749,"Value":181579884904,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":270227,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1007,"Value":18786,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":16073756704,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17648290,"Value":472862342,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1330648,"Value":35785310,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":742,"Value":24426,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":66427,"Value":8374388,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17755499528,"JVMOffHeapMemory":141156128,"OnHeapExecutionMemory":3489660928,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3628923999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1619695,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1221,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":353,"Executor Deserialize CPU Time":74578895,"Executor Run Time":9018,"Executor CPU Time":7688745749,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":1007,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17648290,"Total Records Read":1330648},"Shuffle Write Metrics":{"Shuffle Bytes Written":742,"Shuffle Write Time":66427,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":201,"Index":2,"Attempt":0,"Launch Time":1646432646392,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432655953,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"21741695","Value":"494604037","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1592591","Value":"37377901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1592591","Value":"37377901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1592591","Value":"37377901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1570179","Value":"36769537","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"228","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"751","Value":"24412","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"9831448576","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"7732","Value":"181146","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"10699132","Value":"244362382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"10663812","Value":"243385537","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1098273","Value":"25742748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"9086","Value":"216912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":349,"Value":11372,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":77035871,"Value":3207042551,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":9164,"Value":220073,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":7799167032,"Value":189379051936,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":278416,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1007,"Value":19793,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":16677772352,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21741695,"Value":494604037,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1592591,"Value":37377901,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":746,"Value":25172,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":61513,"Value":8435901,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17755499528,"JVMOffHeapMemory":141156128,"OnHeapExecutionMemory":3489660928,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3628923999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1619695,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1221,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":349,"Executor Deserialize CPU Time":77035871,"Executor Run Time":9164,"Executor CPU Time":7799167032,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1007,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21741695,"Total Records Read":1592591},"Shuffle Write Metrics":{"Shuffle Bytes Written":746,"Shuffle Write Time":61513,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":237,"Index":43,"Attempt":0,"Launch Time":1646432646410,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432656327,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"19771704","Value":"514375741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1468326","Value":"38846227","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1468326","Value":"38846227","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1468326","Value":"38846227","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1447733","Value":"38217270","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"907","Value":"25319","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"10234101760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"8436","Value":"189582","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"10577490","Value":"254939872","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"10549548","Value":"253935085","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1013026","Value":"26755774","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"9462","Value":"226374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":348,"Value":11720,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":78740428,"Value":3285782979,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":9542,"Value":229615,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":8116205332,"Value":197495257268,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":286605,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1007,"Value":20800,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":17281788000,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":19771704,"Value":514375741,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1468326,"Value":38846227,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":743,"Value":25915,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":61959,"Value":8497860,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17755499528,"JVMOffHeapMemory":141156128,"OnHeapExecutionMemory":3489660928,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3628923999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1619695,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1221,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":348,"Executor Deserialize CPU Time":78740428,"Executor Run Time":9542,"Executor CPU Time":8116205332,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1007,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":19771704,"Total Records Read":1468326},"Shuffle Write Metrics":{"Shuffle Bytes Written":743,"Shuffle Write Time":61959,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":205,"Index":3,"Attempt":0,"Launch Time":1646432646394,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432656491,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"24749426","Value":"539125167","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1804783","Value":"40651010","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1804783","Value":"40651010","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"310","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1804783","Value":"40651010","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1778105","Value":"39995375","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"1061","Value":"26380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"469762048","Value":"10703863808","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"8408","Value":"197990","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"13318435","Value":"268258307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"13286263","Value":"267221348","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1244990","Value":"28000764","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"9626","Value":"236000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":354,"Value":12074,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":104217999,"Value":3390000978,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":9703,"Value":239318,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":8288786209,"Value":205784043477,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":294794,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1007,"Value":21807,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":671124512,"Value":17952912512,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":24749426,"Value":539125167,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1804783,"Value":40651010,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":747,"Value":26662,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":61738,"Value":8559598,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17755499528,"JVMOffHeapMemory":141156128,"OnHeapExecutionMemory":3489660928,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3628923999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1619695,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1221,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":354,"Executor Deserialize CPU Time":104217999,"Executor Run Time":9703,"Executor CPU Time":8288786209,"Peak Execution Memory":671124512,"Result Size":8189,"JVM GC Time":1007,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":24749426,"Total Records Read":1804783},"Shuffle Write Metrics":{"Shuffle Bytes Written":747,"Shuffle Write Time":61738,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":229,"Index":24,"Attempt":0,"Launch Time":1646432646407,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432656636,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"20356289","Value":"559481456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1539649","Value":"42190659","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1539649","Value":"42190659","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"319","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1539649","Value":"42190659","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1515507","Value":"41510882","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"258","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"822","Value":"27202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"11106516992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"8718","Value":"206708","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"9818948","Value":"278077255","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"9784858","Value":"277006206","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1061026","Value":"29061790","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"9770","Value":"245770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":352,"Value":12426,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":60311293,"Value":3450312271,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":9847,"Value":249165,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":8458470517,"Value":214242513994,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":302983,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1007,"Value":22814,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":18556928160,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20356289,"Value":559481456,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1539649,"Value":42190659,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":746,"Value":27408,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":60941,"Value":8620539,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17755499528,"JVMOffHeapMemory":141156128,"OnHeapExecutionMemory":3489660928,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3628923999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1619695,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1221,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":352,"Executor Deserialize CPU Time":60311293,"Executor Run Time":9847,"Executor CPU Time":8458470517,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1007,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20356289,"Total Records Read":1539649},"Shuffle Write Metrics":{"Shuffle Bytes Written":746,"Shuffle Write Time":60941,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":225,"Index":23,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432657328,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"22682377","Value":"582163833","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1708146","Value":"43898805","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1708146","Value":"43898805","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"328","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1708146","Value":"43898805","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1671692","Value":"43182574","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"268","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"765","Value":"27967","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"11509170176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"9316","Value":"216024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"11677434","Value":"289754689","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"11650243","Value":"288656449","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1170891","Value":"30232681","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"10463","Value":"256233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":349,"Value":12775,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":72476712,"Value":3522788983,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":10542,"Value":259707,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":9214364294,"Value":223456878288,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":311172,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1007,"Value":23821,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":19160943808,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22682377,"Value":582163833,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1708146,"Value":43898805,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":739,"Value":28147,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":86903,"Value":8707442,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17755499528,"JVMOffHeapMemory":141156128,"OnHeapExecutionMemory":3489660928,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3628923999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1619695,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1221,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":349,"Executor Deserialize CPU Time":72476712,"Executor Run Time":10542,"Executor CPU Time":9214364294,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1007,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22682377,"Total Records Read":1708146},"Shuffle Write Metrics":{"Shuffle Bytes Written":739,"Shuffle Write Time":86903,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":209,"Index":4,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432657571,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"26091211","Value":"608255044","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1890520","Value":"45789325","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1890520","Value":"45789325","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"9","Value":"337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1890520","Value":"45789325","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1860972","Value":"45043546","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"10","Value":"278","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"911","Value":"28878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"469762048","Value":"11978932224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"9461","Value":"225485","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"13289107","Value":"303043796","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"13248488","Value":"301904937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1302426","Value":"31535107","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"10706","Value":"266939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":351,"Value":13126,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":71234605,"Value":3594023588,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":10782,"Value":270489,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":9403730932,"Value":232860609220,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":319361,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1007,"Value":24828,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":671124512,"Value":19832068320,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":26091211,"Value":608255044,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1890520,"Value":45789325,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":749,"Value":28896,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":60438,"Value":8767880,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17755499528,"JVMOffHeapMemory":141156128,"OnHeapExecutionMemory":3489660928,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3628923999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1619695,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1221,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":351,"Executor Deserialize CPU Time":71234605,"Executor Run Time":10782,"Executor CPU Time":9403730932,"Peak Execution Memory":671124512,"Result Size":8189,"JVM GC Time":1007,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":26091211,"Total Records Read":1890520},"Shuffle Write Metrics":{"Shuffle Bytes Written":749,"Shuffle Write Time":60438,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":232,"Index":46,"Attempt":0,"Launch Time":1646432646408,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432658350,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"13508300","Value":"621763344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1023135","Value":"46812460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1023135","Value":"46812460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"6","Value":"343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1023135","Value":"46812460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1009708","Value":"46053254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"283","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"848","Value":"29726","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"234881024","Value":"12213813248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"10121","Value":"235606","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"6850760","Value":"309894556","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"6817396","Value":"308722333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"707425","Value":"32242532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"11494","Value":"278433","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":334,"Value":13460,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":66128947,"Value":3660152535,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":11577,"Value":282066,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":10388138443,"Value":243248747663,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":327550,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":747,"Value":25575,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":436243488,"Value":20268311808,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":13508300,"Value":621763344,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1023135,"Value":46812460,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":739,"Value":29635,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":81189,"Value":8849069,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":142985088,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1612345,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":1781,"MajorGCCount":6,"MajorGCTime":649},"Task Metrics":{"Executor Deserialize Time":334,"Executor Deserialize CPU Time":66128947,"Executor Run Time":11577,"Executor CPU Time":10388138443,"Peak Execution Memory":436243488,"Result Size":8189,"JVM GC Time":747,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":13508300,"Total Records Read":1023135},"Shuffle Write Metrics":{"Shuffle Bytes Written":739,"Shuffle Write Time":81189,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":196,"Index":6,"Attempt":0,"Launch Time":1646432646391,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432658388,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"41","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"15259337","Value":"637022681","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1154264","Value":"47966724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1154264","Value":"47966724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"5","Value":"348","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1154264","Value":"47966724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1136753","Value":"47190007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"289","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"651","Value":"30377","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"12515803136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"10506","Value":"246112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7600974","Value":"317495530","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7568547","Value":"316290880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"795276","Value":"33037808","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"11532","Value":"289965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":334,"Value":13794,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":65524275,"Value":3725676810,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":11615,"Value":293681,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":10324353978,"Value":253573101641,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":335739,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":747,"Value":26322,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":20771664160,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":15259337,"Value":637022681,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1154264,"Value":47966724,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":742,"Value":30377,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":62524,"Value":8911593,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":142985088,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1612345,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":1781,"MajorGCCount":6,"MajorGCTime":649},"Task Metrics":{"Executor Deserialize Time":334,"Executor Deserialize CPU Time":65524275,"Executor Run Time":11615,"Executor CPU Time":10324353978,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":747,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":15259337,"Total Records Read":1154264},"Shuffle Write Metrics":{"Shuffle Bytes Written":742,"Shuffle Write Time":62524,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":200,"Index":7,"Attempt":0,"Launch Time":1646432646392,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432659235,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"18182549","Value":"655205230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1355043","Value":"49321767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1355043","Value":"49321767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"6","Value":"354","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1355043","Value":"49321767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1338255","Value":"48528262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"295","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"976","Value":"31353","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"12817793024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"11287","Value":"257399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"8599656","Value":"326095186","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"8567580","Value":"324858460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"937919","Value":"33975727","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"12380","Value":"302345","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":335,"Value":14129,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":112886071,"Value":3838562881,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":12463,"Value":306144,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":11291662235,"Value":264864763876,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":343928,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":747,"Value":27069,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":21275016512,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":18182549,"Value":655205230,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1355043,"Value":49321767,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":743,"Value":31120,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":63275,"Value":8974868,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":142985088,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1612345,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":1781,"MajorGCCount":6,"MajorGCTime":649},"Task Metrics":{"Executor Deserialize Time":335,"Executor Deserialize CPU Time":112886071,"Executor Run Time":12463,"Executor CPU Time":11291662235,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":747,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":18182549,"Total Records Read":1355043},"Shuffle Write Metrics":{"Shuffle Bytes Written":743,"Shuffle Write Time":63275,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":234,"Index":47,"Attempt":0,"Launch Time":1646432646409,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432659424,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"43","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"16286726","Value":"671491956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1228580","Value":"50550347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1228580","Value":"50550347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"5","Value":"359","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1228580","Value":"50550347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1215094","Value":"49743356","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"644","Value":"31997","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"13119782912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"11593","Value":"268992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"7704937","Value":"333800123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"7674636","Value":"332533096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"851410","Value":"34827137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"12568","Value":"314913","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":329,"Value":14458,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":79771012,"Value":3918333893,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":12658,"Value":318802,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":11530809206,"Value":276395573082,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":352117,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":747,"Value":27816,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":21778368864,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16286726,"Value":671491956,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1228580,"Value":50550347,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":741,"Value":31861,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":71940,"Value":9046808,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":142985088,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1612345,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":1781,"MajorGCCount":6,"MajorGCTime":649},"Task Metrics":{"Executor Deserialize Time":329,"Executor Deserialize CPU Time":79771012,"Executor Run Time":12658,"Executor CPU Time":11530809206,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":747,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16286726,"Total Records Read":1228580},"Shuffle Write Metrics":{"Shuffle Bytes Written":741,"Shuffle Write Time":71940,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":220,"Index":27,"Attempt":0,"Launch Time":1646432646403,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432659672,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"17698934","Value":"689190890","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1333919","Value":"51884266","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1333919","Value":"51884266","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"6","Value":"365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1333919","Value":"51884266","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1319111","Value":"51062467","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"306","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"690","Value":"32687","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"301989888","Value":"13421772800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"11794","Value":"280786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"8355380","Value":"342155503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"8324284","Value":"340857380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"924194","Value":"35751331","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"12817","Value":"327730","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":336,"Value":14794,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":80574981,"Value":3998908874,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":12899,"Value":331701,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":11748055908,"Value":288143628990,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":360306,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":747,"Value":28563,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":503352352,"Value":22281721216,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17698934,"Value":689190890,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1333919,"Value":51884266,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":742,"Value":32603,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":55180,"Value":9101988,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":142985088,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1612345,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":1781,"MajorGCCount":6,"MajorGCTime":649},"Task Metrics":{"Executor Deserialize Time":336,"Executor Deserialize CPU Time":80574981,"Executor Run Time":12899,"Executor CPU Time":11748055908,"Peak Execution Memory":503352352,"Result Size":8189,"JVM GC Time":747,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17698934,"Total Records Read":1333919},"Shuffle Write Metrics":{"Shuffle Bytes Written":742,"Shuffle Write Time":55180,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":236,"Index":48,"Attempt":0,"Launch Time":1646432646410,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432659981,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"19877448","Value":"709068338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1480083","Value":"53364349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1480083","Value":"53364349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"5","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1480083","Value":"53364349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1464397","Value":"52526864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"821","Value":"33508","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"13824425984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"11978","Value":"292764","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"10574238","Value":"352729741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"10546700","Value":"351404080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1024881","Value":"36776212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"13125","Value":"340855","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":333,"Value":15127,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":81336572,"Value":4080245446,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":13212,"Value":344913,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":12038237923,"Value":300181866913,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":368495,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":747,"Value":29310,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":22885736864,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":19877448,"Value":709068338,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1480083,"Value":53364349,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":744,"Value":33347,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":58312,"Value":9160300,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":142985088,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1612345,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":1781,"MajorGCCount":6,"MajorGCTime":649},"Task Metrics":{"Executor Deserialize Time":333,"Executor Deserialize CPU Time":81336572,"Executor Run Time":13212,"Executor CPU Time":12038237923,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":747,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":19877448,"Total Records Read":1480083},"Shuffle Write Metrics":{"Shuffle Bytes Written":744,"Shuffle Write Time":58312,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":238,"Index":49,"Attempt":0,"Launch Time":1646432646411,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432660603,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"46","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"20138112","Value":"729206450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1498633","Value":"54862982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1498633","Value":"54862982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"6","Value":"376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1498633","Value":"54862982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1483487","Value":"54010351","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"318","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"1446","Value":"34954","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"14227079168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"12163","Value":"304927","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"9849078","Value":"362578819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"9812593","Value":"361216673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1038288","Value":"37814500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"13747","Value":"354602","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":333,"Value":15460,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":70053224,"Value":4150298670,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":13834,"Value":358747,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":12253469922,"Value":312435336835,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":376684,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1102,"Value":30412,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":23489752512,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20138112,"Value":729206450,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1498633,"Value":54862982,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":743,"Value":34090,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":67950,"Value":9228250,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":142985088,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1612345,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":1781,"MajorGCCount":6,"MajorGCTime":649},"Task Metrics":{"Executor Deserialize Time":333,"Executor Deserialize CPU Time":70053224,"Executor Run Time":13834,"Executor CPU Time":12253469922,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1102,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20138112,"Total Records Read":1498633},"Shuffle Write Metrics":{"Shuffle Bytes Written":743,"Shuffle Write Time":67950,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":228,"Index":29,"Attempt":0,"Launch Time":1646432646407,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432660700,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"47","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"20388677","Value":"749595127","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1540279","Value":"56403261","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1540279","Value":"56403261","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"6","Value":"382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1540279","Value":"56403261","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1524417","Value":"55534768","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"323","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"840","Value":"35794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"14629732352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"12278","Value":"317205","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"10008625","Value":"372587444","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"9971320","Value":"371187993","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1067095","Value":"38881595","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"13845","Value":"368447","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":334,"Value":15794,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":68148977,"Value":4218447647,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":13928,"Value":372675,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":12390530626,"Value":324825867461,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":384873,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1102,"Value":31514,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":24093768160,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20388677,"Value":749595127,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1540279,"Value":56403261,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":746,"Value":34836,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":57483,"Value":9285733,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":142985088,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1612345,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":83,"MinorGCTime":1781,"MajorGCCount":6,"MajorGCTime":649},"Task Metrics":{"Executor Deserialize Time":334,"Executor Deserialize CPU Time":68148977,"Executor Run Time":13928,"Executor CPU Time":12390530626,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1102,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20388677,"Total Records Read":1540279},"Shuffle Write Metrics":{"Shuffle Bytes Written":746,"Shuffle Write Time":57483,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":224,"Index":28,"Attempt":0,"Launch Time":1646432646405,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432661539,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"22891115","Value":"772486242","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1719119","Value":"58122380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1719119","Value":"58122380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"5","Value":"387","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1719119","Value":"58122380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1696792","Value":"57231560","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"329","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"956","Value":"36750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"15032385536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"12967","Value":"330172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"12201936","Value":"384789380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"12171849","Value":"383359842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1188115","Value":"40069710","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"14684","Value":"383131","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":330,"Value":16124,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":80782664,"Value":4299230311,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":14772,"Value":387447,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":13222344262,"Value":338048211723,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":393062,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1102,"Value":32616,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":24697783808,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22891115,"Value":772486242,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1719119,"Value":58122380,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":745,"Value":35581,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":56992,"Value":9342725,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":143211224,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":87,"MinorGCTime":2215,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":330,"Executor Deserialize CPU Time":80782664,"Executor Run Time":14772,"Executor CPU Time":13222344262,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1102,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22891115,"Total Records Read":1719119},"Shuffle Write Metrics":{"Shuffle Bytes Written":745,"Shuffle Write Time":56992,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":204,"Index":8,"Attempt":0,"Launch Time":1646432646394,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432661641,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"49","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"23437384","Value":"795923626","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1737471","Value":"59859851","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1737471","Value":"59859851","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"5","Value":"392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1737471","Value":"59859851","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1708343","Value":"58939903","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"5","Value":"334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"1337","Value":"38087","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"15435038720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"13464","Value":"343636","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"12578897","Value":"397368277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"12549967","Value":"395909809","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1195580","Value":"41265290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"14786","Value":"397917","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":333,"Value":16457,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":70797625,"Value":4370027936,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":14869,"Value":402316,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":13343032914,"Value":351391244637,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":401251,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1102,"Value":33718,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":25301799456,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":23437384,"Value":795923626,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1737471,"Value":59859851,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":749,"Value":36330,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":53856,"Value":9396581,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":143211224,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":87,"MinorGCTime":2215,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":333,"Executor Deserialize CPU Time":70797625,"Executor Run Time":14869,"Executor CPU Time":13343032914,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1102,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":23437384,"Total Records Read":1737471},"Shuffle Write Metrics":{"Shuffle Bytes Written":749,"Shuffle Write Time":53856,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":208,"Index":9,"Attempt":0,"Launch Time":1646432646395,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432661761,"Failed":false,"Killed":false,"Accumulables":[{"ID":1692,"Name":"local blocks read","Update":"1","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Update":"22864810","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Update":"1697113","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Update":"1697113","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Update":"5","Value":"397","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Update":"1697113","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Update":"1676385","Value":"60616288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Update":"6","Value":"340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Update":"1343","Value":"39430","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Update":"402653184","Value":"15837691904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Update":"13610","Value":"357246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Update":"11418581","Value":"408786858","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Update":"11380207","Value":"407290016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Update":"1172062","Value":"42437352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Update":"14906","Value":"412823","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Update":335,"Value":16792,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Update":91362167,"Value":4461390103,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Update":14989,"Value":417305,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Update":13501962895,"Value":364893207532,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Update":8189,"Value":409440,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Update":1102,"Value":34820,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Update":604015648,"Value":25905815104,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22864810,"Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1697113,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":750,"Value":37080,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Update":55283,"Value":9451864,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11475146248,"JVMOffHeapMemory":143211224,"OnHeapExecutionMemory":1811939328,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":139263071,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1951202399,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":87,"MinorGCTime":2215,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":335,"Executor Deserialize CPU Time":91362167,"Executor Run Time":14989,"Executor CPU Time":13501962895,"Peak Execution Memory":604015648,"Result Size":8189,"JVM GC Time":1102,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22864810,"Total Records Read":1697113},"Shuffle Write Metrics":{"Shuffle Bytes Written":750,"Shuffle Write Time":55283,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":60,"Stage Attempt ID":0,"Stage Name":"treeAggregate at WeightedLeastSquares.scala:107","Number of Tasks":50,"RDD Info":[{"RDD ID":139,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[138],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":136,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"215\",\"name\":\"mapPartitions\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[135],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":138,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[137],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":134,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6438\",\"name\":\"DeserializeToObject\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[133],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":133,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6437\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[132],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":137,"Name":"training instances","Scope":"{\"id\":\"216\",\"name\":\"map\"}","Callsite":"map at Predictor.scala:81","Parent IDs":[136],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":135,"Name":"SQLExecutionRDD","Callsite":"rdd at Predictor.scala:81","Parent IDs":[134],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":132,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_6411\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at Predictor.scala:81","Parent IDs":[108],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[59],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.ml.optim.WeightedLeastSquares.fit(WeightedLeastSquares.scala:107)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:453)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)","Submission Time":1646432646375,"Completion Time":1646432661762,"Accumulables":[{"ID":1692,"Name":"local blocks read","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1695,"Name":"local bytes read","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1696,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1697,"Name":"records read","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2039,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2040,"Name":"time to read broadcast","Value":"397","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2041,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2042,"Name":"time to read broadcast","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2043,"Name":"number of output rows","Value":"60616288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2044,"Name":"time to read broadcast","Value":"340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2045,"Name":"sort time","Value":"39430","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2046,"Name":"peak memory","Value":"15837691904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2047,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2048,"Name":"cumulative time","Value":"357246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2049,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2050,"Name":"num prefix comparisons","Value":"408786858","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2051,"Name":"num record comparisons","Value":"407290016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2052,"Name":"number of output rows","Value":"42437352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2053,"Name":"duration","Value":"412823","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2055,"Name":"internal.metrics.executorDeserializeTime","Value":16792,"Internal":true,"Count Failed Values":true},{"ID":2056,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4461390103,"Internal":true,"Count Failed Values":true},{"ID":2057,"Name":"internal.metrics.executorRunTime","Value":417305,"Internal":true,"Count Failed Values":true},{"ID":2058,"Name":"internal.metrics.executorCpuTime","Value":364893207532,"Internal":true,"Count Failed Values":true},{"ID":2059,"Name":"internal.metrics.resultSize","Value":409440,"Internal":true,"Count Failed Values":true},{"ID":2060,"Name":"internal.metrics.jvmGCTime","Value":34820,"Internal":true,"Count Failed Values":true},{"ID":2062,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.peakExecutionMemory","Value":25905815104,"Internal":true,"Count Failed Values":true},{"ID":2068,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2069,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":50,"Internal":true,"Count Failed Values":true},{"ID":2070,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.shuffle.read.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2075,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":37080,"Internal":true,"Count Failed Values":true},{"ID":2076,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":50,"Internal":true,"Count Failed Values":true},{"ID":2077,"Name":"internal.metrics.shuffle.write.writeTime","Value":9451864,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":61,"Stage Attempt ID":0,"Stage Name":"treeAggregate at WeightedLeastSquares.scala:107","Number of Tasks":6,"RDD Info":[{"RDD ID":141,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[140],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":140,"Name":"ShuffledRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[139],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[60],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.ml.optim.WeightedLeastSquares.fit(WeightedLeastSquares.scala:107)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:453)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)","Submission Time":1646432661763,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":61,"Stage Attempt ID":0,"Task Info":{"Task ID":240,"Index":0,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":61,"Stage Attempt ID":0,"Task Info":{"Task ID":241,"Index":1,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":61,"Stage Attempt ID":0,"Task Info":{"Task ID":242,"Index":2,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":61,"Stage Attempt ID":0,"Task Info":{"Task ID":243,"Index":3,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":61,"Stage Attempt ID":0,"Task Info":{"Task ID":244,"Index":4,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":61,"Stage Attempt ID":0,"Task Info":{"Task ID":245,"Index":5,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":61,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":243,"Index":3,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432661822,"Failed":false,"Killed":false,"Accumulables":[{"ID":2085,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2086,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8650382,"Value":8650382,"Internal":true,"Count Failed Values":true},{"ID":2087,"Name":"internal.metrics.executorRunTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":2088,"Name":"internal.metrics.executorCpuTime","Update":13623739,"Value":13623739,"Internal":true,"Count Failed Values":true},{"ID":2089,"Name":"internal.metrics.resultSize","Update":8655,"Value":8655,"Internal":true,"Count Failed Values":true},{"ID":2092,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2093,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2094,"Name":"internal.metrics.peakExecutionMemory","Update":2112,"Value":2112,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":5193,"Value":5193,"Internal":true,"Count Failed Values":true},{"ID":2101,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":744,"Value":744,"Internal":true,"Count Failed Values":true},{"ID":2103,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2104,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":8,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":8650382,"Executor Run Time":16,"Executor CPU Time":13623739,"Peak Execution Memory":2112,"Result Size":8655,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":5193,"Remote Bytes Read To Disk":0,"Local Bytes Read":744,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":61,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":240,"Index":0,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432661824,"Failed":false,"Killed":false,"Accumulables":[{"ID":2085,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2086,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10014196,"Value":18664578,"Internal":true,"Count Failed Values":true},{"ID":2087,"Name":"internal.metrics.executorRunTime","Update":17,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":2088,"Name":"internal.metrics.executorCpuTime","Update":9753140,"Value":23376879,"Internal":true,"Count Failed Values":true},{"ID":2089,"Name":"internal.metrics.resultSize","Update":8655,"Value":17310,"Internal":true,"Count Failed Values":true},{"ID":2092,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2093,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2094,"Name":"internal.metrics.peakExecutionMemory","Update":2112,"Value":4224,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":6,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":3,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":4440,"Value":9633,"Internal":true,"Count Failed Values":true},{"ID":2101,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2226,"Value":2970,"Internal":true,"Count Failed Values":true},{"ID":2103,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2104,"Name":"internal.metrics.shuffle.read.recordsRead","Update":9,"Value":17,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":10014196,"Executor Run Time":17,"Executor CPU Time":9753140,"Peak Execution Memory":2112,"Result Size":8655,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":6,"Local Blocks Fetched":3,"Fetch Wait Time":0,"Remote Bytes Read":4440,"Remote Bytes Read To Disk":0,"Local Bytes Read":2226,"Total Records Read":9},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":61,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":244,"Index":4,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432661824,"Failed":false,"Killed":false,"Accumulables":[{"ID":2085,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":2086,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10050374,"Value":28714952,"Internal":true,"Count Failed Values":true},{"ID":2087,"Name":"internal.metrics.executorRunTime","Update":17,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":2088,"Name":"internal.metrics.executorCpuTime","Update":12158618,"Value":35535497,"Internal":true,"Count Failed Values":true},{"ID":2089,"Name":"internal.metrics.resultSize","Update":8655,"Value":25965,"Internal":true,"Count Failed Values":true},{"ID":2092,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2093,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2094,"Name":"internal.metrics.peakExecutionMemory","Update":2112,"Value":6336,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":5,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":3,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":3706,"Value":13339,"Internal":true,"Count Failed Values":true},{"ID":2101,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2226,"Value":5196,"Internal":true,"Count Failed Values":true},{"ID":2103,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2104,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":25,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":10050374,"Executor Run Time":17,"Executor CPU Time":12158618,"Peak Execution Memory":2112,"Result Size":8655,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":5,"Local Blocks Fetched":3,"Fetch Wait Time":0,"Remote Bytes Read":3706,"Remote Bytes Read To Disk":0,"Local Bytes Read":2226,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":61,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":245,"Index":5,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432661825,"Failed":false,"Killed":false,"Accumulables":[{"ID":2085,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2086,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10922799,"Value":39637751,"Internal":true,"Count Failed Values":true},{"ID":2087,"Name":"internal.metrics.executorRunTime","Update":17,"Value":67,"Internal":true,"Count Failed Values":true},{"ID":2088,"Name":"internal.metrics.executorCpuTime","Update":11050549,"Value":46586046,"Internal":true,"Count Failed Values":true},{"ID":2089,"Name":"internal.metrics.resultSize","Update":8655,"Value":34620,"Internal":true,"Count Failed Values":true},{"ID":2092,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2093,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2094,"Name":"internal.metrics.peakExecutionMemory","Update":2112,"Value":8448,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":5,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":3,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":3698,"Value":17037,"Internal":true,"Count Failed Values":true},{"ID":2101,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2223,"Value":7419,"Internal":true,"Count Failed Values":true},{"ID":2103,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2104,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":33,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":10922799,"Executor Run Time":17,"Executor CPU Time":11050549,"Peak Execution Memory":2112,"Result Size":8655,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":5,"Local Blocks Fetched":3,"Fetch Wait Time":0,"Remote Bytes Read":3698,"Remote Bytes Read To Disk":0,"Local Bytes Read":2223,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":61,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":241,"Index":1,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432661826,"Failed":false,"Killed":false,"Accumulables":[{"ID":2085,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":2086,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8408346,"Value":48046097,"Internal":true,"Count Failed Values":true},{"ID":2087,"Name":"internal.metrics.executorRunTime","Update":20,"Value":87,"Internal":true,"Count Failed Values":true},{"ID":2088,"Name":"internal.metrics.executorCpuTime","Update":13066840,"Value":59652886,"Internal":true,"Count Failed Values":true},{"ID":2089,"Name":"internal.metrics.resultSize","Update":8655,"Value":43275,"Internal":true,"Count Failed Values":true},{"ID":2092,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2093,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2094,"Name":"internal.metrics.peakExecutionMemory","Update":2112,"Value":10560,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":6,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":3,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":4458,"Value":21495,"Internal":true,"Count Failed Values":true},{"ID":2101,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2216,"Value":9635,"Internal":true,"Count Failed Values":true},{"ID":2103,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2104,"Name":"internal.metrics.shuffle.read.recordsRead","Update":9,"Value":42,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":8408346,"Executor Run Time":20,"Executor CPU Time":13066840,"Peak Execution Memory":2112,"Result Size":8655,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":6,"Local Blocks Fetched":3,"Fetch Wait Time":0,"Remote Bytes Read":4458,"Remote Bytes Read To Disk":0,"Local Bytes Read":2216,"Total Records Read":9},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":61,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":242,"Index":2,"Attempt":0,"Launch Time":1646432661779,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432661828,"Failed":false,"Killed":false,"Accumulables":[{"ID":2085,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":2086,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10626259,"Value":58672356,"Internal":true,"Count Failed Values":true},{"ID":2087,"Name":"internal.metrics.executorRunTime","Update":21,"Value":108,"Internal":true,"Count Failed Values":true},{"ID":2088,"Name":"internal.metrics.executorCpuTime","Update":17333781,"Value":76986667,"Internal":true,"Count Failed Values":true},{"ID":2089,"Name":"internal.metrics.resultSize","Update":8655,"Value":51930,"Internal":true,"Count Failed Values":true},{"ID":2092,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2093,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2094,"Name":"internal.metrics.peakExecutionMemory","Update":2112,"Value":12672,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":5206,"Value":26701,"Internal":true,"Count Failed Values":true},{"ID":2101,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":744,"Value":10379,"Internal":true,"Count Failed Values":true},{"ID":2103,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2104,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":50,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":10626259,"Executor Run Time":21,"Executor CPU Time":17333781,"Peak Execution Memory":2112,"Result Size":8655,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":5206,"Remote Bytes Read To Disk":0,"Local Bytes Read":744,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":61,"Stage Attempt ID":0,"Stage Name":"treeAggregate at WeightedLeastSquares.scala:107","Number of Tasks":6,"RDD Info":[{"RDD ID":141,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[140],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":140,"Name":"ShuffledRDD","Scope":"{\"id\":\"217\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at WeightedLeastSquares.scala:107","Parent IDs":[139],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[60],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.ml.optim.WeightedLeastSquares.fit(WeightedLeastSquares.scala:107)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:453)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)","Submission Time":1646432661763,"Completion Time":1646432661896,"Accumulables":[{"ID":2085,"Name":"internal.metrics.executorDeserializeTime","Value":59,"Internal":true,"Count Failed Values":true},{"ID":2086,"Name":"internal.metrics.executorDeserializeCpuTime","Value":58672356,"Internal":true,"Count Failed Values":true},{"ID":2087,"Name":"internal.metrics.executorRunTime","Value":108,"Internal":true,"Count Failed Values":true},{"ID":2088,"Name":"internal.metrics.executorCpuTime","Value":76986667,"Internal":true,"Count Failed Values":true},{"ID":2089,"Name":"internal.metrics.resultSize","Value":51930,"Internal":true,"Count Failed Values":true},{"ID":2092,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2093,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2094,"Name":"internal.metrics.peakExecutionMemory","Value":12672,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":36,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":14,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":26701,"Internal":true,"Count Failed Values":true},{"ID":2101,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":10379,"Internal":true,"Count Failed Values":true},{"ID":2103,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2104,"Name":"internal.metrics.shuffle.read.recordsRead","Value":50,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":33,"Completion Time":1646432661897,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":34,"Submission Time":1646432662352,"Stage Infos":[{"Stage ID":62,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":145,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6996\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[144],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":143,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6534\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[142],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":144,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6995\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[143],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":142,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6534\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[62],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_6996\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":62,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":145,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6996\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[144],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":143,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6534\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[142],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":144,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6995\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[143],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":142,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6534\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432662353,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_6996\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":246,"Index":0,"Attempt":0,"Launch Time":1646432662357,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":247,"Index":1,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":248,"Index":2,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":249,"Index":3,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":250,"Index":4,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":251,"Index":5,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":252,"Index":6,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":253,"Index":7,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":254,"Index":8,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":255,"Index":9,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerJobStart","Job ID":35,"Submission Time":1646432662486,"Stage Infos":[{"Stage ID":63,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":154,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7016\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[153],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":151,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7011\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[150],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":152,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_7012\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[148,151],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":146,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":149,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":148,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7008\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[147],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":150,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[149],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":147,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[146],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":153,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7015\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[152],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[63],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_7016\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":63,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":154,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7016\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[153],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":151,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7011\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[150],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":152,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_7012\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[148,151],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":146,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":149,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":148,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7008\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[147],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":150,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[149],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":147,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[146],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":153,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7015\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[152],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432662488,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_7016\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerJobStart","Job ID":36,"Submission Time":1646432662582,"Stage Infos":[{"Stage ID":64,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":158,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7092\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[157],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":157,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7091\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[156],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":156,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[155],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":155,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[64],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_7092\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":64,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":158,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7092\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[157],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":157,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7091\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[156],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":156,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[155],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":155,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432662584,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_7092\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":256,"Index":0,"Attempt":0,"Launch Time":1646432662581,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":257,"Index":1,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":258,"Index":2,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":259,"Index":3,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":260,"Index":4,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":261,"Index":5,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":262,"Index":6,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":263,"Index":7,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":264,"Index":8,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":265,"Index":9,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":266,"Index":10,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":267,"Index":11,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":268,"Index":12,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":269,"Index":13,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":270,"Index":14,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":271,"Index":15,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":272,"Index":16,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":273,"Index":17,"Attempt":0,"Launch Time":1646432662586,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":274,"Index":18,"Attempt":0,"Launch Time":1646432662586,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":275,"Index":19,"Attempt":0,"Launch Time":1646432662586,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":276,"Index":0,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":277,"Index":1,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":278,"Index":2,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":279,"Index":3,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":280,"Index":4,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":281,"Index":5,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":282,"Index":6,"Attempt":0,"Launch Time":1646432662615,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":283,"Index":7,"Attempt":0,"Launch Time":1646432662615,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":284,"Index":8,"Attempt":0,"Launch Time":1646432662615,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":285,"Index":9,"Attempt":0,"Launch Time":1646432662615,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":272,"Index":16,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432674939,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"322","Value":"322","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"12295","Value":"12295","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"12306","Value":"12306","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"12298","Value":"12298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1286","Value":"1286","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7728","Value":"7728","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"12099","Value":"12099","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"322","Value":"322","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"14288260","Value":"14288260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4420496,"Value":4420496,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":12322,"Value":12322,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":12013550398,"Value":12013550398,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":2563,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":221,"Value":221,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12099,"Value":12099,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":322,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":14288260,"Value":14288260,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":5819079,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1286811837,"Value":1286811837,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16372518112,"JVMOffHeapMemory":141836200,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1480676,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1094,"MajorGCCount":7,"MajorGCTime":624},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4420496,"Executor Run Time":12322,"Executor CPU Time":12013550398,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":221,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12099,"Shuffle Write Time":14288260,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":275,"Index":19,"Attempt":0,"Launch Time":1646432662586,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432675148,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"310","Value":"632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"12494","Value":"24789","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"2007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"12505","Value":"24811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"12497","Value":"24795","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"5617658","Value":"11436737","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1277","Value":"2563","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"358169690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"175932825","Value":"358169690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7440","Value":"15168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11887","Value":"23986","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"310","Value":"632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"16056189","Value":"30344449","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4983372,"Value":9403868,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":12526,"Value":24848,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":12348889412,"Value":24362439810,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":5126,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":50,"Value":271,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11887,"Value":23986,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":310,"Value":632,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":16056189,"Value":30344449,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":358169690,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":11436737,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1277478353,"Value":2564290190,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":358169690,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4983372,"Executor Run Time":12526,"Executor CPU Time":12348889412,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":50,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11887,"Shuffle Write Time":16056189,"Shuffle Records Written":310},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":263,"Index":7,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432675331,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"325","Value":"957","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"12685","Value":"37474","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"3014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"12696","Value":"37507","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"12690","Value":"12690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"5819811","Value":"5819811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1301","Value":"1301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"180693843","Value":"180693843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7800","Value":"22968","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"12061","Value":"36047","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"325","Value":"957","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"12923625","Value":"43268074","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4905891,"Value":14309759,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":12710,"Value":37558,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":12507022574,"Value":36869462384,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":7689,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":50,"Value":321,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12061,"Value":36047,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":325,"Value":957,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":12923625,"Value":43268074,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":538863533,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":17256548,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1301943253,"Value":3866233443,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":538863533,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4905891,"Executor Run Time":12710,"Executor CPU Time":12507022574,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":50,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12061,"Shuffle Write Time":12923625,"Shuffle Records Written":325},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":260,"Index":4,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432675618,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"299","Value":"1256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"12976","Value":"50450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"4014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"12987","Value":"50494","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"12979","Value":"25669","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"6066650","Value":"11886461","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1518","Value":"2819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"365729723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"185035880","Value":"365729723","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7176","Value":"30144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11443","Value":"47490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"299","Value":"1256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"13279381","Value":"56547455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4478068,"Value":18787827,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":13001,"Value":50559,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":12691474347,"Value":49560936731,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":10252,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":221,"Value":542,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11443,"Value":47490,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":299,"Value":1256,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":13279381,"Value":56547455,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":723899413,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":23323198,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1518946923,"Value":5385180366,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":723899413,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16372518112,"JVMOffHeapMemory":141836200,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1480676,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1094,"MajorGCCount":7,"MajorGCTime":624},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4478068,"Executor Run Time":13001,"Executor CPU Time":12691474347,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":221,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11443,"Shuffle Write Time":13279381,"Shuffle Records Written":299},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":261,"Index":5,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432675717,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"312","Value":"1568","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"13066","Value":"63516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"13079","Value":"63573","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"13071","Value":"38740","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"6096762","Value":"17983223","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1282","Value":"4101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"549669231","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"183939508","Value":"549669231","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7488","Value":"37632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11723","Value":"59213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"312","Value":"1568","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"15472514","Value":"72019969","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5159837,"Value":23947664,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":13095,"Value":63654,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":12998620676,"Value":62559557407,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":12815,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":578,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11723,"Value":59213,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":312,"Value":1568,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":15472514,"Value":72019969,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":907838921,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":29419960,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1282936834,"Value":6668117200,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":907838921,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10415392160,"JVMOffHeapMemory":144626864,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":90,"MinorGCTime":2239,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5159837,"Executor Run Time":13095,"Executor CPU Time":12998620676,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11723,"Shuffle Write Time":15472514,"Shuffle Records Written":312},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":279,"Index":3,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432675983,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"13340","Value":"13340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"13347","Value":"13347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"6429338","Value":"6429338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1369","Value":"1369","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"195074315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"195074315","Value":"195074315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"456","Value":"456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"1232","Value":"1232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"3716687","Value":"3716687","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2672218,"Value":2672218,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":13351,"Value":13351,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":13009380378,"Value":13009380378,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2133,"Value":2133,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":226,"Value":226,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1232,"Value":1232,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":3716687,"Value":3716687,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":195074315,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":6429338,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1369097494,"Value":1369097494,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":195074315,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16372518112,"JVMOffHeapMemory":141836200,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1480676,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":82,"MinorGCTime":1094,"MajorGCCount":7,"MajorGCTime":624},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2672218,"Executor Run Time":13351,"Executor CPU Time":13009380378,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":226,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1232,"Shuffle Write Time":3716687,"Shuffle Records Written":19},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":259,"Index":3,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432676380,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"296","Value":"1864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"13734","Value":"77250","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"6021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"13745","Value":"77318","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"13738","Value":"52478","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"6429338","Value":"24412561","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1331","Value":"5432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"744743546","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"195074315","Value":"744743546","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7104","Value":"44736","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11482","Value":"70695","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"296","Value":"1864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"12177659","Value":"84197628","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4999134,"Value":28946798,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":13759,"Value":77413,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":13580741167,"Value":76140298574,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":15378,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":50,"Value":628,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11482,"Value":70695,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":1864,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":12177659,"Value":84197628,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1102913236,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":35849298,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1331395632,"Value":7999512832,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1102913236,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4999134,"Executor Run Time":13759,"Executor CPU Time":13580741167,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":50,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11482,"Shuffle Write Time":12177659,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":262,"Index":6,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432676542,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"322","Value":"2186","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"13884","Value":"91134","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"13897","Value":"91215","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"13888","Value":"66366","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"5819079","Value":"30231640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1416","Value":"6848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"926980411","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"182236865","Value":"926980411","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7728","Value":"52464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"12103","Value":"82798","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"322","Value":"2186","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"17041087","Value":"101238715","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5426078,"Value":34372876,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":13915,"Value":91328,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":13648043686,"Value":89788342260,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2562,"Value":17940,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":148,"Value":776,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12103,"Value":82798,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":2186,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":17041087,"Value":101238715,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":1285150101,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":41668377,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1416737659,"Value":9416250491,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":1285150101,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141420784,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":81,"MinorGCTime":1230,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":5426078,"Executor Run Time":13915,"Executor CPU Time":13648043686,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":148,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12103,"Shuffle Write Time":17041087,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":280,"Index":4,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432676583,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"16","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"13936","Value":"27276","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"13941","Value":"27288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"6066650","Value":"12495988","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1347","Value":"2716","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"380110195","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"185035880","Value":"380110195","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"384","Value":"840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"1028","Value":"2260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"16","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"4100835","Value":"7817522","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2500834,"Value":5173052,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":13946,"Value":27297,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":13854896032,"Value":26864276410,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2133,"Value":4266,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":262,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1028,"Value":2260,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":4100835,"Value":7817522,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":380110195,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":12495988,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1347973086,"Value":2717070580,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":380110195,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10415392160,"JVMOffHeapMemory":144626864,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":90,"MinorGCTime":2239,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":2500834,"Executor Run Time":13946,"Executor CPU Time":13854896032,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1028,"Shuffle Write Time":4100835,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":268,"Index":12,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432676678,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"305","Value":"2491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"14037","Value":"105171","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"14047","Value":"105262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"14040","Value":"38835","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"6450117","Value":"17886854","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1392","Value":"3955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"554983990","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"196814300","Value":"554983990","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7320","Value":"59784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11668","Value":"94466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"305","Value":"2491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"12642127","Value":"113880842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4889986,"Value":39262862,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":14060,"Value":105388,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":13740627146,"Value":103528969406,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":20503,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":226,"Value":1002,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":94466,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":2491,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":12642127,"Value":113880842,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1481964401,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":48118494,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1392568034,"Value":10808818525,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1481964401,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16372518112,"JVMOffHeapMemory":141975032,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1480676,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":86,"MinorGCTime":1111,"MajorGCCount":7,"MajorGCTime":624},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4889986,"Executor Run Time":14060,"Executor CPU Time":13740627146,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":226,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":12642127,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":267,"Index":11,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432677018,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"318","Value":"2809","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"14372","Value":"119543","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"14384","Value":"119646","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"14376","Value":"53211","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"6369482","Value":"24256336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1412","Value":"5367","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"752622524","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"197638534","Value":"752622524","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7632","Value":"67416","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11947","Value":"106413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"318","Value":"2809","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"12226355","Value":"126107197","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4976605,"Value":44239467,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":14397,"Value":119785,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":14224755877,"Value":117753725283,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":23066,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":50,"Value":1052,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11947,"Value":106413,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":318,"Value":2809,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":12226355,"Value":126107197,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1679602935,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":54487976,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1412989186,"Value":12221807711,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1679602935,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4976605,"Executor Run Time":14397,"Executor CPU Time":14224755877,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":50,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11947,"Shuffle Write Time":12226355,"Shuffle Records Written":318},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":284,"Index":8,"Attempt":0,"Launch Time":1646432662615,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432677190,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"12","Value":"47","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"14542","Value":"41818","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"14549","Value":"41837","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"5674621","Value":"18170609","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1402","Value":"4118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"558327839","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"178217644","Value":"558327839","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"288","Value":"1128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"816","Value":"3076","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"12","Value":"47","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"3727437","Value":"11544959","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3135393,"Value":8308445,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":14553,"Value":41850,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":14404153451,"Value":41268429861,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2133,"Value":6399,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":298,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":3076,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":3727437,"Value":11544959,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":558327839,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":18170609,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1402181147,"Value":4119251727,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":558327839,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10415392160,"JVMOffHeapMemory":144626864,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":90,"MinorGCTime":2239,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":3135393,"Executor Run Time":14553,"Executor CPU Time":14404153451,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":3727437,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":265,"Index":9,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432677423,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"313","Value":"3122","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"14772","Value":"134315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"14787","Value":"134433","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"14780","Value":"81146","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"5617658","Value":"35849298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1436","Value":"8284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"1102913236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"175932825","Value":"1102913236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7512","Value":"74928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11907","Value":"118320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"313","Value":"3122","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"12930336","Value":"139037533","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5054355,"Value":49293822,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":14801,"Value":134586,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":14635133317,"Value":132388858600,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":25629,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":1088,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11907,"Value":118320,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":3122,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":12930336,"Value":139037533,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":1855535760,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":60105634,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1436877220,"Value":13658684931,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":1855535760,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10415392160,"JVMOffHeapMemory":144626864,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":90,"MinorGCTime":2239,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5054355,"Executor Run Time":14801,"Executor CPU Time":14635133317,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11907,"Shuffle Write Time":12930336,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":285,"Index":9,"Attempt":0,"Launch Time":1646432662615,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432677515,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"12","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"14862","Value":"56680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"4000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"14870","Value":"56707","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"5617658","Value":"23788267","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1496","Value":"5614","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"734260664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"175932825","Value":"734260664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"288","Value":"1416","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"816","Value":"3892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"12","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"3752990","Value":"15297949","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3137564,"Value":11446009,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":14874,"Value":56724,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":14670866483,"Value":55939296344,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2132,"Value":8531,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":148,"Value":446,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":3892,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":3752990,"Value":15297949,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":734260664,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":23788267,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1496626114,"Value":5615877841,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":734260664,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141420784,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":81,"MinorGCTime":1230,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3137564,"Executor Run Time":14874,"Executor CPU Time":14670866483,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":148,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":3752990,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":257,"Index":1,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432678492,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"320","Value":"3442","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"741081088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"15842","Value":"150157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"11021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"15857","Value":"150290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"15849","Value":"96995","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"6369482","Value":"42218780","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1537","Value":"9821","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1300551770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"197638534","Value":"1300551770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7680","Value":"82608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11962","Value":"130282","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"320","Value":"3442","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"13051542","Value":"152089075","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4906691,"Value":54200513,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":15871,"Value":150457,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":15697483158,"Value":148086341758,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":28192,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":42,"Value":1130,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":741081088,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11962,"Value":130282,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":3442,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":13051542,"Value":152089075,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":2053174294,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":66475116,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1537819861,"Value":15196504792,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":2053174294,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10415392160,"JVMOffHeapMemory":144626864,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":90,"MinorGCTime":2239,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4906691,"Executor Run Time":15871,"Executor CPU Time":15697483158,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":42,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11962,"Shuffle Write Time":13051542,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":269,"Index":13,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432678521,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"296","Value":"3738","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"808452096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"15871","Value":"166028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"12028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"15887","Value":"166177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"15880","Value":"69091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"6429338","Value":"30685674","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1530","Value":"6897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"947696839","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"195074315","Value":"947696839","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7104","Value":"89712","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11481","Value":"141763","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"296","Value":"3738","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"11954510","Value":"164043585","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":58,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4815839,"Value":59016352,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":15901,"Value":166358,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":15727411065,"Value":163813752823,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":30755,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":42,"Value":1172,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":808452096,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11481,"Value":141763,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":3738,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":11954510,"Value":164043585,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":2248248609,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":72904454,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1530239293,"Value":16726744085,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":2248248609,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10415392160,"JVMOffHeapMemory":144626864,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":90,"MinorGCTime":2239,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4815839,"Executor Run Time":15901,"Executor CPU Time":15727411065,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":42,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11481,"Shuffle Write Time":11954510,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":256,"Index":0,"Attempt":0,"Launch Time":1646432662581,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432678545,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"358","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"875823104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"15903","Value":"181931","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"13028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"15914","Value":"182091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"15906","Value":"112901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"7213446","Value":"49432226","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1603","Value":"11424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1529158390","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"228606620","Value":"1529158390","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"8592","Value":"98304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"12849","Value":"154612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"358","Value":"4096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"13560864","Value":"177604449","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":63,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4845118,"Value":63861470,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":15928,"Value":182286,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":15569109136,"Value":179382861959,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":33318,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":226,"Value":1398,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":875823104,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12849,"Value":154612,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":4096,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":13560864,"Value":177604449,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":2476855229,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":80117900,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1603019861,"Value":18329763946,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":2476855229,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16372518112,"JVMOffHeapMemory":141975032,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1480676,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":86,"MinorGCTime":1111,"MajorGCCount":7,"MajorGCTime":624},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4845118,"Executor Run Time":15928,"Executor CPU Time":15569109136,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":226,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12849,"Shuffle Write Time":13560864,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":276,"Index":0,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432678623,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"18","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"15977","Value":"72657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"15983","Value":"72690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"7213446","Value":"31001713","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1659","Value":"7273","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"962867284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"228606620","Value":"962867284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"432","Value":"1848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"1224","Value":"5116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"18","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"3691180","Value":"18989129","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2909863,"Value":14355872,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":15988,"Value":72712,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":15823736356,"Value":71763032700,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2133,"Value":10664,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":42,"Value":488,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1224,"Value":5116,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":3691180,"Value":18989129,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":962867284,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":31001713,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1659938814,"Value":7275816655,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":962867284,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10415392160,"JVMOffHeapMemory":144626864,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":90,"MinorGCTime":2239,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":2909863,"Executor Run Time":15988,"Executor CPU Time":15823736356,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":42,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1224,"Shuffle Write Time":3691180,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":258,"Index":2,"Attempt":0,"Launch Time":1646432662582,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432678881,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"305","Value":"4401","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"943194112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"16228","Value":"198159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"14028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"16240","Value":"198331","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"16232","Value":"129133","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"6450117","Value":"55882343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1555","Value":"12979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1725972690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"196814300","Value":"1725972690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7320","Value":"105624","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11668","Value":"166280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"305","Value":"4401","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"14782833","Value":"192387282","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":70,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6283506,"Value":70144976,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":16256,"Value":198542,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":15953137885,"Value":195335999844,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2562,"Value":35880,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":172,"Value":1570,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":943194112,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":166280,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":4401,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":14782833,"Value":192387282,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":2673669529,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":86568017,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1555727028,"Value":19885490974,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":2673669529,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141420784,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":81,"MinorGCTime":1230,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":6283506,"Executor Run Time":16256,"Executor CPU Time":15953137885,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":172,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":14782833,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":283,"Index":7,"Attempt":0,"Launch Time":1646432662615,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432679003,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"14","Value":"91","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"16361","Value":"89018","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"16366","Value":"89056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"5819811","Value":"36821524","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1752","Value":"9025","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"1143561127","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"180693843","Value":"1143561127","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"336","Value":"2184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"952","Value":"6068","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"14","Value":"91","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"3377112","Value":"22366241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2158305,"Value":16514177,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":16370,"Value":89082,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":16049216984,"Value":87812249684,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2133,"Value":12797,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":226,"Value":714,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":6068,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":91,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":3377112,"Value":22366241,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":1143561127,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":36821524,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1752895808,"Value":9028712463,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":1143561127,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16372518112,"JVMOffHeapMemory":141975032,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1480676,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":86,"MinorGCTime":1111,"MajorGCCount":7,"MajorGCTime":624},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2158305,"Executor Run Time":16370,"Executor CPU Time":16049216984,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":226,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":3377112,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":273,"Index":17,"Attempt":0,"Launch Time":1646432662586,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432679060,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"324","Value":"4725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"1010565120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"16408","Value":"214567","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"15035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"16423","Value":"214754","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"16415","Value":"85506","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"5819811","Value":"36505485","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1543","Value":"8440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"1128390682","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"180693843","Value":"1128390682","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7776","Value":"113400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"12053","Value":"178333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"324","Value":"4725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"14668291","Value":"207055573","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":75,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4791010,"Value":74935986,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":16439,"Value":214981,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":16263179217,"Value":211599179061,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":38443,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":42,"Value":1612,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1010565120,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12053,"Value":178333,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":324,"Value":4725,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":14668291,"Value":207055573,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":2854363372,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":92387828,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1543485684,"Value":21428976658,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":2854363372,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10415392160,"JVMOffHeapMemory":144626864,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":90,"MinorGCTime":2239,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4791010,"Executor Run Time":16439,"Executor CPU Time":16263179217,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":42,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12053,"Shuffle Write Time":14668291,"Shuffle Records Written":324},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":266,"Index":10,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432679250,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"358","Value":"5083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"1077936128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"16595","Value":"231162","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"16035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"16606","Value":"231360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"16598","Value":"102104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"7213446","Value":"43718931","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1726","Value":"10166","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1356997302","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"228606620","Value":"1356997302","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"8592","Value":"121992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"12847","Value":"191180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"358","Value":"5083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"15282504","Value":"222338077","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":84,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6340441,"Value":81276427,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":16622,"Value":231603,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":16318366339,"Value":227917545400,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2562,"Value":41005,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":172,"Value":1784,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1077936128,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12847,"Value":191180,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":5083,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":15282504,"Value":222338077,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":3082969992,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":99601274,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1726406143,"Value":23155382801,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":3082969992,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141420784,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":81,"MinorGCTime":1230,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":6340441,"Executor Run Time":16622,"Executor CPU Time":16318366339,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":172,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12847,"Shuffle Write Time":15282504,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":270,"Index":14,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432679264,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"301","Value":"5384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"1145307136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"16608","Value":"247770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"17035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"16620","Value":"247980","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"16610","Value":"118714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"6066650","Value":"49785581","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1611","Value":"11777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1542033182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"185035880","Value":"1542033182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7224","Value":"129216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11463","Value":"202643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"301","Value":"5384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"16143352","Value":"238481429","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":94,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4271385,"Value":85547812,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":16636,"Value":248239,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":16334548396,"Value":244252093796,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2562,"Value":43567,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":172,"Value":1956,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1145307136,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11463,"Value":202643,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":301,"Value":5384,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":16143352,"Value":238481429,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":3268005872,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":105667924,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1611786372,"Value":24767169173,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":3268005872,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141420784,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":81,"MinorGCTime":1230,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":4271385,"Executor Run Time":16636,"Executor CPU Time":16334548396,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":172,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11463,"Shuffle Write Time":16143352,"Shuffle Records Written":301},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":281,"Index":5,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432679800,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"15","Value":"106","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"17149","Value":"106167","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"17157","Value":"106213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"6096762","Value":"42918286","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1609","Value":"10634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1327500635","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"183939508","Value":"1327500635","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"360","Value":"2544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"1020","Value":"7088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"15","Value":"106","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"3768705","Value":"26134946","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2632655,"Value":19146832,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":17162,"Value":106244,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":16875755058,"Value":104688004742,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2132,"Value":14929,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":172,"Value":886,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1020,"Value":7088,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":15,"Value":106,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":3768705,"Value":26134946,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":1327500635,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":42918286,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1609582382,"Value":10638294845,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":1327500635,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141420784,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":81,"MinorGCTime":1230,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":2632655,"Executor Run Time":17162,"Executor CPU Time":16875755058,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":172,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1020,"Shuffle Write Time":3768705,"Shuffle Records Written":15},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":282,"Index":6,"Attempt":0,"Launch Time":1646432662615,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432679851,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"14","Value":"120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"17211","Value":"123378","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"17216","Value":"123429","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"5819079","Value":"48737365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1545","Value":"12179","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"1509737500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"182236865","Value":"1509737500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"336","Value":"2880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"952","Value":"8040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"14","Value":"120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"3614260","Value":"29749206","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2032931,"Value":21179763,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":17219,"Value":123463,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":17027278894,"Value":121715283636,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2133,"Value":17062,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":60,"Value":946,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":8040,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":120,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":3614260,"Value":29749206,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":1509737500,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":48737365,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1545125238,"Value":12183420083,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":1509737500,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2032931,"Executor Run Time":17219,"Executor CPU Time":17027278894,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":60,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":3614260,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":264,"Index":8,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432679923,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"320","Value":"5704","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"1212678144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"17284","Value":"265054","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"18035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"17295","Value":"265275","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Update":"17288","Value":"146421","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Update":"5674621","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Update":"1501","Value":"14480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Update":"178217644","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7680","Value":"136896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"12024","Value":"214667","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"320","Value":"5704","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"11821126","Value":"250302555","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":98,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4075155,"Value":89622967,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":17308,"Value":265547,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":17009951238,"Value":261262045034,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":46130,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":226,"Value":2182,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1212678144,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12024,"Value":214667,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":5704,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":11821126,"Value":250302555,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":3446223516,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":111342545,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1501216169,"Value":26268385342,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":3446223516,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16372518112,"JVMOffHeapMemory":141975032,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1480676,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":86,"MinorGCTime":1111,"MajorGCCount":7,"MajorGCTime":624},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":4075155,"Executor Run Time":17308,"Executor CPU Time":17009951238,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":226,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12024,"Shuffle Write Time":11821126,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":271,"Index":15,"Attempt":0,"Launch Time":1646432662585,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432680417,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"313","Value":"6017","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"1280049152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"17772","Value":"282826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"19035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"17783","Value":"283058","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"17776","Value":"136490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"6096762","Value":"55882343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1532","Value":"13309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1725972690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"183939508","Value":"1725972690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7512","Value":"144408","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"11789","Value":"226456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"313","Value":"6017","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"11636863","Value":"261939418","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":103,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4911008,"Value":94533975,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":17796,"Value":283343,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":17597171196,"Value":278859216230,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2563,"Value":48693,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":60,"Value":2242,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1280049152,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11789,"Value":226456,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":6017,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":11636863,"Value":261939418,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":3630163024,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":117439307,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1532831128,"Value":27801216470,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":3630163024,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":4911008,"Executor Run Time":17796,"Executor CPU Time":17597171196,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":60,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11789,"Shuffle Write Time":11636863,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":274,"Index":18,"Attempt":0,"Launch Time":1646432662586,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432680586,"Failed":false,"Killed":false,"Accumulables":[{"ID":2166,"Name":"number of output rows","Update":"320","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Update":"67371008","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Update":"17932","Value":"300758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Update":"17945","Value":"301003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Update":"17937","Value":"154427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Update":"5674621","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Update":"1678","Value":"14987","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Update":"178217644","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Update":"7680","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Update":"12025","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Update":"320","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Update":"15801456","Value":"277740874","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Update":5,"Value":108,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5498704,"Value":100032679,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Update":17962,"Value":301305,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Update":17665869359,"Value":296525085589,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Update":2562,"Value":51255,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Update":172,"Value":2414,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12025,"Value":238481,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Update":15801456,"Value":277740874,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1678929609,"Value":29480146079,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":3808380668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141420784,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":81,"MinorGCTime":1230,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":5,"Executor Deserialize CPU Time":5498704,"Executor Run Time":17962,"Executor CPU Time":17665869359,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":172,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12025,"Shuffle Write Time":15801456,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":63,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":154,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7016\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[153],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":151,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7011\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[150],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":152,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_7012\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[148,151],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":146,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":149,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":148,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7008\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[147],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":150,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[149],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":147,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[146],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":153,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7015\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[152],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432662488,"Completion Time":1646432680586,"Accumulables":[{"ID":2166,"Name":"number of output rows","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2167,"Name":"peak memory","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2169,"Name":"time in aggregation build","Value":"300758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2170,"Name":"avg hash probe bucket list iters","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2172,"Name":"duration","Value":"301003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2173,"Name":"duration","Value":"146421","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2174,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2178,"Name":"filesystem read time (sampled)","Value":"14480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2179,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2180,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2182,"Name":"duration","Value":"154427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2183,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2187,"Name":"filesystem read time (sampled)","Value":"14987","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2188,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2189,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2191,"Name":"data size","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"shuffle bytes written","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2200,"Name":"shuffle records written","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2201,"Name":"shuffle write time","Value":"277740874","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2208,"Name":"internal.metrics.executorDeserializeTime","Value":108,"Internal":true,"Count Failed Values":true},{"ID":2209,"Name":"internal.metrics.executorDeserializeCpuTime","Value":100032679,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.executorRunTime","Value":301305,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.executorCpuTime","Value":296525085589,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.resultSize","Value":51255,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.jvmGCTime","Value":2414,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.peakExecutionMemory","Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":2228,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":238481,"Internal":true,"Count Failed Values":true},{"ID":2229,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":2230,"Name":"internal.metrics.shuffle.write.writeTime","Value":277740874,"Internal":true,"Count Failed Values":true},{"ID":2231,"Name":"internal.metrics.input.bytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":2232,"Name":"internal.metrics.input.recordsRead","Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":2233,"Name":"internal.metrics.input.sampledTimeReadNano","Value":29480146079,"Internal":true,"Count Failed Values":true},{"ID":2234,"Name":"internal.metrics.input.sampledBytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":35,"Completion Time":1646432680588,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":37,"Submission Time":1646432680654,"Stage Infos":[{"Stage ID":66,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":161,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7433\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[160],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":159,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7427\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[154],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":160,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7432\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[159],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[65],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":65,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":154,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7016\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[153],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":151,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7011\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[150],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":152,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_7012\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[148,151],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":146,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":149,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":148,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7008\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[147],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":150,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[149],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":147,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[146],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":153,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7015\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[152],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[66,65],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_7433\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":66,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":161,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7433\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[160],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":159,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7427\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[154],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":160,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7432\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[159],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[65],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432680655,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_7433\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":66,"Stage Attempt ID":0,"Task Info":{"Task ID":286,"Index":0,"Attempt":0,"Launch Time":1646432680676,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":66,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":286,"Index":0,"Attempt":0,"Launch Time":1646432680676,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432680710,"Failed":false,"Killed":false,"Accumulables":[{"ID":2192,"Name":"remote blocks read","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2193,"Name":"local blocks read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2194,"Name":"remote bytes read","Update":"179315","Value":"179315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2196,"Name":"local bytes read","Update":"59166","Value":"59166","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2197,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2198,"Name":"records read","Update":"6337","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2288,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2289,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2291,"Name":"time in aggregation build","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2292,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2293,"Name":"duration","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2296,"Name":"data size","Update":"9144","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2304,"Name":"shuffle bytes written","Update":"2241","Value":"2241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2305,"Name":"shuffle records written","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2306,"Name":"shuffle write time","Update":"203438","Value":"203438","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2307,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2308,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2487615,"Value":2487615,"Internal":true,"Count Failed Values":true},{"ID":2309,"Name":"internal.metrics.executorRunTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2310,"Name":"internal.metrics.executorCpuTime","Update":16845027,"Value":16845027,"Internal":true,"Count Failed Values":true},{"ID":2311,"Name":"internal.metrics.resultSize","Update":3168,"Value":3168,"Internal":true,"Count Failed Values":true},{"ID":2316,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2320,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2321,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2322,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":179315,"Value":179315,"Internal":true,"Count Failed Values":true},{"ID":2323,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2324,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":59166,"Value":59166,"Internal":true,"Count Failed Values":true},{"ID":2325,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2326,"Name":"internal.metrics.shuffle.read.recordsRead","Update":6337,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":2327,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2241,"Value":2241,"Internal":true,"Count Failed Values":true},{"ID":2328,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":381,"Value":381,"Internal":true,"Count Failed Values":true},{"ID":2329,"Name":"internal.metrics.shuffle.write.writeTime","Update":203438,"Value":203438,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2487615,"Executor Run Time":18,"Executor CPU Time":16845027,"Peak Execution Memory":67371008,"Result Size":3168,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":15,"Local Blocks Fetched":5,"Fetch Wait Time":0,"Remote Bytes Read":179315,"Remote Bytes Read To Disk":0,"Local Bytes Read":59166,"Total Records Read":6337},"Shuffle Write Metrics":{"Shuffle Bytes Written":2241,"Shuffle Write Time":203438,"Shuffle Records Written":381},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":66,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":161,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7433\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[160],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":159,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7427\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[154],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":160,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7432\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[159],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[65],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432680655,"Completion Time":1646432680711,"Accumulables":[{"ID":2192,"Name":"remote blocks read","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2193,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2194,"Name":"remote bytes read","Value":"179315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2196,"Name":"local bytes read","Value":"59166","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2197,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2198,"Name":"records read","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2288,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2289,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2291,"Name":"time in aggregation build","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2292,"Name":"avg hash probe bucket list iters","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2293,"Name":"duration","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2296,"Name":"data size","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2304,"Name":"shuffle bytes written","Value":"2241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2305,"Name":"shuffle records written","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2306,"Name":"shuffle write time","Value":"203438","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2307,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":2308,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2487615,"Internal":true,"Count Failed Values":true},{"ID":2309,"Name":"internal.metrics.executorRunTime","Value":18,"Internal":true,"Count Failed Values":true},{"ID":2310,"Name":"internal.metrics.executorCpuTime","Value":16845027,"Internal":true,"Count Failed Values":true},{"ID":2311,"Name":"internal.metrics.resultSize","Value":3168,"Internal":true,"Count Failed Values":true},{"ID":2316,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2320,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":15,"Internal":true,"Count Failed Values":true},{"ID":2321,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2322,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":179315,"Internal":true,"Count Failed Values":true},{"ID":2323,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2324,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":59166,"Internal":true,"Count Failed Values":true},{"ID":2325,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2326,"Name":"internal.metrics.shuffle.read.recordsRead","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":2327,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2241,"Internal":true,"Count Failed Values":true},{"ID":2328,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":381,"Internal":true,"Count Failed Values":true},{"ID":2329,"Name":"internal.metrics.shuffle.write.writeTime","Value":203438,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":37,"Completion Time":1646432680714,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":38,"Submission Time":1646432680781,"Stage Infos":[{"Stage ID":67,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":154,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7016\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[153],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":151,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7011\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[150],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":152,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_7012\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[148,151],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":146,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":149,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":148,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7008\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[147],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":150,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6539\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[149],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":147,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6537\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[146],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":153,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7015\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[152],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":68,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":161,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7433\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[160],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":159,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7427\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[154],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":160,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7432\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[159],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[67],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":69,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":163,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7921\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[162],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":162,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[161],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[68],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[67,68,69],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_8\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 6f352b02-b872-4450-9f89-f8755aa70b77)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"6f352b02-b872-4450-9f89-f8755aa70b77","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":69,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":163,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7921\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[162],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":162,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[161],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[68],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432680782,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_8\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 6f352b02-b872-4450-9f89-f8755aa70b77)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"6f352b02-b872-4450-9f89-f8755aa70b77","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":69,"Stage Attempt ID":0,"Task Info":{"Task ID":287,"Index":0,"Attempt":0,"Launch Time":1646432680788,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":69,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":287,"Index":0,"Attempt":0,"Launch Time":1646432680788,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432680844,"Failed":false,"Killed":false,"Accumulables":[{"ID":2297,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2299,"Name":"remote bytes read","Update":"2241","Value":"2241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2302,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2303,"Name":"records read","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2337,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2338,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2339,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2340,"Name":"cumulative time","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2341,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2342,"Name":"num prefix comparisons","Update":"2767","Value":"2767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2344,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2345,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2346,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2347,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3680909,"Value":3680909,"Internal":true,"Count Failed Values":true},{"ID":2348,"Name":"internal.metrics.executorRunTime","Update":41,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2349,"Name":"internal.metrics.executorCpuTime","Update":35233776,"Value":35233776,"Internal":true,"Count Failed Values":true},{"ID":2350,"Name":"internal.metrics.resultSize","Update":8045,"Value":8045,"Internal":true,"Count Failed Values":true},{"ID":2355,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2359,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2360,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2361,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":2241,"Value":2241,"Internal":true,"Count Failed Values":true},{"ID":2362,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2363,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2364,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2365,"Name":"internal.metrics.shuffle.read.recordsRead","Update":381,"Value":381,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3680909,"Executor Run Time":41,"Executor CPU Time":35233776,"Peak Execution Memory":67174400,"Result Size":8045,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":2241,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":381},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":69,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":163,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7921\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[162],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":162,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[161],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[68],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432680782,"Completion Time":1646432680844,"Accumulables":[{"ID":2297,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2299,"Name":"remote bytes read","Value":"2241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2302,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2303,"Name":"records read","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2337,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2338,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2339,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2340,"Name":"cumulative time","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2341,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2342,"Name":"num prefix comparisons","Value":"2767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2344,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2345,"Name":"duration","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2346,"Name":"internal.metrics.executorDeserializeTime","Value":3,"Internal":true,"Count Failed Values":true},{"ID":2347,"Name":"internal.metrics.executorDeserializeCpuTime","Value":3680909,"Internal":true,"Count Failed Values":true},{"ID":2348,"Name":"internal.metrics.executorRunTime","Value":41,"Internal":true,"Count Failed Values":true},{"ID":2349,"Name":"internal.metrics.executorCpuTime","Value":35233776,"Internal":true,"Count Failed Values":true},{"ID":2350,"Name":"internal.metrics.resultSize","Value":8045,"Internal":true,"Count Failed Values":true},{"ID":2355,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2359,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2360,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2361,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":2241,"Internal":true,"Count Failed Values":true},{"ID":2362,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2363,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2364,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2365,"Name":"internal.metrics.shuffle.read.recordsRead","Value":381,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":38,"Completion Time":1646432680845,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":278,"Index":2,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432681007,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"18","Value":"138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"18367","Value":"141745","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"18372","Value":"141801","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"6450117","Value":"55187482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1621","Value":"13800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1706551800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"196814300","Value":"1706551800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"432","Value":"3312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"1164","Value":"9204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"18","Value":"138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"3572357","Value":"33321563","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2205147,"Value":23384910,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":18375,"Value":141838,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":18187047024,"Value":139902330660,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2133,"Value":19195,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":60,"Value":1006,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1164,"Value":9204,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":138,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":3572357,"Value":33321563,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":1706551800,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":55187482,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1621815154,"Value":13805235237,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":1706551800,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2205147,"Executor Run Time":18375,"Executor CPU Time":18187047024,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":60,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1164,"Shuffle Write Time":3572357,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":252,"Index":6,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432681079,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"18613","Value":"18613","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1293","Value":"1293","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"558631584","Value":"558631584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"77834569","Value":"77834569","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"185206820","Value":"185206820","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2222242,"Value":2222242,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":18702,"Value":18702,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":18255170331,"Value":18255170331,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1703,"Value":1703,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":226,"Value":226,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":77834569,"Value":77834569,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819079,"Value":5819079,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":185206820,"Value":185206820,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":5819079,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1293251806,"Value":1293251806,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16372518112,"JVMOffHeapMemory":141975032,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1480676,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":86,"MinorGCTime":1111,"MajorGCCount":7,"MajorGCTime":624},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2222242,"Executor Run Time":18702,"Executor CPU Time":18255170331,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":226,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":77834569,"Shuffle Write Time":185206820,"Shuffle Records Written":5819079},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":251,"Index":5,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432681088,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"6096762","Value":"11915841","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"18624","Value":"37237","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"6096762","Value":"11915841","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1362","Value":"2655","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"366176373","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"183939508","Value":"366176373","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"585289152","Value":"1143920736","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"80392000","Value":"158226569","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"6096762","Value":"11915841","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"183466356","Value":"368673176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1894904,"Value":4117146,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":18712,"Value":37414,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":18492419908,"Value":36747590239,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1703,"Value":3406,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":60,"Value":286,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":80392000,"Value":158226569,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6096762,"Value":11915841,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":183466356,"Value":368673176,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":366176373,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":11915841,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1362608624,"Value":2655860430,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":366176373,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1894904,"Executor Run Time":18712,"Executor CPU Time":18492419908,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":60,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":80392000,"Shuffle Write Time":183466356,"Shuffle Records Written":6096762},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":277,"Index":1,"Attempt":0,"Launch Time":1646432662614,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432681340,"Failed":false,"Killed":false,"Accumulables":[{"ID":2202,"Name":"number of output rows","Update":"16","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Update":"18690","Value":"160435","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Update":"18698","Value":"160499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Update":"6369482","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Update":"1809","Value":"15609","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Update":"197638534","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Update":"384","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Update":"1088","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Update":"16","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Update":"3746302","Value":"37067865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2779469,"Value":26164379,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Update":18702,"Value":160540,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Update":18403532546,"Value":158305863206,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Update":2132,"Value":21327,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Update":172,"Value":1178,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1088,"Value":10292,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Update":3746302,"Value":37067865,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1809799707,"Value":15615034944,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141420784,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":81,"MinorGCTime":1230,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":2779469,"Executor Run Time":18702,"Executor CPU Time":18403532546,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":172,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1088,"Shuffle Write Time":3746302,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":64,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":158,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7092\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[157],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":157,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7091\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[156],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":156,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[155],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":155,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432662584,"Completion Time":1646432681340,"Accumulables":[{"ID":2202,"Name":"number of output rows","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2203,"Name":"peak memory","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2205,"Name":"time in aggregation build","Value":"160435","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2206,"Name":"avg hash probe bucket list iters","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2238,"Name":"duration","Value":"160499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2239,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2243,"Name":"filesystem read time (sampled)","Value":"15609","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2244,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2245,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2247,"Name":"data size","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"shuffle bytes written","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"shuffle records written","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"shuffle write time","Value":"37067865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"internal.metrics.executorDeserializeTime","Value":36,"Internal":true,"Count Failed Values":true},{"ID":2259,"Name":"internal.metrics.executorDeserializeCpuTime","Value":26164379,"Internal":true,"Count Failed Values":true},{"ID":2260,"Name":"internal.metrics.executorRunTime","Value":160540,"Internal":true,"Count Failed Values":true},{"ID":2261,"Name":"internal.metrics.executorCpuTime","Value":158305863206,"Internal":true,"Count Failed Values":true},{"ID":2262,"Name":"internal.metrics.resultSize","Value":21327,"Internal":true,"Count Failed Values":true},{"ID":2263,"Name":"internal.metrics.jvmGCTime","Value":1178,"Internal":true,"Count Failed Values":true},{"ID":2267,"Name":"internal.metrics.peakExecutionMemory","Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":2278,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":10292,"Internal":true,"Count Failed Values":true},{"ID":2279,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":154,"Internal":true,"Count Failed Values":true},{"ID":2280,"Name":"internal.metrics.shuffle.write.writeTime","Value":37067865,"Internal":true,"Count Failed Values":true},{"ID":2281,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":2282,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2283,"Name":"internal.metrics.input.sampledTimeReadNano","Value":15615034944,"Internal":true,"Count Failed Values":true},{"ID":2284,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":36,"Completion Time":1646432681341,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":39,"Submission Time":1646432681369,"Stage Infos":[{"Stage ID":70,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":158,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7092\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[157],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":157,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7091\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[156],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":156,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[155],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":155,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":71,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":166,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8353\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[165],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":165,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8352\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[164],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":164,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_8347\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[158],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[70],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[70,71],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_8353\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":71,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":166,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8353\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[165],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":165,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8352\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[164],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":164,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_8347\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[158],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[70],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432681370,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_8353\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":71,"Stage Attempt ID":0,"Task Info":{"Task ID":288,"Index":0,"Attempt":0,"Launch Time":1646432681376,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":71,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":288,"Index":0,"Attempt":0,"Launch Time":1646432681376,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432681429,"Failed":false,"Killed":false,"Accumulables":[{"ID":2248,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2249,"Name":"local blocks read","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2250,"Name":"remote bytes read","Update":"7368","Value":"7368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2252,"Name":"local bytes read","Update":"2924","Value":"2924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2253,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2254,"Name":"records read","Update":"154","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2381,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2382,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2384,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2385,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2386,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2389,"Name":"data size","Update":"552","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2397,"Name":"shuffle bytes written","Update":"264","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2398,"Name":"shuffle records written","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2399,"Name":"shuffle write time","Update":"218246","Value":"218246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2400,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2401,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2394436,"Value":2394436,"Internal":true,"Count Failed Values":true},{"ID":2402,"Name":"internal.metrics.executorRunTime","Update":38,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":2403,"Name":"internal.metrics.executorCpuTime","Update":33337130,"Value":33337130,"Internal":true,"Count Failed Values":true},{"ID":2404,"Name":"internal.metrics.resultSize","Update":2737,"Value":2737,"Internal":true,"Count Failed Values":true},{"ID":2409,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2413,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2414,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2415,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":7368,"Value":7368,"Internal":true,"Count Failed Values":true},{"ID":2416,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2417,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2924,"Value":2924,"Internal":true,"Count Failed Values":true},{"ID":2418,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2419,"Name":"internal.metrics.shuffle.read.recordsRead","Update":154,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":2420,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":264,"Value":264,"Internal":true,"Count Failed Values":true},{"ID":2421,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2422,"Name":"internal.metrics.shuffle.write.writeTime","Update":218246,"Value":218246,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2394436,"Executor Run Time":38,"Executor CPU Time":33337130,"Peak Execution Memory":67371008,"Result Size":2737,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":3,"Fetch Wait Time":0,"Remote Bytes Read":7368,"Remote Bytes Read To Disk":0,"Local Bytes Read":2924,"Total Records Read":154},"Shuffle Write Metrics":{"Shuffle Bytes Written":264,"Shuffle Write Time":218246,"Shuffle Records Written":23},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":71,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":166,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8353\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[165],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":165,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8352\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[164],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":164,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_8347\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[158],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[70],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432681370,"Completion Time":1646432681429,"Accumulables":[{"ID":2248,"Name":"remote blocks read","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2249,"Name":"local blocks read","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2250,"Name":"remote bytes read","Value":"7368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2252,"Name":"local bytes read","Value":"2924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2253,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2254,"Name":"records read","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2381,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2382,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2384,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2385,"Name":"avg hash probe bucket list iters","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2386,"Name":"duration","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2389,"Name":"data size","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2397,"Name":"shuffle bytes written","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2398,"Name":"shuffle records written","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2399,"Name":"shuffle write time","Value":"218246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2400,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":2401,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2394436,"Internal":true,"Count Failed Values":true},{"ID":2402,"Name":"internal.metrics.executorRunTime","Value":38,"Internal":true,"Count Failed Values":true},{"ID":2403,"Name":"internal.metrics.executorCpuTime","Value":33337130,"Internal":true,"Count Failed Values":true},{"ID":2404,"Name":"internal.metrics.resultSize","Value":2737,"Internal":true,"Count Failed Values":true},{"ID":2409,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2413,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":7,"Internal":true,"Count Failed Values":true},{"ID":2414,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":3,"Internal":true,"Count Failed Values":true},{"ID":2415,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":7368,"Internal":true,"Count Failed Values":true},{"ID":2416,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2417,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2924,"Internal":true,"Count Failed Values":true},{"ID":2418,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2419,"Name":"internal.metrics.shuffle.read.recordsRead","Value":154,"Internal":true,"Count Failed Values":true},{"ID":2420,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":264,"Internal":true,"Count Failed Values":true},{"ID":2421,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23,"Internal":true,"Count Failed Values":true},{"ID":2422,"Name":"internal.metrics.shuffle.write.writeTime","Value":218246,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":39,"Completion Time":1646432681430,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":40,"Submission Time":1646432681521,"Stage Infos":[{"Stage ID":74,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":168,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8475\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[167],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":167,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_11\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[166],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[73],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":72,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":158,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7092\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[157],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":157,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_7091\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[156],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":156,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[155],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":155,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6559\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":73,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":166,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8353\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[165],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":165,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8352\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[164],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":164,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_8347\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[158],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[72],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[74,72,73],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_12\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 9e07d180-f072-43ec-bc0e-9e1e6480dae3)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"9e07d180-f072-43ec-bc0e-9e1e6480dae3","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":74,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":168,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8475\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[167],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":167,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_11\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[166],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[73],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432681521,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_12\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 9e07d180-f072-43ec-bc0e-9e1e6480dae3)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"9e07d180-f072-43ec-bc0e-9e1e6480dae3","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":74,"Stage Attempt ID":0,"Task Info":{"Task ID":289,"Index":0,"Attempt":0,"Launch Time":1646432681590,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":74,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":289,"Index":0,"Attempt":0,"Launch Time":1646432681590,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432681671,"Failed":false,"Killed":false,"Accumulables":[{"ID":2390,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2392,"Name":"remote bytes read","Update":"264","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2395,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2396,"Name":"records read","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2430,"Name":"sort time","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2431,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2432,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2433,"Name":"cumulative time","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2434,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2435,"Name":"num prefix comparisons","Update":"75","Value":"75","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2437,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2438,"Name":"duration","Update":"31","Value":"31","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2439,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2440,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7808271,"Value":7808271,"Internal":true,"Count Failed Values":true},{"ID":2441,"Name":"internal.metrics.executorRunTime","Update":61,"Value":61,"Internal":true,"Count Failed Values":true},{"ID":2442,"Name":"internal.metrics.executorCpuTime","Update":46402838,"Value":46402838,"Internal":true,"Count Failed Values":true},{"ID":2443,"Name":"internal.metrics.resultSize","Update":3792,"Value":3792,"Internal":true,"Count Failed Values":true},{"ID":2448,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2452,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2453,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2454,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":264,"Value":264,"Internal":true,"Count Failed Values":true},{"ID":2455,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2456,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2457,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2458,"Name":"internal.metrics.shuffle.read.recordsRead","Update":23,"Value":23,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":7808271,"Executor Run Time":61,"Executor CPU Time":46402838,"Peak Execution Memory":67174400,"Result Size":3792,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":264,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":23},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":74,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":168,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8475\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[167],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":167,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_11\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[166],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[73],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432681521,"Completion Time":1646432681671,"Accumulables":[{"ID":2390,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2392,"Name":"remote bytes read","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2395,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2396,"Name":"records read","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2430,"Name":"sort time","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2431,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2432,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2433,"Name":"cumulative time","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2434,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2435,"Name":"num prefix comparisons","Value":"75","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2437,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2438,"Name":"duration","Value":"31","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2439,"Name":"internal.metrics.executorDeserializeTime","Value":7,"Internal":true,"Count Failed Values":true},{"ID":2440,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7808271,"Internal":true,"Count Failed Values":true},{"ID":2441,"Name":"internal.metrics.executorRunTime","Value":61,"Internal":true,"Count Failed Values":true},{"ID":2442,"Name":"internal.metrics.executorCpuTime","Value":46402838,"Internal":true,"Count Failed Values":true},{"ID":2443,"Name":"internal.metrics.resultSize","Value":3792,"Internal":true,"Count Failed Values":true},{"ID":2448,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2452,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2453,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2454,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":264,"Internal":true,"Count Failed Values":true},{"ID":2455,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2456,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2457,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2458,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":40,"Completion Time":1646432681671,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":253,"Index":7,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432682552,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"5819811","Value":"17735652","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"20089","Value":"57326","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"5819811","Value":"17735652","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1379","Value":"4034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"546870216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"180693843","Value":"546870216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"558701856","Value":"1702622592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"78151025","Value":"236377594","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"5819811","Value":"17735652","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"209386471","Value":"578059647","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2218620,"Value":6335766,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":20176,"Value":57590,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":19929502313,"Value":56677092552,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1703,"Value":5109,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":42,"Value":328,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":78151025,"Value":236377594,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819811,"Value":17735652,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":209386471,"Value":578059647,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":546870216,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":17735652,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1379088564,"Value":4034948994,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":546870216,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11040285048,"JVMOffHeapMemory":144973352,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141439672,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":2257,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2218620,"Executor Run Time":20176,"Executor CPU Time":19929502313,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":42,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":78151025,"Shuffle Write Time":209386471,"Shuffle Records Written":5819811},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":247,"Index":1,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432682741,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"6369482","Value":"24105134","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"20271","Value":"77597","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"6369482","Value":"24105134","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1453","Value":"5487","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"744508750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"197638534","Value":"744508750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"611470272","Value":"2314092864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"85250127","Value":"321627721","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"6369482","Value":"24105134","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"197230734","Value":"775290381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1965597,"Value":8301363,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":20365,"Value":77955,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":20134049680,"Value":76811142232,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1703,"Value":6812,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":69,"Value":397,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85250127,"Value":321627721,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6369482,"Value":24105134,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":197230734,"Value":775290381,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":744508750,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":24105134,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1453431742,"Value":5488380736,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":744508750,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1965597,"Executor Run Time":20365,"Executor CPU Time":20134049680,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":69,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85250127,"Shuffle Write Time":197230734,"Shuffle Records Written":6369482},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":248,"Index":2,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432682957,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"6450117","Value":"30555251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"20484","Value":"98081","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"6450117","Value":"30555251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1436","Value":"6923","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"941323050","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"196814300","Value":"941323050","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"619211232","Value":"2933304096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"84724622","Value":"406352343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"6450117","Value":"30555251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"198546101","Value":"973836482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2281011,"Value":10582374,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":20580,"Value":98535,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":20136469434,"Value":96947611666,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1703,"Value":8515,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":232,"Value":629,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":84724622,"Value":406352343,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6450117,"Value":30555251,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":198546101,"Value":973836482,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":941323050,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":30555251,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1436717142,"Value":6925097878,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":941323050,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16372518112,"JVMOffHeapMemory":141975032,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1480676,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":86,"MinorGCTime":1111,"MajorGCCount":7,"MajorGCTime":624},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2281011,"Executor Run Time":20580,"Executor CPU Time":20136469434,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":232,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":84724622,"Shuffle Write Time":198546101,"Shuffle Records Written":6450117},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":255,"Index":9,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432684330,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"5617658","Value":"36172909","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"21875","Value":"119956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"5617658","Value":"36172909","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1457","Value":"8380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"1117255875","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"175932825","Value":"1117255875","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"539295168","Value":"3472599264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"74785065","Value":"481137408","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"5617658","Value":"36172909","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"180093315","Value":"1153929797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1761164,"Value":12343538,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":21954,"Value":120489,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":21727346303,"Value":118674957969,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1703,"Value":10218,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":69,"Value":698,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":74785065,"Value":481137408,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5617658,"Value":36172909,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":180093315,"Value":1153929797,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":1117255875,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":36172909,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1457226469,"Value":8382324347,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":1117255875,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":12142177680,"JVMOffHeapMemory":142781248,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":612956350,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1620183,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":88,"MinorGCTime":1394,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1761164,"Executor Run Time":21954,"Executor CPU Time":21727346303,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":69,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":74785065,"Shuffle Write Time":180093315,"Shuffle Records Written":5617658},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":254,"Index":8,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432685858,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"5674621","Value":"41847530","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"23389","Value":"143345","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"5674621","Value":"41847530","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1569","Value":"9949","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1295473519","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"178217644","Value":"1295473519","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"544763616","Value":"4017362880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"75751912","Value":"556889320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"5674621","Value":"41847530","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"233177659","Value":"1387107456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2755771,"Value":15099309,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":23480,"Value":143969,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":23014245796,"Value":141689203765,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1702,"Value":11920,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":238,"Value":936,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":75751912,"Value":556889320,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5674621,"Value":41847530,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":233177659,"Value":1387107456,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":1295473519,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":41847530,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1569130440,"Value":9951454787,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":1295473519,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141949232,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":84,"MinorGCTime":1353,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2755771,"Executor Run Time":23480,"Executor CPU Time":23014245796,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":238,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":75751912,"Shuffle Write Time":233177659,"Shuffle Records Written":5674621},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":249,"Index":3,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432686126,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"6429338","Value":"48276868","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"23658","Value":"167003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"6429338","Value":"48276868","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1584","Value":"11533","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1490547834","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"195074315","Value":"1490547834","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"617216448","Value":"4634579328","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"83388371","Value":"640277691","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"6429338","Value":"48276868","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"228014101","Value":"1615121557","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2212754,"Value":17312063,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":23750,"Value":167719,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":23511171804,"Value":165200375569,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1703,"Value":13623,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":49,"Value":985,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83388371,"Value":640277691,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6429338,"Value":48276868,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":228014101,"Value":1615121557,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1490547834,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":48276868,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1584630345,"Value":11536085132,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1490547834,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":11040285048,"JVMOffHeapMemory":144973352,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141439672,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1614039,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":2257,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2212754,"Executor Run Time":23750,"Executor CPU Time":23511171804,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":49,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83388371,"Shuffle Write Time":228014101,"Shuffle Records Written":6429338},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":246,"Index":0,"Attempt":0,"Launch Time":1646432662357,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432686459,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"7213446","Value":"55490314","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"23970","Value":"190973","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"7213446","Value":"55490314","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1775","Value":"13308","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1719154454","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"228606620","Value":"1719154454","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"692490816","Value":"5327070144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"98606155","Value":"738883846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"7213446","Value":"55490314","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"281106126","Value":"1896227683","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2838539,"Value":20150602,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":24081,"Value":191800,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":23609067792,"Value":188809443361,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1702,"Value":15325,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":238,"Value":1223,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":98606155,"Value":738883846,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7213446,"Value":55490314,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":281106126,"Value":1896227683,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1719154454,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":55490314,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1775924669,"Value":13312009801,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1719154454,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141949232,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":84,"MinorGCTime":1353,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2838539,"Executor Run Time":24081,"Executor CPU Time":23609067792,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":238,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":98606155,"Shuffle Write Time":281106126,"Shuffle Records Written":7213446},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":250,"Index":4,"Attempt":0,"Launch Time":1646432662358,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432687114,"Failed":false,"Killed":false,"Accumulables":[{"ID":2115,"Name":"number of output rows","Update":"6066650","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Update":"24644","Value":"215617","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Update":"6066650","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Update":"1658","Value":"14966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Update":"185035880","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Update":"582398400","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Update":"79904590","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Update":"6066650","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Update":"242243480","Value":"2138471163","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2657961,"Value":22808563,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Update":24736,"Value":216536,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Update":24264535138,"Value":213073978499,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Update":1702,"Value":17027,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Update":238,"Value":1461,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79904590,"Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6066650,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Update":242243480,"Value":2138471163,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1658728489,"Value":14970738290,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523911752,"JVMOffHeapMemory":141949232,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":141359294,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":680327358,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1539175,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":84,"MinorGCTime":1353,"MajorGCCount":6,"MajorGCTime":577},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2657961,"Executor Run Time":24736,"Executor CPU Time":24264535138,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":238,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":79904590,"Shuffle Write Time":242243480,"Shuffle Records Written":6066650},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":62,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":145,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6996\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[144],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":143,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6534\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[142],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":144,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6995\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[143],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":142,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6534\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Submission Time":1646432662353,"Completion Time":1646432687114,"Accumulables":[{"ID":2115,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2116,"Name":"duration","Value":"215617","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2117,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2121,"Name":"filesystem read time (sampled)","Value":"14966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2122,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2123,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2125,"Name":"data size","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2133,"Name":"shuffle bytes written","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2134,"Name":"shuffle records written","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2135,"Name":"shuffle write time","Value":"2138471163","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2136,"Name":"internal.metrics.executorDeserializeTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.executorDeserializeCpuTime","Value":22808563,"Internal":true,"Count Failed Values":true},{"ID":2138,"Name":"internal.metrics.executorRunTime","Value":216536,"Internal":true,"Count Failed Values":true},{"ID":2139,"Name":"internal.metrics.executorCpuTime","Value":213073978499,"Internal":true,"Count Failed Values":true},{"ID":2140,"Name":"internal.metrics.resultSize","Value":17027,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.jvmGCTime","Value":1461,"Internal":true,"Count Failed Values":true},{"ID":2156,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2157,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2158,"Name":"internal.metrics.shuffle.write.writeTime","Value":2138471163,"Internal":true,"Count Failed Values":true},{"ID":2159,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":2160,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2161,"Name":"internal.metrics.input.sampledTimeReadNano","Value":14970738290,"Internal":true,"Count Failed Values":true},{"ID":2162,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":34,"Completion Time":1646432687115,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":41,"Submission Time":1646432687304,"Stage Infos":[{"Stage ID":75,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":145,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6996\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[144],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":143,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6534\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[142],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":144,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_6995\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[143],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":142,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_6534\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":76,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":50,"RDD Info":[{"RDD ID":177,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[176],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":173,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"282\",\"name\":\"mapPartitions\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[172],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":176,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[175],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":175,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"284\",\"name\":\"map\"}","Callsite":"map at RegressionMetrics.scala:63","Parent IDs":[174],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":174,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"283\",\"name\":\"map\"}","Callsite":"map at LinearRegression.scala:914","Parent IDs":[173],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":172,"Name":"SQLExecutionRDD","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[171],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":171,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8650\",\"name\":\"DeserializeToObject\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[170],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":169,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_8623\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[145],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":170,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8649\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[169],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[75],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":77,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":6,"RDD Info":[{"RDD ID":179,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[178],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":178,"Name":"ShuffledRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[177],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[76],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[75,76,77],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":76,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":50,"RDD Info":[{"RDD ID":177,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[176],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":173,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"282\",\"name\":\"mapPartitions\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[172],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":176,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[175],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":175,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"284\",\"name\":\"map\"}","Callsite":"map at RegressionMetrics.scala:63","Parent IDs":[174],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":174,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"283\",\"name\":\"map\"}","Callsite":"map at LinearRegression.scala:914","Parent IDs":[173],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":172,"Name":"SQLExecutionRDD","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[171],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":171,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8650\",\"name\":\"DeserializeToObject\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[170],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":169,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_8623\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[145],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":170,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8649\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[169],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[75],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)","Submission Time":1646432687308,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":290,"Index":15,"Attempt":0,"Launch Time":1646432687315,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":291,"Index":10,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":292,"Index":5,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":293,"Index":0,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":294,"Index":16,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":295,"Index":11,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":296,"Index":6,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":297,"Index":1,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":298,"Index":17,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":299,"Index":12,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":300,"Index":7,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":301,"Index":2,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":302,"Index":18,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":303,"Index":13,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":304,"Index":8,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":305,"Index":3,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":306,"Index":19,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":307,"Index":14,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":308,"Index":9,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":309,"Index":4,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":310,"Index":35,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":311,"Index":30,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":312,"Index":25,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":313,"Index":20,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":314,"Index":36,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":315,"Index":31,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":316,"Index":26,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":317,"Index":21,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":318,"Index":37,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":319,"Index":32,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":320,"Index":27,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":321,"Index":22,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":322,"Index":38,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":323,"Index":33,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":324,"Index":28,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":325,"Index":23,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":326,"Index":39,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":327,"Index":34,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":328,"Index":29,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":329,"Index":24,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":330,"Index":45,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":331,"Index":40,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":332,"Index":46,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":333,"Index":41,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":334,"Index":47,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":335,"Index":42,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":336,"Index":48,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":337,"Index":43,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":338,"Index":49,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":339,"Index":44,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":311,"Index":30,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432690168,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"5197262","Value":"5197262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"403027","Value":"403027","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"403027","Value":"403027","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"403027","Value":"403027","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"395697","Value":"395697","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"296","Value":"296","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"150994944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"2247","Value":"2247","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"2671694","Value":"2671694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"2648851","Value":"2648851","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"277153","Value":"277153","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"2702","Value":"2702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":68,"Value":68,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":44876474,"Value":44876474,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":2762,"Value":2762,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":2639850383,"Value":2639850383,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":8189,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":41,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":352357648,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5197262,"Value":5197262,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":403027,"Value":403027,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":822,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":87049,"Value":87049,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":68,"Executor Deserialize CPU Time":44876474,"Executor Run Time":2762,"Executor CPU Time":2639850383,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":41,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5197262,"Total Records Read":403027},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":87049,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":310,"Index":35,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432690185,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"5007038","Value":"10204300","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"385946","Value":"788973","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"385946","Value":"788973","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"385946","Value":"788973","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"376002","Value":"771699","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"330","Value":"626","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"301989888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"2243","Value":"4490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"2450583","Value":"5122277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"2429287","Value":"5078138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"263017","Value":"540170","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"2708","Value":"5410","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":73,"Value":141,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":21379005,"Value":66255479,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":2776,"Value":5538,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":2632896372,"Value":5272746755,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":16378,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":704715296,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5007038,"Value":10204300,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":385946,"Value":788973,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":819,"Value":1641,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":120188,"Value":207237,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":73,"Executor Deserialize CPU Time":21379005,"Executor Run Time":2776,"Executor CPU Time":2632896372,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5007038,"Total Records Read":385946},"Shuffle Write Metrics":{"Shuffle Bytes Written":819,"Shuffle Write Time":120188,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":290,"Index":15,"Attempt":0,"Launch Time":1646432687315,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432690277,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"5549390","Value":"15753690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"443853","Value":"1232826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"443853","Value":"1232826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"443853","Value":"1232826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"436895","Value":"1208594","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"367","Value":"993","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"452984832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"2494","Value":"6984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"2885625","Value":"8007902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"2866006","Value":"7944144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"306336","Value":"846506","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"2805","Value":"8215","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":72,"Value":213,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":20026031,"Value":86281510,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":2869,"Value":8407,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":2735606751,"Value":8008353506,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":24567,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":113,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":1057072944,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5549390,"Value":15753690,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":443853,"Value":1232826,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":821,"Value":2462,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":59949,"Value":267186,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":72,"Executor Deserialize CPU Time":20026031,"Executor Run Time":2869,"Executor CPU Time":2735606751,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5549390,"Total Records Read":443853},"Shuffle Write Metrics":{"Shuffle Bytes Written":821,"Shuffle Write Time":59949,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":312,"Index":25,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432690397,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"5062619","Value":"20816309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"397041","Value":"1629867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"397041","Value":"1629867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"397041","Value":"1629867","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"390517","Value":"1599111","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"243","Value":"1236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"603979776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"2474","Value":"9458","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"2548570","Value":"10556472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"2526559","Value":"10470703","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"273437","Value":"1119943","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"2934","Value":"11149","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":282,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":22992700,"Value":109274210,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":2991,"Value":11398,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":2725446037,"Value":10733799543,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":32756,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":32,"Value":145,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":1409430592,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5062619,"Value":20816309,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":397041,"Value":1629867,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":3285,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":97429,"Value":364615,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":22992700,"Executor Run Time":2991,"Executor CPU Time":2725446037,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":32,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5062619,"Total Records Read":397041},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":97429,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":291,"Index":10,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432690408,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"5569222","Value":"26385531","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"436341","Value":"2066208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"436341","Value":"2066208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"436341","Value":"2066208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"426737","Value":"2025848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"4","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"307","Value":"1543","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"754974720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"2360","Value":"11818","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"2806335","Value":"13362807","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"2785368","Value":"13256071","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"299008","Value":"1418951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"2943","Value":"14092","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":351,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":24275574,"Value":133549784,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":3000,"Value":14398,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":2700238899,"Value":13434038442,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":40945,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":219,"Value":364,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":1761788240,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5569222,"Value":26385531,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":436341,"Value":2066208,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":4107,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":69564,"Value":434179,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":24275574,"Executor Run Time":3000,"Executor CPU Time":2700238899,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":219,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5569222,"Total Records Read":436341},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":69564,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":330,"Index":45,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432690417,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"4974479","Value":"31360010","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"387227","Value":"2453435","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"387227","Value":"2453435","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"8","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"387227","Value":"2453435","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"381516","Value":"2407364","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"2","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"232","Value":"1775","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"905969664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"2417","Value":"14235","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"2458158","Value":"15820965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"2436258","Value":"15692329","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"267201","Value":"1686152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"2956","Value":"17048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":70,"Value":421,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":20987444,"Value":154537228,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":3011,"Value":17409,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":2760047074,"Value":16194085516,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":49134,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":32,"Value":396,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":2114145888,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":4974479,"Value":31360010,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":387227,"Value":2453435,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":4929,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":59362,"Value":493541,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":70,"Executor Deserialize CPU Time":20987444,"Executor Run Time":3011,"Executor CPU Time":2760047074,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":32,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":4974479,"Total Records Read":387227},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":59362,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":313,"Index":20,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432690603,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"5077427","Value":"36437437","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"397065","Value":"2850500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"397065","Value":"2850500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"5","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"397065","Value":"2850500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"388533","Value":"2795897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"296","Value":"2071","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"1056964608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"2556","Value":"16791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"2526318","Value":"18347283","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"2507081","Value":"18199410","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"272002","Value":"1958154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"3094","Value":"20142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":95,"Value":516,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19382944,"Value":173920172,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":3165,"Value":20574,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":2829076042,"Value":19023161558,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":57322,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":26,"Value":422,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":2466503536,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5077427,"Value":36437437,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":397065,"Value":2850500,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":5752,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":176748,"Value":670289,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":95,"Executor Deserialize CPU Time":19382944,"Executor Run Time":3165,"Executor CPU Time":2829076042,"Peak Execution Memory":352357648,"Result Size":8188,"JVM GC Time":26,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5077427,"Total Records Read":397065},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":176748,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":331,"Index":40,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432690611,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"5138578","Value":"41576015","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"398664","Value":"3249164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"398664","Value":"3249164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"398664","Value":"3249164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"392344","Value":"3188241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"47","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"309","Value":"2380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"1207959552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"2498","Value":"19289","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"2612328","Value":"20959611","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"2589790","Value":"20789200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"275006","Value":"2233160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"3107","Value":"23249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":97,"Value":613,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":20871715,"Value":194791887,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":3171,"Value":23745,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":2938678400,"Value":21961839958,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":65510,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":26,"Value":448,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":2818861184,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5138578,"Value":41576015,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":398664,"Value":3249164,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":6575,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":106421,"Value":776710,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":97,"Executor Deserialize CPU Time":20871715,"Executor Run Time":3171,"Executor CPU Time":2938678400,"Peak Execution Memory":352357648,"Result Size":8188,"JVM GC Time":26,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5138578,"Total Records Read":398664},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":106421,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":292,"Index":5,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432690704,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"5506047","Value":"47082062","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"425591","Value":"3674755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"425591","Value":"3674755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"425591","Value":"3674755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"418065","Value":"3606306","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"56","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"293","Value":"2673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"1358954496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"2923","Value":"22212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"2784193","Value":"23743804","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"2761418","Value":"23550618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"292397","Value":"2525557","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"3242","Value":"26491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":70,"Value":683,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19277650,"Value":214069537,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":3297,"Value":27042,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":3007622287,"Value":24969462245,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":73699,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":32,"Value":480,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":3171218832,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5506047,"Value":47082062,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":425591,"Value":3674755,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":7397,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":63961,"Value":840671,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":70,"Executor Deserialize CPU Time":19277650,"Executor Run Time":3297,"Executor CPU Time":3007622287,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":32,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5506047,"Total Records Read":425591},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":63961,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":293,"Index":0,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432691165,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"7603473","Value":"54685535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"566756","Value":"4241511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"566756","Value":"4241511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"8","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"566756","Value":"4241511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"555159","Value":"4161465","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"8","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"421","Value":"3094","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"150994944","Value":"1509949440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"3388","Value":"25600","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"3939232","Value":"27683036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"3914157","Value":"27464775","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"388770","Value":"2914327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"3662","Value":"30153","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":95,"Value":778,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19148974,"Value":233218511,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":3728,"Value":30770,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":3297128193,"Value":28266590438,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":81887,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":57,"Value":537,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":3523576480,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":7603473,"Value":54685535,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":566756,"Value":4241511,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":8221,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":83358,"Value":924029,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":95,"Executor Deserialize CPU Time":19148974,"Executor Run Time":3728,"Executor CPU Time":3297128193,"Peak Execution Memory":352357648,"Result Size":8188,"JVM GC Time":57,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":7603473,"Total Records Read":566756},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":83358,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":315,"Index":31,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432691701,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"14179385","Value":"68864920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1069599","Value":"5311110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1069599","Value":"5311110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"4","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1069599","Value":"5311110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1050288","Value":"5211753","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"4","Value":"68","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"503","Value":"3597","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"1811939328","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"3806","Value":"29406","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7084874","Value":"34767910","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7053239","Value":"34518014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"734965","Value":"3649292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"4236","Value":"34389","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":849,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":31484732,"Value":264703243,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":4291,"Value":35061,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":3965472341,"Value":32232062779,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":90076,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":219,"Value":756,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":4026929072,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14179385,"Value":68864920,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1069599,"Value":5311110,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":9044,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":58534,"Value":982563,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":31484732,"Executor Run Time":4291,"Executor CPU Time":3965472341,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":219,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14179385,"Total Records Read":1069599},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":58534,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":314,"Index":36,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432691777,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"14286711","Value":"83151631","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1075303","Value":"6386413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1075303","Value":"6386413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1075303","Value":"6386413","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1050388","Value":"6262141","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"73","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"620","Value":"4217","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"2113929216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"3850","Value":"33256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7028804","Value":"41796714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"6997233","Value":"41515247","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"735414","Value":"4384706","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"4303","Value":"38692","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":73,"Value":922,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":25872668,"Value":290575911,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":4368,"Value":39429,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":4200715931,"Value":36432778710,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":98265,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":73,"Value":829,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":4530281664,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14286711,"Value":83151631,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1075303,"Value":6386413,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":9866,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":76142,"Value":1058705,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3237376776,"JVMOffHeapMemory":148667768,"OnHeapExecutionMemory":2348810240,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2624612879,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":97,"MinorGCTime":2337,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":73,"Executor Deserialize CPU Time":25872668,"Executor Run Time":4368,"Executor CPU Time":4200715931,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":73,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14286711,"Total Records Read":1075303},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":76142,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":318,"Index":37,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432692063,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"16812061","Value":"99963692","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1252080","Value":"7638493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1252080","Value":"7638493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1252080","Value":"7638493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1230187","Value":"7492328","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"78","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"689","Value":"4906","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"2415919104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"4094","Value":"37350","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7918049","Value":"49714763","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7886574","Value":"49401821","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"861213","Value":"5245919","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"4590","Value":"43282","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":72,"Value":994,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":28037050,"Value":318612961,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":4655,"Value":44084,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":4482387472,"Value":40915166182,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":106454,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":73,"Value":902,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":5033634256,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16812061,"Value":99963692,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1252080,"Value":7638493,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":826,"Value":10692,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":55264,"Value":1113969,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3237376776,"JVMOffHeapMemory":148667768,"OnHeapExecutionMemory":2348810240,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2624612879,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":97,"MinorGCTime":2337,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":72,"Executor Deserialize CPU Time":28037050,"Executor Run Time":4655,"Executor CPU Time":4482387472,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":73,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16812061,"Total Records Read":1252080},"Shuffle Write Metrics":{"Shuffle Bytes Written":826,"Shuffle Write Time":55264,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":319,"Index":32,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432692120,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"16283262","Value":"116246954","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1222489","Value":"8860982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1222489","Value":"8860982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"3","Value":"69","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1222489","Value":"8860982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1207733","Value":"8700061","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"4","Value":"82","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"566","Value":"5472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"2717908992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"4096","Value":"41446","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7750719","Value":"57465482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7719792","Value":"57121613","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"844923","Value":"6090842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"4656","Value":"47938","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":1063,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":24313949,"Value":342926910,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":4713,"Value":48797,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":4346473223,"Value":45261639405,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":114643,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":298,"Value":1200,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":5536986848,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16283262,"Value":116246954,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1222489,"Value":8860982,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":11516,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":62396,"Value":1176365,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":24313949,"Executor Run Time":4713,"Executor CPU Time":4346473223,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":298,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16283262,"Total Records Read":1222489},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":62396,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":295,"Index":11,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432692145,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"15063284","Value":"131310238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1163966","Value":"10024948","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1163966","Value":"10024948","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"5","Value":"74","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1163966","Value":"10024948","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1143328","Value":"9843389","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"3","Value":"85","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"510","Value":"5982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"3019898880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"4149","Value":"45595","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7298870","Value":"64764352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7269686","Value":"64391299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"800303","Value":"6891145","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"4681","Value":"52619","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":68,"Value":1131,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":23640106,"Value":366567016,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":4739,"Value":53536,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":4355489875,"Value":49617129280,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":122832,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":298,"Value":1498,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":6040339440,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":15063284,"Value":131310238,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1163966,"Value":10024948,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":821,"Value":12337,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":49605,"Value":1225970,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":68,"Executor Deserialize CPU Time":23640106,"Executor Run Time":4739,"Executor CPU Time":4355489875,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":298,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":15063284,"Total Records Read":1163966},"Shuffle Write Metrics":{"Shuffle Bytes Written":821,"Shuffle Write Time":49605,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":294,"Index":16,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432692277,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"15206714","Value":"146516952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1185888","Value":"11210836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1185888","Value":"11210836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"76","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1185888","Value":"11210836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1167901","Value":"11011290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"90","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"636","Value":"6618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"3321888768","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"4329","Value":"49924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7540656","Value":"72305008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7513451","Value":"71904750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"817587","Value":"7708732","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"4805","Value":"57424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":72,"Value":1203,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":21045729,"Value":387612745,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":4869,"Value":58405,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":4697480603,"Value":54314609883,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":131021,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":73,"Value":1571,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":6543692032,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":15206714,"Value":146516952,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1185888,"Value":11210836,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":13161,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":54508,"Value":1280478,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3237376776,"JVMOffHeapMemory":148667768,"OnHeapExecutionMemory":2348810240,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2624612879,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":97,"MinorGCTime":2337,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":72,"Executor Deserialize CPU Time":21045729,"Executor Run Time":4869,"Executor CPU Time":4697480603,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":73,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":15206714,"Total Records Read":1185888},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":54508,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":299,"Index":12,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432692504,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"17852704","Value":"164369656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1354716","Value":"12565552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1354716","Value":"12565552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"3","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1354716","Value":"12565552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1335669","Value":"12346959","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"4","Value":"94","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"640","Value":"7258","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"3623878656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"4521","Value":"54445","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"8042908","Value":"80347916","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"8012151","Value":"79916901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"934847","Value":"8643579","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"5040","Value":"62464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":68,"Value":1271,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":23335851,"Value":410948596,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":5098,"Value":63503,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":4691994552,"Value":59006604435,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":139210,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":298,"Value":1869,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":7047044624,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17852704,"Value":164369656,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1354716,"Value":12565552,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":13984,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":63580,"Value":1344058,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":68,"Executor Deserialize CPU Time":23335851,"Executor Run Time":5098,"Executor CPU Time":4691994552,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":298,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17852704,"Total Records Read":1354716},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":63580,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":333,"Index":41,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432692839,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"13740133","Value":"178109789","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1035151","Value":"13600703","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1035151","Value":"13600703","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"7","Value":"86","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1035151","Value":"13600703","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1016060","Value":"13363019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"103","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"614","Value":"7872","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"234881024","Value":"3858759680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"4875","Value":"59320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7046687","Value":"87394603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7013519","Value":"86930420","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"711445","Value":"9355024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"5335","Value":"67799","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":95,"Value":1366,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":23211677,"Value":434160273,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":5402,"Value":68905,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":4822896551,"Value":63829500986,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":147398,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":135,"Value":2004,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":436243728,"Value":7483288352,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":13740133,"Value":178109789,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1035151,"Value":13600703,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":820,"Value":14804,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":88769,"Value":1432827,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":95,"Executor Deserialize CPU Time":23211677,"Executor Run Time":5402,"Executor CPU Time":4822896551,"Peak Execution Memory":436243728,"Result Size":8188,"JVM GC Time":135,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":13740133,"Total Records Read":1035151},"Shuffle Write Metrics":{"Shuffle Bytes Written":820,"Shuffle Write Time":88769,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":326,"Index":39,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432692957,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"20255703","Value":"198365492","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1503082","Value":"15103785","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1503082","Value":"15103785","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"92","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1503082","Value":"15103785","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1476378","Value":"14839397","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"108","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"810","Value":"8682","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"4261412864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"4871","Value":"64191","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"9929413","Value":"97324016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"9892652","Value":"96823072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1033506","Value":"10388530","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"5484","Value":"73283","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":73,"Value":1439,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":21423754,"Value":455584027,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":5548,"Value":74453,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":5427467384,"Value":69256968370,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":155587,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":73,"Value":2077,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":8087304240,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20255703,"Value":198365492,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1503082,"Value":15103785,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":15626,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":53696,"Value":1486523,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3237376776,"JVMOffHeapMemory":148667768,"OnHeapExecutionMemory":2348810240,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2624612879,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":97,"MinorGCTime":2337,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":73,"Executor Deserialize CPU Time":21423754,"Executor Run Time":5548,"Executor CPU Time":5427467384,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":73,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20255703,"Total Records Read":1503082},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":53696,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":298,"Index":17,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432693090,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"17279386","Value":"215644878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1325325","Value":"16429110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1325325","Value":"16429110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"98","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1325325","Value":"16429110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1311802","Value":"16151199","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"113","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"652","Value":"9334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"4563402752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"5060","Value":"69251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"8008126","Value":"105332142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7980301","Value":"104803373","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"919517","Value":"11308047","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"5617","Value":"78900","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":72,"Value":1511,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":24937860,"Value":480521887,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":5682,"Value":80135,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":5504579519,"Value":74761547889,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":163776,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":73,"Value":2150,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":8590656832,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17279386,"Value":215644878,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1325325,"Value":16429110,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":819,"Value":16445,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":53393,"Value":1539916,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3237376776,"JVMOffHeapMemory":148667768,"OnHeapExecutionMemory":2348810240,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2624612879,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":97,"MinorGCTime":2337,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":72,"Executor Deserialize CPU Time":24937860,"Executor Run Time":5682,"Executor CPU Time":5504579519,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":73,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17279386,"Total Records Read":1325325},"Shuffle Write Metrics":{"Shuffle Bytes Written":819,"Shuffle Write Time":53393,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":327,"Index":34,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432693102,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"20325382","Value":"235970260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1507900","Value":"17937010","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1507900","Value":"17937010","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1507900","Value":"17937010","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1488908","Value":"17640107","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"4","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"767","Value":"10101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"4966055936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"5046","Value":"74297","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"10001165","Value":"115333307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"9964616","Value":"114767989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1042165","Value":"12350212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"5638","Value":"84538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":1580,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":30225683,"Value":510747570,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":5696,"Value":85831,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":5302393655,"Value":80063941544,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":171965,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":298,"Value":2448,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":9194672720,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20325382,"Value":235970260,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1507900,"Value":17937010,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":17269,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":54384,"Value":1594300,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":30225683,"Executor Run Time":5696,"Executor CPU Time":5302393655,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":298,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20325382,"Total Records Read":1507900},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":54384,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":322,"Index":38,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432693478,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"21789512","Value":"257759772","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1603400","Value":"19540410","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1603400","Value":"19540410","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"106","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1603400","Value":"19540410","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1564114","Value":"19204221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"122","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"874","Value":"10975","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"5368709120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"5328","Value":"79625","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"11511301","Value":"126844608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"11482858","Value":"126250847","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1096067","Value":"13446279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"6005","Value":"90543","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":72,"Value":1652,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":21551017,"Value":532298587,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":6070,"Value":91901,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":5852382342,"Value":85916323886,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":180154,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":105,"Value":2553,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":9798688608,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21789512,"Value":257759772,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1603400,"Value":19540410,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":18093,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":99626,"Value":1693926,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3237376776,"JVMOffHeapMemory":148667768,"OnHeapExecutionMemory":2348810240,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2624612879,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":97,"MinorGCTime":2337,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":72,"Executor Deserialize CPU Time":21551017,"Executor Run Time":6070,"Executor CPU Time":5852382342,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":105,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21789512,"Total Records Read":1603400},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":99626,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":316,"Index":26,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432693561,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"14350655","Value":"272110427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1106404","Value":"20646814","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1106404","Value":"20646814","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"8","Value":"114","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1106404","Value":"20646814","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1090201","Value":"20294422","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"2","Value":"124","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"705","Value":"11680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"5670699008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"5331","Value":"84956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7013679","Value":"133858287","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"6983027","Value":"133233874","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"762998","Value":"14209277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"6099","Value":"96642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":73,"Value":1725,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18983242,"Value":551281829,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":6151,"Value":98052,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":5772793648,"Value":91689117534,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":188343,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":45,"Value":2598,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":10302041200,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14350655,"Value":272110427,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1106404,"Value":20646814,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":18916,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":67778,"Value":1761704,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":73,"Executor Deserialize CPU Time":18983242,"Executor Run Time":6151,"Executor CPU Time":5772793648,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":45,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14350655,"Total Records Read":1106404},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":67778,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":307,"Index":14,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432693604,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"22521225","Value":"294631652","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1704079","Value":"22350893","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1704079","Value":"22350893","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1704079","Value":"22350893","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1676732","Value":"21971154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"3","Value":"127","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"742","Value":"12422","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"6073352192","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"5490","Value":"90446","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"10756664","Value":"144614951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"10719584","Value":"143953458","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1174256","Value":"15383533","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"6139","Value":"102781","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":68,"Value":1793,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":26338506,"Value":577620335,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":6198,"Value":104250,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":5806228005,"Value":97495345539,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":196532,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":298,"Value":2896,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":10906057088,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22521225,"Value":294631652,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1704079,"Value":22350893,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":19740,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":90570,"Value":1852274,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":68,"Executor Deserialize CPU Time":26338506,"Executor Run Time":6198,"Executor CPU Time":5806228005,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":298,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22521225,"Total Records Read":1704079},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":90570,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":306,"Index":19,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432693679,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"22027093","Value":"316658745","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1697367","Value":"24048260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1697367","Value":"24048260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1697367","Value":"24048260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1677876","Value":"23649030","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"876","Value":"13298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"6476005376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"5487","Value":"95933","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"10836844","Value":"155451795","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"10801236","Value":"154754694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1175348","Value":"16558881","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"6205","Value":"108986","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":73,"Value":1866,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":22682657,"Value":600302992,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":6270,"Value":110520,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":6065929732,"Value":103561275271,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":204721,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":105,"Value":3001,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":11510072976,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22027093,"Value":316658745,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1697367,"Value":24048260,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":20563,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":53349,"Value":1905623,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3237376776,"JVMOffHeapMemory":148667768,"OnHeapExecutionMemory":2348810240,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2624612879,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":97,"MinorGCTime":2337,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":73,"Executor Deserialize CPU Time":22682657,"Executor Run Time":6270,"Executor CPU Time":6065929732,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":105,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22027093,"Total Records Read":1697367},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":53349,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":317,"Index":21,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432694009,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"14140207","Value":"330798952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1091142","Value":"25139402","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1091142","Value":"25139402","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"7","Value":"125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1091142","Value":"25139402","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1068266","Value":"24717296","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"141","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"829","Value":"14127","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"6777995264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"5365","Value":"101298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"6841900","Value":"162293695","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"6814278","Value":"161568972","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"747499","Value":"17306380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"6502","Value":"115488","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":97,"Value":1963,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":29008161,"Value":629311153,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":6567,"Value":117087,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":5869480030,"Value":109430755301,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":212909,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":510,"Value":3511,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":12013425568,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14140207,"Value":330798952,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1091142,"Value":25139402,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":821,"Value":21384,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":89580,"Value":1995203,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":97,"Executor Deserialize CPU Time":29008161,"Executor Run Time":6567,"Executor CPU Time":5869480030,"Peak Execution Memory":503352592,"Result Size":8188,"JVM GC Time":510,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14140207,"Total Records Read":1091142},"Shuffle Write Metrics":{"Shuffle Bytes Written":821,"Shuffle Write Time":89580,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":323,"Index":33,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432694109,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"21849278","Value":"352648230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1616064","Value":"26755466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1616064","Value":"26755466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"127","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1616064","Value":"26755466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1590300","Value":"26307596","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"4","Value":"145","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"746","Value":"14873","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"7180648448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"6009","Value":"107307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"11739245","Value":"174032940","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"11710528","Value":"173279500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1113704","Value":"18420084","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"6645","Value":"122133","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":2032,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":23756398,"Value":653067551,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":6702,"Value":123789,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":6308433840,"Value":115739189141,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":221098,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":298,"Value":3809,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":12617441456,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21849278,"Value":352648230,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1616064,"Value":26755466,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":22206,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":51759,"Value":2046962,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":23756398,"Executor Run Time":6702,"Executor CPU Time":6308433840,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":298,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21849278,"Total Records Read":1616064},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":51759,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":302,"Index":18,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432694192,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"23325788","Value":"375974018","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1776905","Value":"28532371","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1776905","Value":"28532371","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"133","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1776905","Value":"28532371","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1751997","Value":"28059593","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"950","Value":"15823","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"469762048","Value":"7650410496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"6044","Value":"113351","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"11893096","Value":"185926036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"11865777","Value":"185145277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1226396","Value":"19646480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"6719","Value":"128852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":73,"Value":2105,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41762447,"Value":694829998,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":6784,"Value":130573,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":6578792091,"Value":122317981232,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":229287,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":105,"Value":3914,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":671124752,"Value":13288566208,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":23325788,"Value":375974018,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1776905,"Value":28532371,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":23030,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":49980,"Value":2096942,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3237376776,"JVMOffHeapMemory":148667768,"OnHeapExecutionMemory":2348810240,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2624612879,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":97,"MinorGCTime":2337,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":73,"Executor Deserialize CPU Time":41762447,"Executor Run Time":6784,"Executor CPU Time":6578792091,"Peak Execution Memory":671124752,"Result Size":8189,"JVM GC Time":105,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":23325788,"Total Records Read":1776905},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":49980,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":303,"Index":13,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432694380,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"23718187","Value":"399692205","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1791015","Value":"30323386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1791015","Value":"30323386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"139","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1791015","Value":"30323386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1758874","Value":"29818467","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"4","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"756","Value":"16579","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"469762048","Value":"8120172544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"6240","Value":"119591","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"11937551","Value":"197863587","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"11907003","Value":"197052280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1231326","Value":"20877806","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"6915","Value":"135767","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":2174,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":25418544,"Value":720248542,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":6973,"Value":137546,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":6577627933,"Value":128895609165,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":237476,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":298,"Value":4212,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":671124752,"Value":13959690960,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":23718187,"Value":399692205,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1791015,"Value":30323386,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":23852,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":55080,"Value":2152022,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":25418544,"Executor Run Time":6973,"Executor CPU Time":6577627933,"Peak Execution Memory":671124752,"Result Size":8189,"JVM GC Time":298,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":23718187,"Total Records Read":1791015},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":55080,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":321,"Index":22,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432694684,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"30","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"17648290","Value":"417340495","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1330648","Value":"31654034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1330648","Value":"31654034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"1","Value":"140","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1330648","Value":"31654034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1310953","Value":"31129420","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"163","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"1078","Value":"17657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"8422162432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"6621","Value":"126212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"8269956","Value":"206133543","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"8241301","Value":"205293581","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"917559","Value":"21795365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"7180","Value":"142947","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":99,"Value":2273,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":23217969,"Value":743466511,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":7241,"Value":144787,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":6515667855,"Value":135411277020,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":245664,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":510,"Value":4722,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":14463043552,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17648290,"Value":417340495,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1330648,"Value":31654034,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":24675,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":75311,"Value":2227333,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":99,"Executor Deserialize CPU Time":23217969,"Executor Run Time":7241,"Executor CPU Time":6515667855,"Peak Execution Memory":503352592,"Result Size":8188,"JVM GC Time":510,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17648290,"Total Records Read":1330648},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":75311,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":335,"Index":42,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432694782,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"31","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"16633555","Value":"433974050","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1252125","Value":"32906159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1252125","Value":"32906159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"8","Value":"148","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1252125","Value":"32906159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1237479","Value":"32366899","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"610","Value":"18267","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"8724152320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"6707","Value":"132919","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7978094","Value":"214111637","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7948557","Value":"213242138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"866570","Value":"22661935","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"7278","Value":"150225","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":83,"Value":2356,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18237725,"Value":761704236,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":7355,"Value":152142,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":6760521039,"Value":142171798059,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":253852,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":510,"Value":5232,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":14966396144,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":31,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16633555,"Value":433974050,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1252125,"Value":32906159,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":25499,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":31,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":79254,"Value":2306587,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":83,"Executor Deserialize CPU Time":18237725,"Executor Run Time":7355,"Executor CPU Time":6760521039,"Peak Execution Memory":503352592,"Result Size":8188,"JVM GC Time":510,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16633555,"Total Records Read":1252125},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":79254,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":297,"Index":1,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432695053,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"18420350","Value":"452394400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1358796","Value":"34264955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1358796","Value":"34264955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"7","Value":"155","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1358796","Value":"34264955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1331797","Value":"33698696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"181","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"1421","Value":"19688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"9026142208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"6760","Value":"139679","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"9392292","Value":"223503929","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"9357092","Value":"222599230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"932727","Value":"23594662","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"7548","Value":"157773","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":94,"Value":2450,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":21173577,"Value":782877813,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":7612,"Value":159754,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":6881341976,"Value":149053140035,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":262040,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":510,"Value":5742,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":15469748736,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":18420350,"Value":452394400,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1358796,"Value":34264955,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":26322,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":85400,"Value":2391987,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":94,"Executor Deserialize CPU Time":21173577,"Executor Run Time":7612,"Executor CPU Time":6881341976,"Peak Execution Memory":503352592,"Result Size":8188,"JVM GC Time":510,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":18420350,"Total Records Read":1358796},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":85400,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":337,"Index":43,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432695473,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"19771704","Value":"472166104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1468326","Value":"35733281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1468326","Value":"35733281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"7","Value":"162","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1468326","Value":"35733281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1447733","Value":"35146429","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"190","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"724","Value":"20412","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"9428795392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"7264","Value":"146943","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"10577490","Value":"234081419","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"10549548","Value":"233148778","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1013026","Value":"24607688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"7969","Value":"165742","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":95,"Value":2545,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":20438428,"Value":803316241,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":8034,"Value":167788,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":7261521587,"Value":156314661622,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":270228,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":588,"Value":6330,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":16073764624,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":19771704,"Value":472166104,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1468326,"Value":35733281,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":27145,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":73750,"Value":2465737,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":95,"Executor Deserialize CPU Time":20438428,"Executor Run Time":8034,"Executor CPU Time":7261521587,"Peak Execution Memory":604015888,"Result Size":8188,"JVM GC Time":588,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":19771704,"Total Records Read":1468326},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":73750,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":339,"Index":44,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432695786,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"20467942","Value":"492634046","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1520355","Value":"37253636","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1520355","Value":"37253636","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1520355","Value":"37253636","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1500662","Value":"36647091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"199","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"753","Value":"21165","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"9831448576","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"7567","Value":"154510","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"10159321","Value":"244240740","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"10122495","Value":"243271273","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1049813","Value":"25657501","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"8282","Value":"174024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":98,"Value":2643,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":24394597,"Value":827710838,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":8345,"Value":176133,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":7587522147,"Value":163902183769,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":278416,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":588,"Value":6918,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":16677780512,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20467942,"Value":492634046,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1520355,"Value":37253636,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":27967,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":69969,"Value":2535706,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":98,"Executor Deserialize CPU Time":24394597,"Executor Run Time":8345,"Executor CPU Time":7587522147,"Peak Execution Memory":604015888,"Result Size":8188,"JVM GC Time":588,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20467942,"Total Records Read":1520355},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":69969,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":301,"Index":2,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432695890,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"21741695","Value":"514375741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1592591","Value":"38846227","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1592591","Value":"38846227","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"7","Value":"175","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1592591","Value":"38846227","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1570179","Value":"38217270","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"1153","Value":"22318","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"10234101760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"7717","Value":"162227","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"10699132","Value":"254939872","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"10663812","Value":"253935085","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1098273","Value":"26755774","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"8384","Value":"182408","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":97,"Value":2740,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":22121570,"Value":849832408,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":8448,"Value":184581,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":7527924878,"Value":171430108647,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":286604,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":588,"Value":7506,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":17281796400,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21741695,"Value":514375741,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1592591,"Value":38846227,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":28791,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":78839,"Value":2614545,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":97,"Executor Deserialize CPU Time":22121570,"Executor Run Time":8448,"Executor CPU Time":7527924878,"Peak Execution Memory":604015888,"Result Size":8188,"JVM GC Time":588,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21741695,"Total Records Read":1592591},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":78839,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":329,"Index":24,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432695950,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"20356289","Value":"534732030","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1539649","Value":"40385876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1539649","Value":"40385876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"7","Value":"182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1539649","Value":"40385876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1515507","Value":"39732777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"8","Value":"216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"827","Value":"23145","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"10636754944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"7798","Value":"170025","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"9818948","Value":"264758820","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"9784858","Value":"263719943","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1061026","Value":"27816800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"8448","Value":"190856","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":81,"Value":2821,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":34328167,"Value":884160575,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":8525,"Value":193106,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":7766058668,"Value":179196167315,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":294792,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":588,"Value":8094,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":17885812288,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20356289,"Value":534732030,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1539649,"Value":40385876,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":29615,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":70448,"Value":2684993,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":81,"Executor Deserialize CPU Time":34328167,"Executor Run Time":8525,"Executor CPU Time":7766058668,"Peak Execution Memory":604015888,"Result Size":8188,"JVM GC Time":588,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20356289,"Total Records Read":1539649},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":70448,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":309,"Index":4,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432696644,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"26091211","Value":"560823241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1890520","Value":"42276396","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1890520","Value":"42276396","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1890520","Value":"42276396","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1860972","Value":"41593749","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"225","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"1076","Value":"24221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"469762048","Value":"11106516992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"8328","Value":"178353","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"13289107","Value":"278047927","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"13248488","Value":"276968431","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1302426","Value":"29119226","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"9141","Value":"199997","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":95,"Value":2916,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":38005152,"Value":922165727,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":9204,"Value":202310,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":8452912787,"Value":187649080102,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":302980,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":588,"Value":8682,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":671124752,"Value":18556937040,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":26091211,"Value":560823241,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1890520,"Value":42276396,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":825,"Value":30440,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":68439,"Value":2753432,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":95,"Executor Deserialize CPU Time":38005152,"Executor Run Time":9204,"Executor CPU Time":8452912787,"Peak Execution Memory":671124752,"Result Size":8188,"JVM GC Time":588,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":26091211,"Total Records Read":1890520},"Shuffle Write Metrics":{"Shuffle Bytes Written":825,"Shuffle Write Time":68439,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":325,"Index":23,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432696845,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"22682377","Value":"583505618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1708146","Value":"43984542","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1708146","Value":"43984542","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"7","Value":"191","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1708146","Value":"43984542","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1671692","Value":"43265441","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"234","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"797","Value":"25018","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"11509170176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"8600","Value":"186953","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"11677434","Value":"289725361","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"11650243","Value":"288618674","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1170891","Value":"30290117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"9343","Value":"209340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":97,"Value":3013,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":24091305,"Value":946257032,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":9406,"Value":211716,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":8695214598,"Value":196344294700,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":311168,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":588,"Value":9270,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":19160952928,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22682377,"Value":583505618,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1708146,"Value":43984542,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":820,"Value":31260,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":54671,"Value":2808103,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":97,"Executor Deserialize CPU Time":24091305,"Executor Run Time":9406,"Executor CPU Time":8695214598,"Peak Execution Memory":604015888,"Result Size":8188,"JVM GC Time":588,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22682377,"Total Records Read":1708146},"Shuffle Write Metrics":{"Shuffle Bytes Written":820,"Shuffle Write Time":54671,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":305,"Index":3,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432697026,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"24749426","Value":"608255044","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1804783","Value":"45789325","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1804783","Value":"45789325","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"7","Value":"198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1804783","Value":"45789325","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1778105","Value":"45043546","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"926","Value":"25944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"469762048","Value":"11978932224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"8733","Value":"195686","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"13318435","Value":"303043796","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"13286263","Value":"301904937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1244990","Value":"31535107","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"9523","Value":"218863","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":92,"Value":3105,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":23333266,"Value":969590298,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":9588,"Value":221304,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":8824056782,"Value":205168351482,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8188,"Value":319356,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":588,"Value":9858,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":671124752,"Value":19832077680,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":24749426,"Value":608255044,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1804783,"Value":45789325,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":32083,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":84712,"Value":2892815,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":7346570720,"JVMOffHeapMemory":145177792,"OnHeapExecutionMemory":3523215360,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275802639,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":3799017999,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1551560,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":89,"MinorGCTime":1613,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":92,"Executor Deserialize CPU Time":23333266,"Executor Run Time":9588,"Executor CPU Time":8824056782,"Peak Execution Memory":671124752,"Result Size":8188,"JVM GC Time":588,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":24749426,"Total Records Read":1804783},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":84712,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":332,"Index":46,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432698396,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"13508300","Value":"621763344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1023135","Value":"46812460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1023135","Value":"46812460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"8","Value":"206","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1023135","Value":"46812460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1009708","Value":"46053254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"2","Value":"245","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"673","Value":"26617","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"234881024","Value":"12213813248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"10219","Value":"205905","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"6850760","Value":"309894556","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"6817396","Value":"308722333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"707425","Value":"32242532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"10934","Value":"229797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":3176,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":22617947,"Value":992208245,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":10988,"Value":232292,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":10766166711,"Value":215934518193,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":327545,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":53,"Value":9911,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":436243728,"Value":20268321408,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":13508300,"Value":621763344,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1023135,"Value":46812460,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":821,"Value":32904,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":69047,"Value":2961862,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":22617947,"Executor Run Time":10988,"Executor CPU Time":10766166711,"Peak Execution Memory":436243728,"Result Size":8189,"JVM GC Time":53,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":13508300,"Total Records Read":1023135},"Shuffle Write Metrics":{"Shuffle Bytes Written":821,"Shuffle Write Time":69047,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":300,"Index":7,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432698614,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"41","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"18182549","Value":"639945893","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1355043","Value":"48167503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1355043","Value":"48167503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"6","Value":"212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1355043","Value":"48167503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1338255","Value":"47391509","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"12","Value":"257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"594","Value":"27211","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"12515803136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"10570","Value":"216475","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"8599656","Value":"318494212","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"8567580","Value":"317289913","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"937919","Value":"33180451","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"11152","Value":"240949","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":73,"Value":3249,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":25357099,"Value":1017565344,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":11204,"Value":243496,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":10902701208,"Value":226837219401,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":335734,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":53,"Value":9964,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":20771674000,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":18182549,"Value":639945893,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1355043,"Value":48167503,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":33726,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":57456,"Value":3019318,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":73,"Executor Deserialize CPU Time":25357099,"Executor Run Time":11204,"Executor CPU Time":10902701208,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":53,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":18182549,"Total Records Read":1355043},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":57456,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":334,"Index":47,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432698773,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"16286726","Value":"656232619","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1228580","Value":"49396083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1228580","Value":"49396083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"7","Value":"219","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1228580","Value":"49396083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1215094","Value":"48606603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"3","Value":"260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"529","Value":"27740","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"12817793024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"10774","Value":"227249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7704937","Value":"326199149","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7674636","Value":"324964549","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"851410","Value":"34031861","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"11310","Value":"252259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":70,"Value":3319,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":31457505,"Value":1049022849,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":11365,"Value":254861,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":11165335932,"Value":238002555333,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":343923,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":53,"Value":10017,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":21275026592,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16286726,"Value":656232619,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1228580,"Value":49396083,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":821,"Value":34547,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":49731,"Value":3069049,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":70,"Executor Deserialize CPU Time":31457505,"Executor Run Time":11365,"Executor CPU Time":11165335932,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":53,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16286726,"Total Records Read":1228580},"Shuffle Write Metrics":{"Shuffle Bytes Written":821,"Shuffle Write Time":49731,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":296,"Index":6,"Attempt":0,"Launch Time":1646432687316,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432699349,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"43","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"15259337","Value":"671491956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1154264","Value":"50550347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1154264","Value":"50550347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"0","Value":"219","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1154264","Value":"50550347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1136753","Value":"49743356","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"687","Value":"28427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"13119782912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"11378","Value":"238627","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"7600974","Value":"333800123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"7568547","Value":"332533096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"795276","Value":"34827137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"11886","Value":"264145","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":3390,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":18891216,"Value":1067914065,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":11940,"Value":266801,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":11700016445,"Value":249702571778,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":352112,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":10079,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":21778379184,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":15259337,"Value":671491956,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1154264,"Value":50550347,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":35371,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":58626,"Value":3127675,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":18891216,"Executor Run Time":11940,"Executor CPU Time":11700016445,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":15259337,"Total Records Read":1154264},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":58626,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":320,"Index":27,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432700178,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"17698934","Value":"689190890","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1333919","Value":"51884266","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1333919","Value":"51884266","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"2","Value":"221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1333919","Value":"51884266","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1319111","Value":"51062467","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"7","Value":"272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"582","Value":"29009","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"301989888","Value":"13421772800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"12149","Value":"250776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"8355380","Value":"342155503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"8324284","Value":"340857380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"924194","Value":"35751331","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"12716","Value":"276861","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":3461,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":17879429,"Value":1085793494,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":12769,"Value":279570,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":12510472911,"Value":262213044689,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":360301,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":10141,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":22281731776,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17698934,"Value":689190890,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1333919,"Value":51884266,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":36193,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":54779,"Value":3182454,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":17879429,"Executor Run Time":12769,"Executor CPU Time":12510472911,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17698934,"Total Records Read":1333919},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":54779,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":328,"Index":29,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432701080,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"20388677","Value":"709579567","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1540279","Value":"53424545","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1540279","Value":"53424545","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"8","Value":"229","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1540279","Value":"53424545","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1524417","Value":"52586884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"2","Value":"274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"679","Value":"29688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"13824425984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"12968","Value":"263744","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"10008625","Value":"352164128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"9971320","Value":"350828700","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1067095","Value":"36818426","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"13616","Value":"290477","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":3532,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":19152600,"Value":1104946094,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":13670,"Value":293240,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":13463815994,"Value":275676860683,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":368490,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":10203,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":22885747664,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20388677,"Value":709579567,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1540279,"Value":53424545,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":820,"Value":37013,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":54247,"Value":3236701,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":19152600,"Executor Run Time":13670,"Executor CPU Time":13463815994,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20388677,"Total Records Read":1540279},"Shuffle Write Metrics":{"Shuffle Bytes Written":820,"Shuffle Write Time":54247,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":308,"Index":9,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432701456,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"46","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"22864810","Value":"732444377","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1697113","Value":"55121658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1697113","Value":"55121658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"1","Value":"230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1697113","Value":"55121658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1676385","Value":"54263269","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"5","Value":"279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"786","Value":"30474","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"14227079168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"13279","Value":"277023","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"11418581","Value":"363582709","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"11380207","Value":"362208907","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1172062","Value":"37990488","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"13995","Value":"304472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":70,"Value":3602,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":20018525,"Value":1124964619,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":14050,"Value":307290,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":13706731155,"Value":289383591838,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":376679,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":10265,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":23489763552,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22864810,"Value":732444377,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1697113,"Value":55121658,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":37835,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":72247,"Value":3308948,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":70,"Executor Deserialize CPU Time":20018525,"Executor Run Time":14050,"Executor CPU Time":13706731155,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22864810,"Total Records Read":1697113},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":72247,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":336,"Index":48,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432701882,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"47","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"19877448","Value":"752321825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1480083","Value":"56601741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1480083","Value":"56601741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"0","Value":"230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1480083","Value":"56601741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1464397","Value":"55727666","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"673","Value":"31147","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"14629732352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"13763","Value":"290786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"10574238","Value":"374156947","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"10546700","Value":"372755607","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1024881","Value":"39015369","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"14421","Value":"318893","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":70,"Value":3672,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":27219141,"Value":1152183760,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":14476,"Value":321766,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":14210368240,"Value":303593960078,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":384868,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":10327,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":24093779440,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":19877448,"Value":752321825,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1480083,"Value":56601741,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":38658,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":56504,"Value":3365452,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":70,"Executor Deserialize CPU Time":27219141,"Executor Run Time":14476,"Executor CPU Time":14210368240,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":19877448,"Total Records Read":1480083},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":56504,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":338,"Index":49,"Attempt":0,"Launch Time":1646432687319,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432702157,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"20138112","Value":"772459937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1498633","Value":"58100374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1498633","Value":"58100374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"0","Value":"230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1498633","Value":"58100374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1483487","Value":"57211153","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"2","Value":"290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"679","Value":"31826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"15032385536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"14049","Value":"304835","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"9849078","Value":"384006025","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"9812593","Value":"382568200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1038288","Value":"40053657","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"14696","Value":"333589","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":3743,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":20537516,"Value":1172721276,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":14750,"Value":336516,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":14517213437,"Value":318111173515,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":393057,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":10389,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":24697795328,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20138112,"Value":772459937,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1498633,"Value":58100374,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":39480,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":49350,"Value":3414802,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":20537516,"Executor Run Time":14750,"Executor CPU Time":14517213437,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20138112,"Total Records Read":1498633},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":49350,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":304,"Index":8,"Attempt":0,"Launch Time":1646432687317,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432702190,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"49","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"23437384","Value":"795897321","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1737471","Value":"59837845","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1737471","Value":"59837845","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"8","Value":"238","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1737471","Value":"59837845","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1708343","Value":"58919496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"2","Value":"292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"816","Value":"32642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"15435038720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"13991","Value":"318826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"12578897","Value":"396584922","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"12549967","Value":"395118167","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1195580","Value":"41249237","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"14729","Value":"348318","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":3814,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":22419177,"Value":1195140453,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":14783,"Value":351299,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":14540679383,"Value":332651852898,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":401246,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":10451,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":25301811216,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":23437384,"Value":795897321,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1737471,"Value":59837845,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":40304,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":48312,"Value":3463114,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":22419177,"Executor Run Time":14783,"Executor CPU Time":14540679383,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":23437384,"Total Records Read":1737471},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":48312,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":324,"Index":28,"Attempt":0,"Launch Time":1646432687318,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432702271,"Failed":false,"Killed":false,"Accumulables":[{"ID":2127,"Name":"local blocks read","Update":"1","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Update":"22891115","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Update":"1719119","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Update":"1719119","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Update":"1","Value":"239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Update":"1719119","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Update":"1696792","Value":"60616288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Update":"9","Value":"301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Update":"784","Value":"33426","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Update":"402653184","Value":"15837691904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Update":"14078","Value":"332904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Update":"12201936","Value":"408786858","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Update":"12171849","Value":"407290016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Update":"1188115","Value":"42437352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Update":"14810","Value":"363128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":3885,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Update":21254357,"Value":1216394810,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Update":14863,"Value":366162,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Update":14483129976,"Value":347134982874,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Update":8189,"Value":409435,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Update":62,"Value":10513,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":25905827104,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22891115,"Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1719119,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":41127,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Update":47539,"Value":3510653,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14756925208,"JVMOffHeapMemory":145281424,"OnHeapExecutionMemory":2650800128,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":275879002,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2926679130,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628367,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":93,"MinorGCTime":1458,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":21254357,"Executor Run Time":14863,"Executor CPU Time":14483129976,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":62,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22891115,"Total Records Read":1719119},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":47539,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":76,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":50,"RDD Info":[{"RDD ID":177,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[176],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":173,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"282\",\"name\":\"mapPartitions\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[172],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":176,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[175],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":175,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"284\",\"name\":\"map\"}","Callsite":"map at RegressionMetrics.scala:63","Parent IDs":[174],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":174,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"283\",\"name\":\"map\"}","Callsite":"map at LinearRegression.scala:914","Parent IDs":[173],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":172,"Name":"SQLExecutionRDD","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[171],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":171,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8650\",\"name\":\"DeserializeToObject\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[170],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":169,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_8623\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[145],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":170,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8649\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[169],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[75],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)","Submission Time":1646432687308,"Completion Time":1646432702271,"Accumulables":[{"ID":2127,"Name":"local blocks read","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2130,"Name":"local bytes read","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2131,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2132,"Name":"records read","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2474,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2475,"Name":"time to read broadcast","Value":"239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2476,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2477,"Name":"time to read broadcast","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2478,"Name":"number of output rows","Value":"60616288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2479,"Name":"time to read broadcast","Value":"301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2480,"Name":"sort time","Value":"33426","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2481,"Name":"peak memory","Value":"15837691904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2482,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2483,"Name":"cumulative time","Value":"332904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2484,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2485,"Name":"num prefix comparisons","Value":"408786858","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2486,"Name":"num record comparisons","Value":"407290016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"number of output rows","Value":"42437352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2488,"Name":"duration","Value":"363128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2490,"Name":"internal.metrics.executorDeserializeTime","Value":3885,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.executorDeserializeCpuTime","Value":1216394810,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.executorRunTime","Value":366162,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.executorCpuTime","Value":347134982874,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.resultSize","Value":409435,"Internal":true,"Count Failed Values":true},{"ID":2495,"Name":"internal.metrics.jvmGCTime","Value":10513,"Internal":true,"Count Failed Values":true},{"ID":2497,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.peakExecutionMemory","Value":25905827104,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":50,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.read.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":41127,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":50,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.write.writeTime","Value":3510653,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":77,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":6,"RDD Info":[{"RDD ID":179,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[178],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":178,"Name":"ShuffledRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[177],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[76],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)","Submission Time":1646432702272,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":77,"Stage Attempt ID":0,"Task Info":{"Task ID":340,"Index":0,"Attempt":0,"Launch Time":1646432702279,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":77,"Stage Attempt ID":0,"Task Info":{"Task ID":341,"Index":1,"Attempt":0,"Launch Time":1646432702279,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":77,"Stage Attempt ID":0,"Task Info":{"Task ID":342,"Index":2,"Attempt":0,"Launch Time":1646432702279,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":77,"Stage Attempt ID":0,"Task Info":{"Task ID":343,"Index":3,"Attempt":0,"Launch Time":1646432702279,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":77,"Stage Attempt ID":0,"Task Info":{"Task ID":344,"Index":4,"Attempt":0,"Launch Time":1646432702280,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":77,"Stage Attempt ID":0,"Task Info":{"Task ID":345,"Index":5,"Attempt":0,"Launch Time":1646432702280,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":77,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":342,"Index":2,"Attempt":0,"Launch Time":1646432702279,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432702349,"Failed":false,"Killed":false,"Accumulables":[{"ID":2520,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2521,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6259435,"Value":6259435,"Internal":true,"Count Failed Values":true},{"ID":2522,"Name":"internal.metrics.executorRunTime","Update":9,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2523,"Name":"internal.metrics.executorCpuTime","Update":5673462,"Value":5673462,"Internal":true,"Count Failed Values":true},{"ID":2524,"Name":"internal.metrics.resultSize","Update":9231,"Value":9231,"Internal":true,"Count Failed Values":true},{"ID":2527,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2528,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2529,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":2352,"Internal":true,"Count Failed Values":true},{"ID":2533,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2534,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2535,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":4941,"Value":4941,"Internal":true,"Count Failed Values":true},{"ID":2536,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2537,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1647,"Value":1647,"Internal":true,"Count Failed Values":true},{"ID":2538,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2539,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":8,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":6259435,"Executor Run Time":9,"Executor CPU Time":5673462,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":6,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":4941,"Remote Bytes Read To Disk":0,"Local Bytes Read":1647,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":77,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":340,"Index":0,"Attempt":0,"Launch Time":1646432702279,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432702351,"Failed":false,"Killed":false,"Accumulables":[{"ID":2520,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2521,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6947547,"Value":13206982,"Internal":true,"Count Failed Values":true},{"ID":2522,"Name":"internal.metrics.executorRunTime","Update":9,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2523,"Name":"internal.metrics.executorCpuTime","Update":6443540,"Value":12117002,"Internal":true,"Count Failed Values":true},{"ID":2524,"Name":"internal.metrics.resultSize","Update":9231,"Value":18462,"Internal":true,"Count Failed Values":true},{"ID":2527,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2528,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2529,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":4704,"Internal":true,"Count Failed Values":true},{"ID":2533,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2534,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2535,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":5765,"Value":10706,"Internal":true,"Count Failed Values":true},{"ID":2536,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2537,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1645,"Value":3292,"Internal":true,"Count Failed Values":true},{"ID":2538,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2539,"Name":"internal.metrics.shuffle.read.recordsRead","Update":9,"Value":17,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":6947547,"Executor Run Time":9,"Executor CPU Time":6443540,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":5765,"Remote Bytes Read To Disk":0,"Local Bytes Read":1645,"Total Records Read":9},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":77,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":344,"Index":4,"Attempt":0,"Launch Time":1646432702280,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432702351,"Failed":false,"Killed":false,"Accumulables":[{"ID":2520,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2521,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7001124,"Value":20208106,"Internal":true,"Count Failed Values":true},{"ID":2522,"Name":"internal.metrics.executorRunTime","Update":9,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2523,"Name":"internal.metrics.executorCpuTime","Update":5206923,"Value":17323925,"Internal":true,"Count Failed Values":true},{"ID":2524,"Name":"internal.metrics.resultSize","Update":9231,"Value":27693,"Internal":true,"Count Failed Values":true},{"ID":2527,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2528,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2529,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":7056,"Internal":true,"Count Failed Values":true},{"ID":2533,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":6,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2534,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2535,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":4939,"Value":15645,"Internal":true,"Count Failed Values":true},{"ID":2536,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2537,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1646,"Value":4938,"Internal":true,"Count Failed Values":true},{"ID":2538,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2539,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":25,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":7001124,"Executor Run Time":9,"Executor CPU Time":5206923,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":6,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":4939,"Remote Bytes Read To Disk":0,"Local Bytes Read":1646,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":77,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":341,"Index":1,"Attempt":0,"Launch Time":1646432702279,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432702351,"Failed":false,"Killed":false,"Accumulables":[{"ID":2520,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2521,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6776618,"Value":26984724,"Internal":true,"Count Failed Values":true},{"ID":2522,"Name":"internal.metrics.executorRunTime","Update":10,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":2523,"Name":"internal.metrics.executorCpuTime","Update":5551700,"Value":22875625,"Internal":true,"Count Failed Values":true},{"ID":2524,"Name":"internal.metrics.resultSize","Update":9231,"Value":36924,"Internal":true,"Count Failed Values":true},{"ID":2527,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2528,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2529,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":9408,"Internal":true,"Count Failed Values":true},{"ID":2533,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":2534,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2535,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":5758,"Value":21403,"Internal":true,"Count Failed Values":true},{"ID":2536,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2537,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1649,"Value":6587,"Internal":true,"Count Failed Values":true},{"ID":2538,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2539,"Name":"internal.metrics.shuffle.read.recordsRead","Update":9,"Value":34,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":6776618,"Executor Run Time":10,"Executor CPU Time":5551700,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":2,"Fetch Wait Time":2,"Remote Bytes Read":5758,"Remote Bytes Read To Disk":0,"Local Bytes Read":1649,"Total Records Read":9},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":77,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":345,"Index":5,"Attempt":0,"Launch Time":1646432702280,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432702353,"Failed":false,"Killed":false,"Accumulables":[{"ID":2520,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2521,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7655570,"Value":34640294,"Internal":true,"Count Failed Values":true},{"ID":2522,"Name":"internal.metrics.executorRunTime","Update":8,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2523,"Name":"internal.metrics.executorCpuTime","Update":5199979,"Value":28075604,"Internal":true,"Count Failed Values":true},{"ID":2524,"Name":"internal.metrics.resultSize","Update":9231,"Value":46155,"Internal":true,"Count Failed Values":true},{"ID":2527,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2528,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2529,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":11760,"Internal":true,"Count Failed Values":true},{"ID":2533,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":6,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2534,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2535,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":4924,"Value":26327,"Internal":true,"Count Failed Values":true},{"ID":2536,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2537,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1638,"Value":8225,"Internal":true,"Count Failed Values":true},{"ID":2538,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2539,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":42,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":7655570,"Executor Run Time":8,"Executor CPU Time":5199979,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":6,"Local Blocks Fetched":2,"Fetch Wait Time":2,"Remote Bytes Read":4924,"Remote Bytes Read To Disk":0,"Local Bytes Read":1638,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":77,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":343,"Index":3,"Attempt":0,"Launch Time":1646432702279,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432702354,"Failed":false,"Killed":false,"Accumulables":[{"ID":2520,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2521,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8075156,"Value":42715450,"Internal":true,"Count Failed Values":true},{"ID":2522,"Name":"internal.metrics.executorRunTime","Update":9,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":2523,"Name":"internal.metrics.executorCpuTime","Update":7392986,"Value":35468590,"Internal":true,"Count Failed Values":true},{"ID":2524,"Name":"internal.metrics.resultSize","Update":9231,"Value":55386,"Internal":true,"Count Failed Values":true},{"ID":2527,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2528,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2529,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":14112,"Internal":true,"Count Failed Values":true},{"ID":2533,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":6,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":2534,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2535,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":4931,"Value":31258,"Internal":true,"Count Failed Values":true},{"ID":2536,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2537,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1644,"Value":9869,"Internal":true,"Count Failed Values":true},{"ID":2538,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2539,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":50,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":8075156,"Executor Run Time":9,"Executor CPU Time":7392986,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":6,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":4931,"Remote Bytes Read To Disk":0,"Local Bytes Read":1644,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":77,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":6,"RDD Info":[{"RDD ID":179,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[178],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":178,"Name":"ShuffledRDD","Scope":"{\"id\":\"285\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[177],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[76],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionTrainingSummary.(LinearRegression.scala:868)\norg.apache.spark.ml.regression.LinearRegression.trainWithNormal(LinearRegression.scala:460)\norg.apache.spark.ml.regression.LinearRegression.$anonfun$train$1(LinearRegression.scala:347)\norg.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)\nscala.util.Try$.apply(Try.scala:213)\norg.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:329)\norg.apache.spark.ml.regression.LinearRegression.train(LinearRegression.scala:186)\norg.apache.spark.ml.Predictor.fit(Predictor.scala:151)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)","Submission Time":1646432702272,"Completion Time":1646432702355,"Accumulables":[{"ID":2520,"Name":"internal.metrics.executorDeserializeTime","Value":40,"Internal":true,"Count Failed Values":true},{"ID":2521,"Name":"internal.metrics.executorDeserializeCpuTime","Value":42715450,"Internal":true,"Count Failed Values":true},{"ID":2522,"Name":"internal.metrics.executorRunTime","Value":54,"Internal":true,"Count Failed Values":true},{"ID":2523,"Name":"internal.metrics.executorCpuTime","Value":35468590,"Internal":true,"Count Failed Values":true},{"ID":2524,"Name":"internal.metrics.resultSize","Value":55386,"Internal":true,"Count Failed Values":true},{"ID":2527,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2528,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2529,"Name":"internal.metrics.peakExecutionMemory","Value":14112,"Internal":true,"Count Failed Values":true},{"ID":2533,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":38,"Internal":true,"Count Failed Values":true},{"ID":2534,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":12,"Internal":true,"Count Failed Values":true},{"ID":2535,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":31258,"Internal":true,"Count Failed Values":true},{"ID":2536,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2537,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":9869,"Internal":true,"Count Failed Values":true},{"ID":2538,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":2539,"Name":"internal.metrics.shuffle.read.recordsRead","Value":50,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":41,"Completion Time":1646432702355,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":42,"Submission Time":1646432702607,"Stage Infos":[{"Stage ID":78,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":183,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9208\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[182],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":182,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9207\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[181],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":181,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8746\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[180],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":180,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8746\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[78],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_9208\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":78,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":183,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9208\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[182],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":182,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9207\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[181],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":181,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8746\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[180],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":180,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8746\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432702610,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_9208\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":346,"Index":0,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":347,"Index":1,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":348,"Index":2,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":349,"Index":3,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":350,"Index":4,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":351,"Index":5,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":352,"Index":6,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":353,"Index":7,"Attempt":0,"Launch Time":1646432702616,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":354,"Index":8,"Attempt":0,"Launch Time":1646432702616,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":78,"Stage Attempt ID":0,"Task Info":{"Task ID":355,"Index":9,"Attempt":0,"Launch Time":1646432702616,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerJobStart","Job ID":43,"Submission Time":1646432702668,"Stage Infos":[{"Stage ID":79,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":192,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9228\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[191],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":189,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9223\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[188],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":184,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":185,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[184],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":186,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9220\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[185],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":190,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_9224\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[186,189],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":188,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[187],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":187,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":191,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9227\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[190],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[79],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_9228\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":79,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":192,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9228\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[191],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":189,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9223\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[188],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":184,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":185,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[184],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":186,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9220\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[185],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":190,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_9224\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[186,189],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":188,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[187],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":187,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":191,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9227\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[190],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432702670,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_9228\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":356,"Index":0,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":357,"Index":1,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":358,"Index":2,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":359,"Index":3,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":360,"Index":4,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":361,"Index":5,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":362,"Index":6,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":363,"Index":7,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":364,"Index":8,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":365,"Index":9,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":366,"Index":10,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":367,"Index":11,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":368,"Index":12,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":369,"Index":13,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":370,"Index":14,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":371,"Index":15,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":372,"Index":16,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":373,"Index":17,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":374,"Index":18,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":375,"Index":19,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerJobStart","Job ID":44,"Submission Time":1646432702723,"Stage Infos":[{"Stage ID":80,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":196,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9304\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[195],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":194,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[193],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":195,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9303\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[194],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":193,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[80],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_9304\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":80,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":196,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9304\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[195],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":194,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[193],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":195,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9303\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[194],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":193,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432702725,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_9304\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":376,"Index":0,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":377,"Index":1,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":378,"Index":2,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":379,"Index":3,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":380,"Index":4,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":381,"Index":5,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":382,"Index":6,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":383,"Index":7,"Attempt":0,"Launch Time":1646432702738,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":384,"Index":8,"Attempt":0,"Launch Time":1646432702738,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":80,"Stage Attempt ID":0,"Task Info":{"Task ID":385,"Index":9,"Attempt":0,"Launch Time":1646432702738,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":365,"Index":9,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432714736,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"313","Value":"313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"11997","Value":"11997","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"12013","Value":"12013","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"12006","Value":"12006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"5617658","Value":"5617658","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1280","Value":"1280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"175932825","Value":"175932825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7512","Value":"7512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11907","Value":"11907","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"313","Value":"313","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"12851455","Value":"12851455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2765225,"Value":2765225,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":12026,"Value":12026,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":11845529119,"Value":11845529119,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":2563,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":86,"Value":86,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11907,"Value":11907,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":313,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":12851455,"Value":12851455,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":5617658,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1280452812,"Value":1280452812,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":175932825,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4008697456,"JVMOffHeapMemory":144421960,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":92,"MinorGCTime":1322,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2765225,"Executor Run Time":12026,"Executor CPU Time":11845529119,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":86,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11907,"Shuffle Write Time":12851455,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":382,"Index":6,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432714867,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"12103","Value":"12103","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"12109","Value":"12109","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"5819079","Value":"5819079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1316","Value":"1316","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"182236865","Value":"182236865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"336","Value":"336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"952","Value":"952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3650409","Value":"3650409","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1749320,"Value":1749320,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":12113,"Value":12113,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":11975727431,"Value":11975727431,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2132,"Value":2132,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":952,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3650409,"Value":3650409,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":5819079,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1316101120,"Value":1316101120,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":182236865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10810309800,"JVMOffHeapMemory":145019640,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1552270,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":94,"MinorGCTime":1728,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1749320,"Executor Run Time":12113,"Executor CPU Time":11975727431,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":3650409,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":383,"Index":7,"Attempt":0,"Launch Time":1646432702738,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432715067,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"14","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"12304","Value":"24407","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"12310","Value":"24419","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"5819811","Value":"11638890","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1287","Value":"2603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"362930708","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"180693843","Value":"362930708","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"336","Value":"672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"952","Value":"1904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"14","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3421382","Value":"7071791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1804390,"Value":3553710,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":12313,"Value":24426,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":12054981658,"Value":24030709089,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2133,"Value":4265,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":182,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":952,"Value":1904,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":14,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3421382,"Value":7071791,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":362930708,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":11638890,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1287024229,"Value":2603125349,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":362930708,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4008697456,"JVMOffHeapMemory":144421960,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":92,"MinorGCTime":1322,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1804390,"Executor Run Time":12313,"Executor CPU Time":12054981658,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":145,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":952,"Shuffle Write Time":3421382,"Shuffle Records Written":14},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":374,"Index":18,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432715504,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"320","Value":"633","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"134742016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"12767","Value":"24764","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"2000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"12782","Value":"24795","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"12774","Value":"12774","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"5674621","Value":"5674621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"1286","Value":"1286","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"178217644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"178217644","Value":"178217644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7680","Value":"15192","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"12025","Value":"23932","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"320","Value":"633","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"12884013","Value":"25735468","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2260497,"Value":5025722,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":12796,"Value":24822,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":12685805360,"Value":24531334479,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":5126,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":31,"Value":117,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":134742016,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12025,"Value":23932,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":633,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":12884013,"Value":25735468,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":354150469,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":11292279,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1286272096,"Value":2566724908,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":354150469,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6186343392,"JVMOffHeapMemory":147490248,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":101,"MinorGCTime":2393,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2260497,"Executor Run Time":12796,"Executor CPU Time":12685805360,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":31,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12025,"Shuffle Write Time":12884013,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":371,"Index":15,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432715611,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"313","Value":"946","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"12879","Value":"37643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"12890","Value":"37685","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"12882","Value":"25656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"6096762","Value":"11771383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"1328","Value":"2614","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"362157152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"183939508","Value":"362157152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7512","Value":"22704","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11789","Value":"35721","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"313","Value":"946","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"12367487","Value":"38102955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2782842,"Value":7808564,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":12903,"Value":37725,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":12770336637,"Value":37301671116,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2562,"Value":7688,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":45,"Value":162,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11789,"Value":35721,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":313,"Value":946,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":12367487,"Value":38102955,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":538089977,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":17389041,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1328236424,"Value":3894961332,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":538089977,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10810309800,"JVMOffHeapMemory":145019640,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1552270,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":94,"MinorGCTime":1728,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2782842,"Executor Run Time":12903,"Executor CPU Time":12770336637,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":45,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11789,"Shuffle Write Time":12367487,"Shuffle Records Written":313},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":375,"Index":19,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432715634,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"310","Value":"1256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"12900","Value":"50543","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"4007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"12912","Value":"50597","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"12903","Value":"38559","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"5617658","Value":"17389041","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"1322","Value":"3936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"538089977","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"175932825","Value":"538089977","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7440","Value":"30144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11887","Value":"47608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"310","Value":"1256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"13550248","Value":"51653203","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3247256,"Value":11055820,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":12925,"Value":50650,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":12770944600,"Value":50072615716,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2562,"Value":10250,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":45,"Value":207,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11887,"Value":47608,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":310,"Value":1256,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":13550248,"Value":51653203,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":714022802,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":23006699,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1322148385,"Value":5217109717,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":714022802,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10810309800,"JVMOffHeapMemory":145019640,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1552270,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":94,"MinorGCTime":1728,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3247256,"Executor Run Time":12925,"Executor CPU Time":12770944600,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":45,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11887,"Shuffle Write Time":13550248,"Shuffle Records Written":310},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":373,"Index":17,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432715722,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"324","Value":"1580","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"12986","Value":"63529","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"5014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"13002","Value":"63599","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"12995","Value":"51554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"5819811","Value":"23208852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"1320","Value":"5256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"718783820","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"180693843","Value":"718783820","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7776","Value":"37920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"12053","Value":"59661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"324","Value":"1580","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11754258","Value":"63407461","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2701716,"Value":13757536,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":13014,"Value":63664,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":12771850376,"Value":62844466092,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":12813,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":352,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12053,"Value":59661,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":324,"Value":1580,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11754258,"Value":63407461,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":894716645,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":28826510,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1320695018,"Value":6537804735,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":894716645,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4008697456,"JVMOffHeapMemory":144421960,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":92,"MinorGCTime":1322,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2701716,"Executor Run Time":13014,"Executor CPU Time":12771850376,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":145,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12053,"Shuffle Write Time":11754258,"Shuffle Records Written":324},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":384,"Index":8,"Attempt":0,"Launch Time":1646432702738,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432715745,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"12","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"202113024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"12981","Value":"37388","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"3000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"12988","Value":"37407","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"5674621","Value":"17313511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1529","Value":"4132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"541148352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"178217644","Value":"541148352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"288","Value":"960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"816","Value":"2720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"12","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3385128","Value":"10456919","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1740003,"Value":5293713,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":12991,"Value":37417,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":12637750420,"Value":36668459509,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2133,"Value":6398,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":31,"Value":213,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":202113024,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":2720,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3385128,"Value":10456919,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":541148352,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":17313511,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1529335722,"Value":4132461071,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":541148352,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6186343392,"JVMOffHeapMemory":147490248,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":101,"MinorGCTime":2393,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1740003,"Executor Run Time":12991,"Executor CPU Time":12637750420,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":31,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":3385128,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":379,"Index":3,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432715845,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"19","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"269484032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"13082","Value":"50470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"4000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"13088","Value":"50495","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"6429338","Value":"23742849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1374","Value":"5506","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"736222667","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"195074315","Value":"736222667","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"456","Value":"1416","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"1232","Value":"3952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"19","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3703630","Value":"14160549","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2020610,"Value":7314323,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":13091,"Value":50508,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":12882293786,"Value":49550753295,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2133,"Value":8531,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":358,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":269484032,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1232,"Value":3952,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":19,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3703630,"Value":14160549,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":736222667,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":23742849,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1374778999,"Value":5507240070,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":736222667,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":4008697456,"JVMOffHeapMemory":144421960,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":92,"MinorGCTime":1322,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2020610,"Executor Run Time":13091,"Executor CPU Time":12882293786,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":145,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1232,"Shuffle Write Time":3703630,"Shuffle Records Written":19},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":362,"Index":6,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432715882,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"322","Value":"1902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"13147","Value":"76676","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"13162","Value":"76761","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"13154","Value":"25160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"5819079","Value":"11436737","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1286","Value":"2566","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"358169690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"182236865","Value":"358169690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7728","Value":"45648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"12103","Value":"71764","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"322","Value":"1902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"12076177","Value":"75483638","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2140314,"Value":15897850,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":13175,"Value":76839,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":13084980297,"Value":75929446389,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":15376,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":31,"Value":383,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12103,"Value":71764,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":1902,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":12076177,"Value":75483638,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":1076953510,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":34645589,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1286285723,"Value":7824090458,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":1076953510,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6186343392,"JVMOffHeapMemory":147490248,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":101,"MinorGCTime":2393,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2140314,"Executor Run Time":13175,"Executor CPU Time":13084980297,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":31,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12103,"Shuffle Write Time":12076177,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":359,"Index":3,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432716312,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"296","Value":"2198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"13581","Value":"90257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"7021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"13591","Value":"90352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"13584","Value":"38744","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"6429338","Value":"17866075","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1368","Value":"3934","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"553244005","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"195074315","Value":"553244005","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7104","Value":"52752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11482","Value":"83246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"296","Value":"2198","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11693426","Value":"87177064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3040531,"Value":18938381,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":13603,"Value":90442,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":13477971381,"Value":89407417770,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2562,"Value":17938,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":45,"Value":428,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11482,"Value":83246,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":2198,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11693426,"Value":87177064,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1272027825,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":41074927,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1368778951,"Value":9192869409,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1272027825,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10810309800,"JVMOffHeapMemory":145019640,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1552270,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":94,"MinorGCTime":1728,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":3040531,"Executor Run Time":13603,"Executor CPU Time":13477971381,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":45,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11482,"Shuffle Write Time":11693426,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":378,"Index":2,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432716492,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"18","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"336855040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"13729","Value":"64199","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"5000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"13734","Value":"64229","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"6450117","Value":"30192966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1436","Value":"6942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"933036967","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"196814300","Value":"933036967","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"432","Value":"1848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"1164","Value":"5116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"18","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3754289","Value":"17914838","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1897524,"Value":9211847,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":13739,"Value":64247,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":13596689833,"Value":63147443128,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2132,"Value":10663,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":45,"Value":403,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":336855040,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1164,"Value":5116,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3754289,"Value":17914838,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":933036967,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":30192966,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1436953728,"Value":6944193798,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":933036967,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10810309800,"JVMOffHeapMemory":145019640,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1552270,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":94,"MinorGCTime":1728,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1897524,"Executor Run Time":13739,"Executor CPU Time":13596689833,"Peak Execution Memory":67371008,"Result Size":2132,"JVM GC Time":45,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1164,"Shuffle Write Time":3754289,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":363,"Index":7,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432716557,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"325","Value":"2523","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"13826","Value":"104083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"8028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"13837","Value":"104189","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"13829","Value":"52573","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"5819811","Value":"23685886","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1397","Value":"5331","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"733937848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"180693843","Value":"733937848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7800","Value":"60552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"12061","Value":"95307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"325","Value":"2523","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11607772","Value":"98784836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2869151,"Value":21807532,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":13849,"Value":104291,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":13741071669,"Value":103148489439,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2562,"Value":20500,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":45,"Value":473,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12061,"Value":95307,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":325,"Value":2523,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11607772,"Value":98784836,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":1452721668,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":46894738,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1397506280,"Value":10590375689,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":1452721668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10810309800,"JVMOffHeapMemory":145019640,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1552270,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":94,"MinorGCTime":1728,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2869151,"Executor Run Time":13849,"Executor CPU Time":13741071669,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":45,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12061,"Shuffle Write Time":11607772,"Shuffle Records Written":325},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":357,"Index":1,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432716777,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"320","Value":"2843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"14041","Value":"118124","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"14057","Value":"118246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"14050","Value":"66623","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"6369482","Value":"30055368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1421","Value":"6752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"931576382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"197638534","Value":"931576382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7680","Value":"68232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11962","Value":"107269","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"320","Value":"2843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11799905","Value":"110584741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2953834,"Value":24761366,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":14069,"Value":118360,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":13831441379,"Value":116979930818,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":23063,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":618,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11962,"Value":107269,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":2843,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11799905,"Value":110584741,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1650360202,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":53264220,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1421392651,"Value":12011768340,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1650360202,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10194255400,"JVMOffHeapMemory":144555296,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":95,"MinorGCTime":1422,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2953834,"Executor Run Time":14069,"Executor CPU Time":13831441379,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":145,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11962,"Shuffle Write Time":11799905,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":370,"Index":14,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432717110,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"301","Value":"3144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"14375","Value":"132499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"14390","Value":"132636","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"14383","Value":"65937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"6066650","Value":"29275502","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"1416","Value":"6672","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"903819700","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"185035880","Value":"903819700","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7224","Value":"75456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11463","Value":"118732","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"301","Value":"3144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11609694","Value":"122194435","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2174127,"Value":26935493,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":14402,"Value":132762,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":14206974499,"Value":131186905317,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":25626,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":39,"Value":657,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11463,"Value":118732,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":301,"Value":3144,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11609694,"Value":122194435,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1835396082,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":59330870,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1416050894,"Value":13427819234,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1835396082,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6186343392,"JVMOffHeapMemory":147490248,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":101,"MinorGCTime":2393,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2174127,"Executor Run Time":14402,"Executor CPU Time":14206974499,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":39,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11463,"Shuffle Write Time":11609694,"Shuffle Records Written":301},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":377,"Index":1,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432717171,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"16","Value":"93","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"404226048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"14407","Value":"78606","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"6000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"14412","Value":"78641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"6369482","Value":"36562448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1434","Value":"8376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1130675501","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"197638534","Value":"1130675501","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"384","Value":"2232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"1088","Value":"6204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"16","Value":"93","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3669724","Value":"21584562","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1791635,"Value":11003482,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":14416,"Value":78663,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":14273097687,"Value":77420540815,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2133,"Value":12796,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":59,"Value":462,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":404226048,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1088,"Value":6204,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":93,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3669724,"Value":21584562,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1130675501,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":36562448,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1434684096,"Value":8378877894,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1130675501,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1791635,"Executor Run Time":14416,"Executor CPU Time":14273097687,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":59,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1088,"Shuffle Write Time":3669724,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":385,"Index":9,"Attempt":0,"Launch Time":1646432702738,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432717421,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"12","Value":"105","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"471597056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"14658","Value":"93264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"7000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"14662","Value":"93303","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"5617658","Value":"42180106","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1404","Value":"9780","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"1306608326","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"175932825","Value":"1306608326","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"288","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"816","Value":"7020","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"12","Value":"105","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3343989","Value":"24928551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1785866,"Value":12789348,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":14666,"Value":93329,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":14476082059,"Value":91896622874,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2133,"Value":14929,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":59,"Value":521,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":471597056,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":816,"Value":7020,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":12,"Value":105,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3343989,"Value":24928551,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":1306608326,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":42180106,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1404008510,"Value":9782886404,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":1306608326,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1785866,"Executor Run Time":14666,"Executor CPU Time":14476082059,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":59,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":816,"Shuffle Write Time":3343989,"Shuffle Records Written":12},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":376,"Index":0,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432717925,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"18","Value":"123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"538968064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"15160","Value":"108424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"8000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"15168","Value":"108471","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"7213446","Value":"49393552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1639","Value":"11419","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1535214946","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"228606620","Value":"1535214946","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"432","Value":"2952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"1224","Value":"8244","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"18","Value":"123","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3552603","Value":"28481154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1742431,"Value":14531779,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":15171,"Value":108500,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":15010708406,"Value":106907331280,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2133,"Value":17062,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":39,"Value":560,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":538968064,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1224,"Value":8244,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":123,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3552603,"Value":28481154,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1535214946,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":49393552,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1639889065,"Value":11422775469,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1535214946,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6186343392,"JVMOffHeapMemory":147490248,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":101,"MinorGCTime":2393,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1742431,"Executor Run Time":15171,"Executor CPU Time":15010708406,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":39,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1224,"Shuffle Write Time":3552603,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":360,"Index":4,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432718476,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"299","Value":"3443","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"741081088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"15745","Value":"148244","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"11028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"15755","Value":"148391","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"15748","Value":"82371","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"6066650","Value":"36122018","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1456","Value":"8208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1116612262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"185035880","Value":"1116612262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7176","Value":"82632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11443","Value":"130175","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"299","Value":"3443","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"12032390","Value":"134226825","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2423445,"Value":29358938,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":15767,"Value":148529,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":15621777272,"Value":146808682589,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":28189,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":59,"Value":716,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":741081088,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11443,"Value":130175,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":299,"Value":3443,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":12032390,"Value":134226825,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":2020431962,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":65397520,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1456992191,"Value":14884811425,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":2020431962,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2423445,"Executor Run Time":15767,"Executor CPU Time":15621777272,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":59,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11443,"Shuffle Write Time":12032390,"Shuffle Records Written":299},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":380,"Index":4,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432718517,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"16","Value":"139","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"606339072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"15754","Value":"124178","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"9000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"15761","Value":"124232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"6066650","Value":"55460202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1511","Value":"12930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"1720250826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"185035880","Value":"1720250826","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"384","Value":"3336","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"1028","Value":"9272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"16","Value":"139","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3480927","Value":"31962081","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1786189,"Value":16317968,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":15764,"Value":124264,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":15643309504,"Value":122550640784,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2133,"Value":19195,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":39,"Value":599,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":606339072,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1028,"Value":9272,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":16,"Value":139,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3480927,"Value":31962081,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":1720250826,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":55460202,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1511875343,"Value":12934650812,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":1720250826,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6186343392,"JVMOffHeapMemory":147490248,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":101,"MinorGCTime":2393,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1786189,"Executor Run Time":15764,"Executor CPU Time":15643309504,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":39,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1028,"Shuffle Write Time":3480927,"Shuffle Records Written":16},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":358,"Index":2,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432719097,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"305","Value":"3748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"808452096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"16363","Value":"164607","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"12028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"16378","Value":"164769","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"16371","Value":"98742","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"6450117","Value":"42572135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1792","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1313426562","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"196814300","Value":"1313426562","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7320","Value":"89952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11668","Value":"141843","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"305","Value":"3748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"10676235","Value":"144903060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2223740,"Value":31582678,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":16389,"Value":164918,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":16001002695,"Value":162809685284,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":30752,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":39,"Value":755,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":808452096,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":141843,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":3748,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":10676235,"Value":144903060,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":2217246262,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":71847637,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1792665977,"Value":16677477402,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":2217246262,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6186343392,"JVMOffHeapMemory":147490248,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":101,"MinorGCTime":2393,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2223740,"Executor Run Time":16389,"Executor CPU Time":16001002695,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":39,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":10676235,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":372,"Index":16,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432719165,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"322","Value":"4070","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"875823104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"16434","Value":"181041","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"13028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"16444","Value":"181213","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"16437","Value":"82374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"5819079","Value":"35094581","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"1501","Value":"8173","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"1086056565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"182236865","Value":"1086056565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7728","Value":"97680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"12099","Value":"153942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"322","Value":"4070","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"12020252","Value":"156923312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2261610,"Value":33844288,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":16457,"Value":181375,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":16301922794,"Value":179111608078,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":33315,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":59,"Value":814,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":875823104,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12099,"Value":153942,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":322,"Value":4070,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":12020252,"Value":156923312,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":2399483127,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":77666716,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1501019048,"Value":18178496450,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":2399483127,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2261610,"Executor Run Time":16457,"Executor CPU Time":16301922794,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":59,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12099,"Shuffle Write Time":12020252,"Shuffle Records Written":322},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":369,"Index":13,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432719893,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"296","Value":"4366","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"943194112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"17159","Value":"198200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1007","Value":"14035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"17174","Value":"198387","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"17167","Value":"99541","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"6429338","Value":"41523919","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"1779","Value":"9952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1281130880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"195074315","Value":"1281130880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7104","Value":"104784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11481","Value":"165423","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"296","Value":"4366","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"10870492","Value":"167793804","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2718930,"Value":36563218,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":17186,"Value":198561,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":16740081232,"Value":195851689310,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":35878,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":157,"Value":971,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":943194112,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11481,"Value":165423,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":296,"Value":4366,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":10870492,"Value":167793804,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":2594557442,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":84096054,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1779826133,"Value":19958322583,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":2594557442,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10194255400,"JVMOffHeapMemory":144555296,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":95,"MinorGCTime":1422,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2718930,"Executor Run Time":17186,"Executor CPU Time":16740081232,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":157,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11481,"Shuffle Write Time":10870492,"Shuffle Records Written":296},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":361,"Index":5,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432720292,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"312","Value":"4678","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"1010565120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"17558","Value":"215758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"15035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"17574","Value":"215961","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"17566","Value":"116308","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"6096762","Value":"48668897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1740","Value":"11740","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1497366070","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"183939508","Value":"1497366070","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7488","Value":"112272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11723","Value":"177146","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"312","Value":"4678","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"10869389","Value":"178663193","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1987525,"Value":38550743,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":17585,"Value":216146,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":17166279299,"Value":213017968609,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":38441,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":157,"Value":1128,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1010565120,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11723,"Value":177146,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":312,"Value":4678,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":10869389,"Value":178663193,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":2778496950,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":90192816,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1740639960,"Value":21698962543,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":2778496950,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10194255400,"JVMOffHeapMemory":144555296,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":95,"MinorGCTime":1422,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1987525,"Executor Run Time":17585,"Executor CPU Time":17166279299,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":157,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11723,"Shuffle Write Time":10869389,"Shuffle Records Written":312},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":367,"Index":11,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432720414,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"318","Value":"4996","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"1077936128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"17684","Value":"233442","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"16035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"17694","Value":"233655","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"17687","Value":"117228","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"6369482","Value":"47893401","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"1727","Value":"11679","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"197638534","Value":"1478769414","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7632","Value":"119904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11947","Value":"189093","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"318","Value":"4996","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11034708","Value":"189697901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":4,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3052144,"Value":41602887,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":17705,"Value":233851,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":17549482363,"Value":230567450972,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2562,"Value":41003,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":45,"Value":1173,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1077936128,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11947,"Value":189093,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":318,"Value":4996,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11034708,"Value":189697901,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":2976135484,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":96562298,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1727536853,"Value":23426499396,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":2976135484,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10810309800,"JVMOffHeapMemory":145019640,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1552270,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":94,"MinorGCTime":1728,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":4,"Executor Deserialize CPU Time":3052144,"Executor Run Time":17705,"Executor CPU Time":17549482363,"Peak Execution Memory":67371008,"Result Size":2562,"JVM GC Time":45,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11947,"Shuffle Write Time":11034708,"Shuffle Records Written":318},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":368,"Index":12,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432720798,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"305","Value":"5301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"1145307136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"18068","Value":"251510","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"17035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"18078","Value":"251733","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"18071","Value":"135299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"6450117","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"1561","Value":"13240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"196814300","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7320","Value":"127224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"11668","Value":"200761","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"305","Value":"5301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11630903","Value":"201328804","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2197667,"Value":43800554,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":18090,"Value":251941,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":17907803530,"Value":248475254502,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":43566,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":69,"Value":1242,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1145307136,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":11668,"Value":200761,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":305,"Value":5301,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11630903,"Value":201328804,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":3172949784,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":103012415,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1561899946,"Value":24988399342,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":3172949784,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2197667,"Executor Run Time":18090,"Executor CPU Time":17907803530,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":69,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":11668,"Shuffle Write Time":11630903,"Shuffle Records Written":305},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":80,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":381,"Index":5,"Attempt":0,"Launch Time":1646432702736,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432720896,"Failed":false,"Killed":false,"Accumulables":[{"ID":2637,"Name":"number of output rows","Update":"15","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Update":"67371008","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Update":"18133","Value":"142311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Update":"18137","Value":"142369","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Update":"6096762","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Update":"1608","Value":"14538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Update":"183939508","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Update":"360","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Update":"1020","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Update":"15","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Update":"3441654","Value":"35403735","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1786756,"Value":18104724,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Update":18141,"Value":142405,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Update":17909298398,"Value":140459939182,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Update":2133,"Value":21328,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Update":69,"Value":668,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1020,"Value":10292,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":15,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Update":3441654,"Value":35403735,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1608245170,"Value":14542895982,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1786756,"Executor Run Time":18141,"Executor CPU Time":17909298398,"Peak Execution Memory":67371008,"Result Size":2133,"JVM GC Time":69,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":1020,"Shuffle Write Time":3441654,"Shuffle Records Written":15},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":80,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":196,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9304\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[195],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":194,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[193],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":195,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9303\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[194],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":193,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432702725,"Completion Time":1646432720897,"Accumulables":[{"ID":2637,"Name":"number of output rows","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2638,"Name":"peak memory","Value":"673710080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"time in aggregation build","Value":"142311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2641,"Name":"avg hash probe bucket list iters","Value":"10000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2673,"Name":"duration","Value":"142369","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2674,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2678,"Name":"filesystem read time (sampled)","Value":"14538","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2679,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2682,"Name":"data size","Value":"3696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2690,"Name":"shuffle bytes written","Value":"10292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2691,"Name":"shuffle records written","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2692,"Name":"shuffle write time","Value":"35403735","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2693,"Name":"internal.metrics.executorDeserializeTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":2694,"Name":"internal.metrics.executorDeserializeCpuTime","Value":18104724,"Internal":true,"Count Failed Values":true},{"ID":2695,"Name":"internal.metrics.executorRunTime","Value":142405,"Internal":true,"Count Failed Values":true},{"ID":2696,"Name":"internal.metrics.executorCpuTime","Value":140459939182,"Internal":true,"Count Failed Values":true},{"ID":2697,"Name":"internal.metrics.resultSize","Value":21328,"Internal":true,"Count Failed Values":true},{"ID":2698,"Name":"internal.metrics.jvmGCTime","Value":668,"Internal":true,"Count Failed Values":true},{"ID":2702,"Name":"internal.metrics.peakExecutionMemory","Value":673710080,"Internal":true,"Count Failed Values":true},{"ID":2713,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":10292,"Internal":true,"Count Failed Values":true},{"ID":2714,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":154,"Internal":true,"Count Failed Values":true},{"ID":2715,"Name":"internal.metrics.shuffle.write.writeTime","Value":35403735,"Internal":true,"Count Failed Values":true},{"ID":2716,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":2717,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2718,"Name":"internal.metrics.input.sampledTimeReadNano","Value":14542895982,"Internal":true,"Count Failed Values":true},{"ID":2719,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":44,"Completion Time":1646432720897,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":45,"Submission Time":1646432720984,"Stage Infos":[{"Stage ID":81,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":196,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9304\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[195],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":194,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[193],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":195,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9303\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[194],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":193,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":82,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":199,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9656\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[198],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":197,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_9650\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[196],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":198,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9655\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[197],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[81],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[81,82],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_9656\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":82,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":199,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9656\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[198],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":197,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_9650\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[196],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":198,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9655\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[197],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[81],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432720984,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_9656\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":82,"Stage Attempt ID":0,"Task Info":{"Task ID":386,"Index":0,"Attempt":0,"Launch Time":1646432721075,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":82,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":386,"Index":0,"Attempt":0,"Launch Time":1646432721075,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432721108,"Failed":false,"Killed":false,"Accumulables":[{"ID":2683,"Name":"remote blocks read","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2684,"Name":"local blocks read","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2685,"Name":"remote bytes read","Update":"8176","Value":"8176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2687,"Name":"local bytes read","Update":"2116","Value":"2116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2688,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2689,"Name":"records read","Update":"154","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2723,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2724,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2726,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2727,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2728,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2731,"Name":"data size","Update":"552","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2739,"Name":"shuffle bytes written","Update":"265","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2740,"Name":"shuffle records written","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2741,"Name":"shuffle write time","Update":"204072","Value":"204072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2742,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2743,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2128368,"Value":2128368,"Internal":true,"Count Failed Values":true},{"ID":2744,"Name":"internal.metrics.executorRunTime","Update":9,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2745,"Name":"internal.metrics.executorCpuTime","Update":6405758,"Value":6405758,"Internal":true,"Count Failed Values":true},{"ID":2746,"Name":"internal.metrics.resultSize","Update":2737,"Value":2737,"Internal":true,"Count Failed Values":true},{"ID":2751,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2755,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2756,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2757,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":8176,"Value":8176,"Internal":true,"Count Failed Values":true},{"ID":2758,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2759,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2116,"Value":2116,"Internal":true,"Count Failed Values":true},{"ID":2760,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2761,"Name":"internal.metrics.shuffle.read.recordsRead","Update":154,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":2762,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":265,"Value":265,"Internal":true,"Count Failed Values":true},{"ID":2763,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2764,"Name":"internal.metrics.shuffle.write.writeTime","Update":204072,"Value":204072,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2128368,"Executor Run Time":9,"Executor CPU Time":6405758,"Peak Execution Memory":67371008,"Result Size":2737,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":8176,"Remote Bytes Read To Disk":0,"Local Bytes Read":2116,"Total Records Read":154},"Shuffle Write Metrics":{"Shuffle Bytes Written":265,"Shuffle Write Time":204072,"Shuffle Records Written":23},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":82,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":199,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9656\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[198],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":197,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_9650\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[196],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":198,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9655\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[197],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[81],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432720984,"Completion Time":1646432721109,"Accumulables":[{"ID":2683,"Name":"remote blocks read","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2684,"Name":"local blocks read","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2685,"Name":"remote bytes read","Value":"8176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2687,"Name":"local bytes read","Value":"2116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2688,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2689,"Name":"records read","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2723,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2724,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2726,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2727,"Name":"avg hash probe bucket list iters","Value":"1000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2728,"Name":"duration","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2731,"Name":"data size","Value":"552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2739,"Name":"shuffle bytes written","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2740,"Name":"shuffle records written","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2741,"Name":"shuffle write time","Value":"204072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2742,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":2743,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2128368,"Internal":true,"Count Failed Values":true},{"ID":2744,"Name":"internal.metrics.executorRunTime","Value":9,"Internal":true,"Count Failed Values":true},{"ID":2745,"Name":"internal.metrics.executorCpuTime","Value":6405758,"Internal":true,"Count Failed Values":true},{"ID":2746,"Name":"internal.metrics.resultSize","Value":2737,"Internal":true,"Count Failed Values":true},{"ID":2751,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2755,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":8,"Internal":true,"Count Failed Values":true},{"ID":2756,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":2757,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":8176,"Internal":true,"Count Failed Values":true},{"ID":2758,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2759,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2116,"Internal":true,"Count Failed Values":true},{"ID":2760,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2761,"Name":"internal.metrics.shuffle.read.recordsRead","Value":154,"Internal":true,"Count Failed Values":true},{"ID":2762,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":265,"Internal":true,"Count Failed Values":true},{"ID":2763,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23,"Internal":true,"Count Failed Values":true},{"ID":2764,"Name":"internal.metrics.shuffle.write.writeTime","Value":204072,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":45,"Completion Time":1646432721115,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":364,"Index":8,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432721149,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"320","Value":"5621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"1212678144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"18418","Value":"269928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"18035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"18429","Value":"270162","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"18422","Value":"134730","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"5674621","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1600","Value":"13340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"178217644","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"7680","Value":"134904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"12024","Value":"212785","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"320","Value":"5621","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11189715","Value":"212518519","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2665966,"Value":46466520,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":18440,"Value":270381,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":18268117078,"Value":266743371580,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":46129,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":69,"Value":1311,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1212678144,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12024,"Value":212785,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":320,"Value":5621,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11189715,"Value":212518519,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":3351167428,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":108687036,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1600515518,"Value":26588914860,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":3351167428,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2665966,"Executor Run Time":18440,"Executor CPU Time":18268117078,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":69,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12024,"Shuffle Write Time":11189715,"Shuffle Records Written":320},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerJobStart","Job ID":46,"Submission Time":1646432721165,"Stage Infos":[{"Stage ID":84,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":199,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9656\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[198],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":197,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_9650\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[196],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":198,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9655\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[197],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[83],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":85,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":201,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9909\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[200],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":200,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[199],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[84],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":83,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":196,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9304\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[195],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":194,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[193],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":195,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9303\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[194],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":193,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8771\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[84,85,83],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_6\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId ddc0e167-b619-4974-8fb1-246eb5c00202)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"ddc0e167-b619-4974-8fb1-246eb5c00202","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":85,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":201,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9909\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[200],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":200,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[199],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[84],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432721166,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_6\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId ddc0e167-b619-4974-8fb1-246eb5c00202)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"ddc0e167-b619-4974-8fb1-246eb5c00202","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":85,"Stage Attempt ID":0,"Task Info":{"Task ID":387,"Index":0,"Attempt":0,"Launch Time":1646432721172,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":85,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":387,"Index":0,"Attempt":0,"Launch Time":1646432721172,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432721190,"Failed":false,"Killed":false,"Accumulables":[{"ID":2732,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2734,"Name":"remote bytes read","Update":"265","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2737,"Name":"fetch wait time","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2738,"Name":"records read","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2772,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2773,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2774,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2775,"Name":"cumulative time","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2776,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2777,"Name":"num prefix comparisons","Update":"79","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2779,"Name":"number of output rows","Update":"23","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2780,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2781,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2782,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2190028,"Value":2190028,"Internal":true,"Count Failed Values":true},{"ID":2783,"Name":"internal.metrics.executorRunTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2784,"Name":"internal.metrics.executorCpuTime","Update":3921142,"Value":3921142,"Internal":true,"Count Failed Values":true},{"ID":2785,"Name":"internal.metrics.resultSize","Update":3792,"Value":3792,"Internal":true,"Count Failed Values":true},{"ID":2790,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2794,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2795,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2796,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":265,"Value":265,"Internal":true,"Count Failed Values":true},{"ID":2797,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2798,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2799,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2800,"Name":"internal.metrics.shuffle.read.recordsRead","Update":23,"Value":23,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2190028,"Executor Run Time":6,"Executor CPU Time":3921142,"Peak Execution Memory":67174400,"Result Size":3792,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":1,"Remote Bytes Read":265,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":23},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":85,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":201,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9909\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[200],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":200,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_5\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[199],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[84],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432721166,"Completion Time":1646432721191,"Accumulables":[{"ID":2732,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2734,"Name":"remote bytes read","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2737,"Name":"fetch wait time","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2738,"Name":"records read","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2772,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2773,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2774,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2775,"Name":"cumulative time","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2776,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2777,"Name":"num prefix comparisons","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2779,"Name":"number of output rows","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2780,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2781,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":2782,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2190028,"Internal":true,"Count Failed Values":true},{"ID":2783,"Name":"internal.metrics.executorRunTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":2784,"Name":"internal.metrics.executorCpuTime","Value":3921142,"Internal":true,"Count Failed Values":true},{"ID":2785,"Name":"internal.metrics.resultSize","Value":3792,"Internal":true,"Count Failed Values":true},{"ID":2790,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2794,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2795,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2796,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":265,"Internal":true,"Count Failed Values":true},{"ID":2797,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2798,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2799,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2800,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":46,"Completion Time":1646432721192,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":356,"Index":0,"Attempt":0,"Launch Time":1646432702686,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432722038,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"358","Value":"5979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"1280049152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"19307","Value":"289235","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"19035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"19318","Value":"289480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Update":"19310","Value":"154040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Update":"1810","Value":"15150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"8592","Value":"143496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"12849","Value":"225634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"358","Value":"5979","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11246502","Value":"223765021","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":3,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2473577,"Value":48940097,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":19329,"Value":289710,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":19163400482,"Value":285906772062,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":48692,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":69,"Value":1380,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1280049152,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12849,"Value":225634,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":5979,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11246502,"Value":223765021,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":3579774048,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":115900482,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1810182418,"Value":28399097278,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":3579774048,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":3,"Executor Deserialize CPU Time":2473577,"Executor Run Time":19329,"Executor CPU Time":19163400482,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":69,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12849,"Shuffle Write Time":11246502,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":350,"Index":4,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432722191,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"6066650","Value":"6066650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"19470","Value":"19470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"6066650","Value":"6066650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"1389","Value":"1389","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"185035880","Value":"185035880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"185035880","Value":"185035880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"582398400","Value":"582398400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"79904590","Value":"79904590","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"6066650","Value":"6066650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"187384366","Value":"187384366","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1486718,"Value":1486718,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":19558,"Value":19558,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":19380137449,"Value":19380137449,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1703,"Value":1703,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":69,"Value":69,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79904590,"Value":79904590,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6066650,"Value":6066650,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":187384366,"Value":187384366,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":185035880,"Value":185035880,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":6066650,"Value":6066650,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1389689233,"Value":1389689233,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":185035880,"Value":185035880,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1486718,"Executor Run Time":19558,"Executor CPU Time":19380137449,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":69,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":79904590,"Shuffle Write Time":187384366,"Shuffle Records Written":6066650},"Input Metrics":{"Bytes Read":185035880,"Records Read":6066650},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":354,"Index":8,"Attempt":0,"Launch Time":1646432702616,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432722508,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"5674621","Value":"11741271","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"19789","Value":"39259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"5674621","Value":"11741271","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"1344","Value":"2733","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"178217644","Value":"363253524","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"178217644","Value":"363253524","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"544763616","Value":"1127162016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"75751912","Value":"155656502","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"5674621","Value":"11741271","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"180250698","Value":"367635064","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1496612,"Value":2983330,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":19874,"Value":39432,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":19686791899,"Value":39066929348,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1703,"Value":3406,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":69,"Value":138,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":75751912,"Value":155656502,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5674621,"Value":11741271,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":180250698,"Value":367635064,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":178217644,"Value":363253524,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":5674621,"Value":11741271,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1344265600,"Value":2733954833,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":178217644,"Value":363253524,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":16292853544,"JVMOffHeapMemory":146230832,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1524,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1496612,"Executor Run Time":19874,"Executor CPU Time":19686791899,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":69,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":75751912,"Shuffle Write Time":180250698,"Shuffle Records Written":5674621},"Input Metrics":{"Bytes Read":178217644,"Records Read":5674621},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":366,"Index":10,"Attempt":0,"Launch Time":1646432702687,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432722927,"Failed":false,"Killed":false,"Accumulables":[{"ID":2571,"Name":"number of output rows","Update":"358","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Update":"67371008","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Update":"20192","Value":"309427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Update":"1000","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Update":"20207","Value":"309687","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Update":"20200","Value":"155499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Update":"2763","Value":"16003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Update":"8592","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Update":"12847","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Update":"358","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Update":"11098656","Value":"234863677","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2341774,"Value":51281871,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Update":20219,"Value":309929,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Update":18810619308,"Value":304717391370,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Update":2563,"Value":51255,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":1427,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":12847,"Value":238481,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":358,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Update":11098656,"Value":234863677,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Update":2763182415,"Value":31162279693,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":3808380668,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6458964792,"JVMOffHeapMemory":147555256,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":104,"MinorGCTime":2416,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2341774,"Executor Run Time":20219,"Executor CPU Time":18810619308,"Peak Execution Memory":67371008,"Result Size":2563,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":12847,"Shuffle Write Time":11098656,"Shuffle Records Written":358},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":79,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":192,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9228\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[191],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":189,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9223\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[188],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":184,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":185,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[184],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":186,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9220\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[185],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":190,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_9224\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[186,189],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":188,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[187],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":187,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":191,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9227\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[190],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432702670,"Completion Time":1646432722927,"Accumulables":[{"ID":2571,"Name":"number of output rows","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2572,"Name":"peak memory","Value":"1347420160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2574,"Name":"time in aggregation build","Value":"309427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2575,"Name":"avg hash probe bucket list iters","Value":"20035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2607,"Name":"duration","Value":"309687","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2608,"Name":"duration","Value":"154040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2609,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2613,"Name":"filesystem read time (sampled)","Value":"15150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2614,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"duration","Value":"155499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2618,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2622,"Name":"filesystem read time (sampled)","Value":"16003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2623,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2624,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2626,"Name":"data size","Value":"152088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2634,"Name":"shuffle bytes written","Value":"238481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2635,"Name":"shuffle records written","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2636,"Name":"shuffle write time","Value":"234863677","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2643,"Name":"internal.metrics.executorDeserializeTime","Value":50,"Internal":true,"Count Failed Values":true},{"ID":2644,"Name":"internal.metrics.executorDeserializeCpuTime","Value":51281871,"Internal":true,"Count Failed Values":true},{"ID":2645,"Name":"internal.metrics.executorRunTime","Value":309929,"Internal":true,"Count Failed Values":true},{"ID":2646,"Name":"internal.metrics.executorCpuTime","Value":304717391370,"Internal":true,"Count Failed Values":true},{"ID":2647,"Name":"internal.metrics.resultSize","Value":51255,"Internal":true,"Count Failed Values":true},{"ID":2648,"Name":"internal.metrics.jvmGCTime","Value":1427,"Internal":true,"Count Failed Values":true},{"ID":2652,"Name":"internal.metrics.peakExecutionMemory","Value":1347420160,"Internal":true,"Count Failed Values":true},{"ID":2663,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":238481,"Internal":true,"Count Failed Values":true},{"ID":2664,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":2665,"Name":"internal.metrics.shuffle.write.writeTime","Value":234863677,"Internal":true,"Count Failed Values":true},{"ID":2666,"Name":"internal.metrics.input.bytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true},{"ID":2667,"Name":"internal.metrics.input.recordsRead","Value":123113928,"Internal":true,"Count Failed Values":true},{"ID":2668,"Name":"internal.metrics.input.sampledTimeReadNano","Value":31162279693,"Internal":true,"Count Failed Values":true},{"ID":2669,"Name":"internal.metrics.input.sampledBytesRead","Value":3808380668,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":43,"Completion Time":1646432722929,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":348,"Index":2,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432722940,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"6450117","Value":"18191388","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"20210","Value":"59469","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"6450117","Value":"18191388","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"1482","Value":"4215","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"196814300","Value":"560067824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"196814300","Value":"560067824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"619211232","Value":"1746373248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"84724622","Value":"240381124","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"6450117","Value":"18191388","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"229067543","Value":"596702607","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1817330,"Value":4800660,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":20306,"Value":59738,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":20093981500,"Value":59160910848,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1702,"Value":5108,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":56,"Value":194,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":84724622,"Value":240381124,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6450117,"Value":18191388,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":229067543,"Value":596702607,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":196814300,"Value":560067824,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":6450117,"Value":18191388,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1482418514,"Value":4216373347,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":196814300,"Value":560067824,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10810309800,"JVMOffHeapMemory":145019640,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1552270,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":94,"MinorGCTime":1728,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1817330,"Executor Run Time":20306,"Executor CPU Time":20093981500,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":56,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":84724622,"Shuffle Write Time":229067543,"Shuffle Records Written":6450117},"Input Metrics":{"Bytes Read":196814300,"Records Read":6450117},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerJobStart","Job ID":47,"Submission Time":1646432722959,"Stage Infos":[{"Stage ID":86,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":192,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9228\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[191],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":189,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9223\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[188],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":184,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":185,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[184],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":186,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9220\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[185],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":190,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_9224\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[186,189],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":188,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[187],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":187,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":191,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9227\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[190],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":87,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":204,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10300\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[203],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":202,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_10294\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[192],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":203,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10299\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[202],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[86],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[86,87],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_10300\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":87,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":204,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10300\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[203],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":202,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_10294\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[192],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":203,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10299\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[202],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[86],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432722960,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_10300\",\"name\":\"Exchange\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":87,"Stage Attempt ID":0,"Task Info":{"Task ID":388,"Index":0,"Attempt":0,"Launch Time":1646432722966,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":87,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":388,"Index":0,"Attempt":0,"Launch Time":1646432722966,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432722999,"Failed":false,"Killed":false,"Accumulables":[{"ID":2627,"Name":"remote blocks read","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2628,"Name":"local blocks read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2629,"Name":"remote bytes read","Update":"178398","Value":"178398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2631,"Name":"local bytes read","Update":"60083","Value":"60083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2632,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2633,"Name":"records read","Update":"6337","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2816,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2817,"Name":"peak memory","Update":"67371008","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2819,"Name":"time in aggregation build","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2820,"Name":"avg hash probe bucket list iters","Update":"1009","Value":"1009","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2821,"Name":"duration","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2824,"Name":"data size","Update":"9144","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2832,"Name":"shuffle bytes written","Update":"2252","Value":"2252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2833,"Name":"shuffle records written","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2834,"Name":"shuffle write time","Update":"174693","Value":"174693","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2835,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2836,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2526075,"Value":2526075,"Internal":true,"Count Failed Values":true},{"ID":2837,"Name":"internal.metrics.executorRunTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2838,"Name":"internal.metrics.executorCpuTime","Update":17188776,"Value":17188776,"Internal":true,"Count Failed Values":true},{"ID":2839,"Name":"internal.metrics.resultSize","Update":3168,"Value":3168,"Internal":true,"Count Failed Values":true},{"ID":2844,"Name":"internal.metrics.peakExecutionMemory","Update":67371008,"Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2848,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2849,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2850,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":178398,"Value":178398,"Internal":true,"Count Failed Values":true},{"ID":2851,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2852,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":60083,"Value":60083,"Internal":true,"Count Failed Values":true},{"ID":2853,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2854,"Name":"internal.metrics.shuffle.read.recordsRead","Update":6337,"Value":6337,"Internal":true,"Count Failed Values":true},{"ID":2855,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2252,"Value":2252,"Internal":true,"Count Failed Values":true},{"ID":2856,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":381,"Value":381,"Internal":true,"Count Failed Values":true},{"ID":2857,"Name":"internal.metrics.shuffle.write.writeTime","Update":174693,"Value":174693,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2526075,"Executor Run Time":18,"Executor CPU Time":17188776,"Peak Execution Memory":67371008,"Result Size":3168,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":15,"Local Blocks Fetched":5,"Fetch Wait Time":0,"Remote Bytes Read":178398,"Remote Bytes Read To Disk":0,"Local Bytes Read":60083,"Total Records Read":6337},"Shuffle Write Metrics":{"Shuffle Bytes Written":2252,"Shuffle Write Time":174693,"Shuffle Records Written":381},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":87,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":204,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10300\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[203],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":202,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_10294\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[192],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":203,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10299\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[202],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[86],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432722960,"Completion Time":1646432722999,"Accumulables":[{"ID":2627,"Name":"remote blocks read","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2628,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2629,"Name":"remote bytes read","Value":"178398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2631,"Name":"local bytes read","Value":"60083","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2632,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2633,"Name":"records read","Value":"6337","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2816,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2817,"Name":"peak memory","Value":"67371008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2819,"Name":"time in aggregation build","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2820,"Name":"avg hash probe bucket list iters","Value":"1009","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2821,"Name":"duration","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2824,"Name":"data size","Value":"9144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2832,"Name":"shuffle bytes written","Value":"2252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2833,"Name":"shuffle records written","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2834,"Name":"shuffle write time","Value":"174693","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2835,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":2836,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2526075,"Internal":true,"Count Failed Values":true},{"ID":2837,"Name":"internal.metrics.executorRunTime","Value":18,"Internal":true,"Count Failed Values":true},{"ID":2838,"Name":"internal.metrics.executorCpuTime","Value":17188776,"Internal":true,"Count Failed Values":true},{"ID":2839,"Name":"internal.metrics.resultSize","Value":3168,"Internal":true,"Count Failed Values":true},{"ID":2844,"Name":"internal.metrics.peakExecutionMemory","Value":67371008,"Internal":true,"Count Failed Values":true},{"ID":2848,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":15,"Internal":true,"Count Failed Values":true},{"ID":2849,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2850,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":178398,"Internal":true,"Count Failed Values":true},{"ID":2851,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2852,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":60083,"Internal":true,"Count Failed Values":true},{"ID":2853,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2854,"Name":"internal.metrics.shuffle.read.recordsRead","Value":6337,"Internal":true,"Count Failed Values":true},{"ID":2855,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2252,"Internal":true,"Count Failed Values":true},{"ID":2856,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":381,"Internal":true,"Count Failed Values":true},{"ID":2857,"Name":"internal.metrics.shuffle.write.writeTime","Value":174693,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":47,"Completion Time":1646432723001,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":48,"Submission Time":1646432723055,"Stage Infos":[{"Stage ID":88,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":20,"RDD Info":[{"RDD ID":192,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9228\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[191],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":189,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9223\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[188],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":184,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":185,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8749\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[184],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":186,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9220\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[185],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":190,"Name":"UnionRDD","Scope":"{\"id\":\"spark_plan_9224\",\"name\":\"Union\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[186,189],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":188,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[187],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":187,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8751\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":191,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9227\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[190],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":20,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":89,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":1,"RDD Info":[{"RDD ID":204,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10300\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[203],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":202,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_10294\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[192],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":203,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10299\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[202],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[88],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":90,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":206,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10637\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[205],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":205,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[204],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[89],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[88,89,90],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_10\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 14e5fd26-8741-4dac-93f4-01caf7c213e3)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"14e5fd26-8741-4dac-93f4-01caf7c213e3","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":90,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":206,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10637\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[205],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":205,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[204],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[89],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432723055,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"spark_plan_10\",\"name\":\"BroadcastQueryStage\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"broadcast exchange (runId 14e5fd26-8741-4dac-93f4-01caf7c213e3)","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"14e5fd26-8741-4dac-93f4-01caf7c213e3","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":90,"Stage Attempt ID":0,"Task Info":{"Task ID":389,"Index":0,"Attempt":0,"Launch Time":1646432723064,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":90,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":389,"Index":0,"Attempt":0,"Launch Time":1646432723064,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432723084,"Failed":false,"Killed":false,"Accumulables":[{"ID":2825,"Name":"remote blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2827,"Name":"remote bytes read","Update":"2252","Value":"2252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2830,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2831,"Name":"records read","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2865,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2866,"Name":"peak memory","Update":"67174400","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2867,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2868,"Name":"cumulative time","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2869,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2870,"Name":"num prefix comparisons","Update":"2797","Value":"2797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2872,"Name":"number of output rows","Update":"381","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2873,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2874,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2875,"Name":"internal.metrics.executorDeserializeCpuTime","Update":2354781,"Value":2354781,"Internal":true,"Count Failed Values":true},{"ID":2876,"Name":"internal.metrics.executorRunTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2877,"Name":"internal.metrics.executorCpuTime","Update":5219690,"Value":5219690,"Internal":true,"Count Failed Values":true},{"ID":2878,"Name":"internal.metrics.resultSize","Update":8045,"Value":8045,"Internal":true,"Count Failed Values":true},{"ID":2883,"Name":"internal.metrics.peakExecutionMemory","Update":67174400,"Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2887,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2888,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2889,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":2252,"Value":2252,"Internal":true,"Count Failed Values":true},{"ID":2890,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2891,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2892,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2893,"Name":"internal.metrics.shuffle.read.recordsRead","Update":381,"Value":381,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":2354781,"Executor Run Time":6,"Executor CPU Time":5219690,"Peak Execution Memory":67174400,"Result Size":8045,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":1,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":2252,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":381},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":90,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Number of Tasks":1,"RDD Info":[{"RDD ID":206,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10637\",\"name\":\"WholeStageCodegen\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at CompletableFuture.java:1604","Parent IDs":[205],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":205,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_7\",\"name\":\"ShuffleQueryStage\"}","Callsite":"inputRDDs at RunningWindowFunctionExec.scala:218","Parent IDs":[204],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[89],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:330)\njava.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.$anonfun$run$1(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\nscala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:68)\norg.apache.spark.util.threads.SparkThreadLocalCapturingHelper.runWithCaptured$(SparkThreadLocalForwardingThreadPoolExecutor.scala:54)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.runWithCaptured(SparkThreadLocalForwardingThreadPoolExecutor.scala:101)\norg.apache.spark.util.threads.SparkThreadLocalCapturingRunnable.run(SparkThreadLocalForwardingThreadPoolExecutor.scala:104)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432723055,"Completion Time":1646432723085,"Accumulables":[{"ID":2825,"Name":"remote blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2827,"Name":"remote bytes read","Value":"2252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2830,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2831,"Name":"records read","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2865,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2866,"Name":"peak memory","Value":"67174400","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2867,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2868,"Name":"cumulative time","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2869,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2870,"Name":"num prefix comparisons","Value":"2797","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2872,"Name":"number of output rows","Value":"381","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2873,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2874,"Name":"internal.metrics.executorDeserializeTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":2875,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2354781,"Internal":true,"Count Failed Values":true},{"ID":2876,"Name":"internal.metrics.executorRunTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":2877,"Name":"internal.metrics.executorCpuTime","Value":5219690,"Internal":true,"Count Failed Values":true},{"ID":2878,"Name":"internal.metrics.resultSize","Value":8045,"Internal":true,"Count Failed Values":true},{"ID":2883,"Name":"internal.metrics.peakExecutionMemory","Value":67174400,"Internal":true,"Count Failed Values":true},{"ID":2887,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2888,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2889,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":2252,"Internal":true,"Count Failed Values":true},{"ID":2890,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2891,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2892,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2893,"Name":"internal.metrics.shuffle.read.recordsRead","Value":381,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":48,"Completion Time":1646432723086,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":355,"Index":9,"Attempt":0,"Launch Time":1646432702616,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432724284,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"5617658","Value":"23809046","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"21569","Value":"81038","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"5617658","Value":"23809046","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"1392","Value":"5607","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"175932825","Value":"736000649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"175932825","Value":"736000649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"539295168","Value":"2285668416","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"74785065","Value":"315166189","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"5617658","Value":"23809046","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"187519138","Value":"784221745","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1488136,"Value":6288796,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":21653,"Value":81391,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":21371804061,"Value":80532714909,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1703,"Value":6811,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":164,"Value":358,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":74785065,"Value":315166189,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5617658,"Value":23809046,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":187519138,"Value":784221745,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":175932825,"Value":736000649,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":5617658,"Value":23809046,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1392547829,"Value":5608921176,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":175932825,"Value":736000649,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10194255400,"JVMOffHeapMemory":144555296,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":95,"MinorGCTime":1422,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1488136,"Executor Run Time":21653,"Executor CPU Time":21371804061,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":164,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":74785065,"Shuffle Write Time":187519138,"Shuffle Records Written":5617658},"Input Metrics":{"Bytes Read":175932825,"Records Read":5617658},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":352,"Index":6,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432724327,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"5819079","Value":"29628125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"21605","Value":"102643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"5819079","Value":"29628125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"1531","Value":"7138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"182236865","Value":"918237514","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"182236865","Value":"918237514","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"558631584","Value":"2844300000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"77834569","Value":"393000758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"5819079","Value":"29628125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"222780429","Value":"1007002174","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1780759,"Value":8069555,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":21696,"Value":103087,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":21541757110,"Value":102074472019,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1702,"Value":8513,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":56,"Value":414,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":77834569,"Value":393000758,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819079,"Value":29628125,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":222780429,"Value":1007002174,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":182236865,"Value":918237514,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":5819079,"Value":29628125,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1531279459,"Value":7140200635,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":182236865,"Value":918237514,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10810309800,"JVMOffHeapMemory":145041432,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1552270,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":96,"MinorGCTime":1747,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1780759,"Executor Run Time":21696,"Executor CPU Time":21541757110,"Peak Execution Memory":0,"Result Size":1702,"JVM GC Time":56,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":77834569,"Shuffle Write Time":222780429,"Shuffle Records Written":5819079},"Input Metrics":{"Bytes Read":182236865,"Records Read":5819079},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":353,"Index":7,"Attempt":0,"Launch Time":1646432702616,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432724446,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"5819811","Value":"35447936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"21731","Value":"124374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"5819811","Value":"35447936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"1440","Value":"8578","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"180693843","Value":"1098931357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"180693843","Value":"1098931357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"558701856","Value":"3403001856","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"78151025","Value":"471151783","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"5819811","Value":"35447936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"193974430","Value":"1200976604","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1565508,"Value":9635063,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":21815,"Value":124902,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":21647524608,"Value":123721996627,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1703,"Value":10216,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":461,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":78151025,"Value":471151783,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5819811,"Value":35447936,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":193974430,"Value":1200976604,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":180693843,"Value":1098931357,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":5819811,"Value":35447936,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1440091473,"Value":8580292108,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":180693843,"Value":1098931357,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6458964792,"JVMOffHeapMemory":147555256,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":104,"MinorGCTime":2416,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1565508,"Executor Run Time":21815,"Executor CPU Time":21647524608,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":78151025,"Shuffle Write Time":193974430,"Shuffle Records Written":5819811},"Input Metrics":{"Bytes Read":180693843,"Records Read":5819811},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":351,"Index":5,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432724454,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"6096762","Value":"41544698","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"21736","Value":"146110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"6096762","Value":"41544698","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"1379","Value":"9957","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"183939508","Value":"1282870865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"183939508","Value":"1282870865","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"585289152","Value":"3988291008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"80392000","Value":"551543783","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"6096762","Value":"41544698","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"192504242","Value":"1393480846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1486871,"Value":11121934,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":21823,"Value":146725,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":21555663283,"Value":145277659910,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1703,"Value":11919,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":164,"Value":625,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":80392000,"Value":551543783,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6096762,"Value":41544698,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":192504242,"Value":1393480846,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":183939508,"Value":1282870865,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":6096762,"Value":41544698,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1379943704,"Value":9960235812,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":183939508,"Value":1282870865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10194255400,"JVMOffHeapMemory":144555296,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":95,"MinorGCTime":1422,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1486871,"Executor Run Time":21823,"Executor CPU Time":21555663283,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":164,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":80392000,"Shuffle Write Time":192504242,"Shuffle Records Written":6096762},"Input Metrics":{"Bytes Read":183939508,"Records Read":6096762},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":347,"Index":1,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432724560,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"6369482","Value":"47914180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"21841","Value":"167951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"6369482","Value":"47914180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"1753","Value":"11710","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"197638534","Value":"1480509399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"197638534","Value":"1480509399","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"611470272","Value":"4599761280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"85250127","Value":"636793910","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"6369482","Value":"47914180","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"199587611","Value":"1593068457","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1480211,"Value":12602145,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":21930,"Value":168655,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":21437091694,"Value":166714751604,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1703,"Value":13622,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":164,"Value":789,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85250127,"Value":636793910,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6369482,"Value":47914180,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":199587611,"Value":1593068457,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":197638534,"Value":1480509399,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":6369482,"Value":47914180,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1753759989,"Value":11713995801,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":197638534,"Value":1480509399,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":10194255400,"JVMOffHeapMemory":144555296,"OnHeapExecutionMemory":471597056,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":749776308,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1485237,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":95,"MinorGCTime":1422,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1480211,"Executor Run Time":21930,"Executor CPU Time":21437091694,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":164,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85250127,"Shuffle Write Time":199587611,"Shuffle Records Written":6369482},"Input Metrics":{"Bytes Read":197638534,"Records Read":6369482},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":349,"Index":3,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432725887,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"6429338","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"23167","Value":"191118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"6429338","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"1778","Value":"13488","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"195074315","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"195074315","Value":"1675583714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"617216448","Value":"5216977728","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"83388371","Value":"720182281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"6429338","Value":"54343518","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"204433817","Value":"1797502274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":2,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1781391,"Value":14383536,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":23255,"Value":191910,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":22891368096,"Value":189606119700,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1703,"Value":15325,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":836,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83388371,"Value":720182281,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":6429338,"Value":54343518,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":204433817,"Value":1797502274,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":195074315,"Value":1675583714,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":6429338,"Value":54343518,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":1778221380,"Value":13492217181,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":195074315,"Value":1675583714,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":6458964792,"JVMOffHeapMemory":147555256,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1653124,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":104,"MinorGCTime":2416,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":2,"Executor Deserialize CPU Time":1781391,"Executor Run Time":23255,"Executor CPU Time":22891368096,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83388371,"Shuffle Write Time":204433817,"Shuffle Records Written":6429338},"Input Metrics":{"Bytes Read":195074315,"Records Read":6429338},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":78,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":346,"Index":0,"Attempt":0,"Launch Time":1646432702615,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432727974,"Failed":false,"Killed":false,"Accumulables":[{"ID":2550,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Update":"25243","Value":"216361","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Update":"2005","Value":"15493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Update":"228606620","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Update":"692490816","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Update":"98606155","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Update":"7213446","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Update":"226235175","Value":"2023737449","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Update":1,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1512546,"Value":15896082,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Update":25343,"Value":217253,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Update":24908691744,"Value":214514811444,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Update":1703,"Value":17028,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Update":69,"Value":905,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":98606155,"Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":7213446,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Update":226235175,"Value":2023737449,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Update":7213446,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Update":2005992998,"Value":15498210179,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Update":228606620,"Value":1904190334,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17666779848,"JVMOffHeapMemory":146350128,"OnHeapExecutionMemory":538968064,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":278179252,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":817147316,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":102,"MinorGCTime":1544,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":1,"Executor Deserialize CPU Time":1512546,"Executor Run Time":25343,"Executor CPU Time":24908691744,"Peak Execution Memory":0,"Result Size":1703,"JVM GC Time":69,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":98606155,"Shuffle Write Time":226235175,"Shuffle Records Written":7213446},"Input Metrics":{"Bytes Read":228606620,"Records Read":7213446},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":78,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":183,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9208\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[182],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":182,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9207\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[181],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":181,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8746\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[180],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":180,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8746\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432702610,"Completion Time":1646432727974,"Accumulables":[{"ID":2550,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"duration","Value":"216361","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2552,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2556,"Name":"filesystem read time (sampled)","Value":"15493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2557,"Name":"filesystem read data size (sampled)","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2558,"Name":"filesystem read data size","Value":"1904190334","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2560,"Name":"data size","Value":"5909468544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2568,"Name":"shuffle bytes written","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2569,"Name":"shuffle records written","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2570,"Name":"shuffle write time","Value":"2023737449","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2577,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.executorDeserializeCpuTime","Value":15896082,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.executorRunTime","Value":217253,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.executorCpuTime","Value":214514811444,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.resultSize","Value":17028,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.jvmGCTime","Value":905,"Internal":true,"Count Failed Values":true},{"ID":2597,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2598,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2599,"Name":"internal.metrics.shuffle.write.writeTime","Value":2023737449,"Internal":true,"Count Failed Values":true},{"ID":2600,"Name":"internal.metrics.input.bytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true},{"ID":2601,"Name":"internal.metrics.input.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2602,"Name":"internal.metrics.input.sampledTimeReadNano","Value":15498210179,"Internal":true,"Count Failed Values":true},{"ID":2603,"Name":"internal.metrics.input.sampledBytesRead","Value":1904190334,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":42,"Completion Time":1646432727974,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":49,"Submission Time":1646432728083,"Stage Infos":[{"Stage ID":93,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":6,"RDD Info":[{"RDD ID":217,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[216],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":216,"Name":"ShuffledRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[215],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[92],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":91,"Stage Attempt ID":0,"Stage Name":"rdd at LinearRegression.scala:912","Number of Tasks":10,"RDD Info":[{"RDD ID":183,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9208\",\"name\":\"Exchange\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[182],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":182,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_9207\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[181],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":181,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_8746\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[180],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":180,"Name":"FileScanRDD","Scope":"{\"id\":\"spark_plan_8746\",\"name\":\"Scan csv \"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.rdd(Dataset.scala:3331)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:912)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":92,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":50,"RDD Info":[{"RDD ID":215,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[214],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":212,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"352\",\"name\":\"map\"}","Callsite":"map at LinearRegression.scala:914","Parent IDs":[211],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":209,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10952\",\"name\":\"DeserializeToObject\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[208],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":208,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10951\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[207],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":210,"Name":"SQLExecutionRDD","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[209],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":207,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_10925\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[183],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":214,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[213],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":211,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"351\",\"name\":\"mapPartitions\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[210],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":213,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"353\",\"name\":\"map\"}","Callsite":"map at RegressionMetrics.scala:63","Parent IDs":[212],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[91],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[93,91,92],"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":92,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":50,"RDD Info":[{"RDD ID":215,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[214],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":212,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"352\",\"name\":\"map\"}","Callsite":"map at LinearRegression.scala:914","Parent IDs":[211],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":209,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10952\",\"name\":\"DeserializeToObject\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[208],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":208,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10951\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[207],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":210,"Name":"SQLExecutionRDD","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[209],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":207,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_10925\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[183],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":214,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[213],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":211,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"351\",\"name\":\"mapPartitions\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[210],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":213,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"353\",\"name\":\"map\"}","Callsite":"map at RegressionMetrics.scala:63","Parent IDs":[212],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[91],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432728088,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":390,"Index":0,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":391,"Index":10,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":392,"Index":15,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":393,"Index":5,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":394,"Index":1,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":395,"Index":11,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":396,"Index":16,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":397,"Index":6,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":398,"Index":2,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":399,"Index":12,"Attempt":0,"Launch Time":1646432728095,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":400,"Index":17,"Attempt":0,"Launch Time":1646432728095,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":401,"Index":7,"Attempt":0,"Launch Time":1646432728095,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":402,"Index":3,"Attempt":0,"Launch Time":1646432728096,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":403,"Index":13,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":404,"Index":18,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":405,"Index":8,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":406,"Index":4,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":407,"Index":14,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":408,"Index":19,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":409,"Index":9,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":410,"Index":20,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":411,"Index":30,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":412,"Index":35,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":413,"Index":25,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":414,"Index":21,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":415,"Index":31,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":416,"Index":36,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":417,"Index":26,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":418,"Index":22,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":419,"Index":32,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":420,"Index":37,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":421,"Index":27,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":422,"Index":23,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":423,"Index":33,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":424,"Index":38,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":425,"Index":28,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":426,"Index":24,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":427,"Index":34,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":428,"Index":39,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":429,"Index":29,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":430,"Index":40,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":431,"Index":45,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":432,"Index":41,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":433,"Index":46,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":434,"Index":42,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":435,"Index":47,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":436,"Index":43,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":437,"Index":48,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":438,"Index":44,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":92,"Stage Attempt ID":0,"Task Info":{"Task ID":439,"Index":49,"Attempt":0,"Launch Time":1646432728100,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":392,"Index":15,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432730060,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"5549390","Value":"5549390","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"443853","Value":"443853","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"443853","Value":"443853","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"443853","Value":"443853","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"436895","Value":"436895","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"247","Value":"247","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"150994944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"1741","Value":"1741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"2885625","Value":"2885625","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"2866006","Value":"2866006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"130559","Value":"130559","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"1884","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9447193,"Value":9447193,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":1932,"Value":1932,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":1850727787,"Value":1850727787,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":8189,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":352357648,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5549390,"Value":5549390,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":443853,"Value":443853,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":824,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":49213,"Value":49213,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":9447193,"Executor Run Time":1932,"Executor CPU Time":1850727787,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":23,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5549390,"Total Records Read":443853},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":49213,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":411,"Index":30,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432730212,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"5197262","Value":"10746652","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"403027","Value":"846880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"403027","Value":"846880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"3","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"403027","Value":"846880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"395697","Value":"832592","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"304","Value":"551","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"301989888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"1834","Value":"3575","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"2671694","Value":"5557319","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"2648851","Value":"5514857","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"118544","Value":"249103","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"2017","Value":"3901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8244770,"Value":17691963,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":2086,"Value":4018,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":1989927409,"Value":3840655196,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":16377,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":11,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":704715296,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5197262,"Value":10746652,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":403027,"Value":846880,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":1647,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":76053,"Value":125266,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":8244770,"Executor Run Time":2086,"Executor CPU Time":1989927409,"Peak Execution Memory":352357648,"Result Size":8188,"JVM GC Time":11,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5197262,"Total Records Read":403027},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":76053,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":391,"Index":10,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432730286,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"5569222","Value":"16315874","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"436341","Value":"1283221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"436341","Value":"1283221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"4","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"436341","Value":"1283221","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"426737","Value":"1259329","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"205","Value":"756","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"452984832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"1950","Value":"5525","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"2806335","Value":"8363654","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"2785368","Value":"8300225","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"127729","Value":"376832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"2091","Value":"5992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8451380,"Value":26143343,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":2156,"Value":6174,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":2064336251,"Value":5904991447,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":24565,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":11,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":1057072944,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5569222,"Value":16315874,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":436341,"Value":1283221,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":825,"Value":2472,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":54025,"Value":179291,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":8451380,"Executor Run Time":2156,"Executor CPU Time":2064336251,"Peak Execution Memory":352357648,"Result Size":8188,"JVM GC Time":11,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5569222,"Total Records Read":436341},"Shuffle Write Metrics":{"Shuffle Bytes Written":825,"Shuffle Write Time":54025,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":413,"Index":25,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432730309,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"5062619","Value":"21378493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"397041","Value":"1680262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"397041","Value":"1680262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"15","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"397041","Value":"1680262","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"390517","Value":"1649846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"285","Value":"1041","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"603979776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"1926","Value":"7451","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"2548570","Value":"10912224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"2526559","Value":"10826784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"117080","Value":"493912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"2108","Value":"8100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7605897,"Value":33749240,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":2181,"Value":8355,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":2031454234,"Value":7936445681,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":32754,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":11,"Value":56,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":1409430592,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5062619,"Value":21378493,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":397041,"Value":1680262,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":3295,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":79928,"Value":259219,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":7605897,"Executor Run Time":2181,"Executor CPU Time":2031454234,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":11,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5062619,"Total Records Read":397041},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":79928,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":412,"Index":35,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432730502,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"5007038","Value":"26385531","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"385946","Value":"2066208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"385946","Value":"2066208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"3","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"385946","Value":"2066208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"376002","Value":"2025848","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"200","Value":"1241","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"754974720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"2196","Value":"9647","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"2450583","Value":"13362807","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"2429287","Value":"13256071","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"112985","Value":"606897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"2326","Value":"10426","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8863291,"Value":42612531,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":2374,"Value":10729,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":2280730808,"Value":10217176489,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":40943,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":23,"Value":79,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":1761788240,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5007038,"Value":26385531,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":385946,"Value":2066208,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":818,"Value":4113,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":53276,"Value":312495,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":8863291,"Executor Run Time":2374,"Executor CPU Time":2280730808,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":23,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5007038,"Total Records Read":385946},"Shuffle Write Metrics":{"Shuffle Bytes Written":818,"Shuffle Write Time":53276,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":393,"Index":5,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432730593,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"5506047","Value":"31891578","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"425591","Value":"2491799","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"425591","Value":"2491799","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"8","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"425591","Value":"2491799","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"418065","Value":"2443913","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"7","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"212","Value":"1453","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"905969664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"2253","Value":"11900","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"2784193","Value":"16147000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"2761418","Value":"16017489","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"125668","Value":"732565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"2392","Value":"12818","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":65,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8784194,"Value":51396725,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":2463,"Value":13192,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":2354749306,"Value":12571925795,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":49132,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":28,"Value":107,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":2114145888,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5506047,"Value":31891578,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":425591,"Value":2491799,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":4935,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":50518,"Value":363013,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8784194,"Executor Run Time":2463,"Executor CPU Time":2354749306,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":28,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5506047,"Total Records Read":425591},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":50518,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":431,"Index":45,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432730604,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"4974479","Value":"36866057","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"387227","Value":"2879026","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"387227","Value":"2879026","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"14","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"387227","Value":"2879026","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"381516","Value":"2825429","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"271","Value":"1724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"1056964608","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"2205","Value":"14105","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"2458158","Value":"18605158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"2436258","Value":"18453747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"114315","Value":"846880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"2404","Value":"15222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":15,"Value":80,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7636436,"Value":59033161,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":2467,"Value":15659,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":2322297185,"Value":14894222980,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":57321,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":28,"Value":135,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":2466503536,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":4974479,"Value":36866057,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":387227,"Value":2879026,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":5759,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":51670,"Value":414683,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":15,"Executor Deserialize CPU Time":7636436,"Executor Run Time":2467,"Executor CPU Time":2322297185,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":28,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":4974479,"Total Records Read":387227},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":51670,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":410,"Index":20,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432730795,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"5077427","Value":"41943484","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"397065","Value":"3276091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"397065","Value":"3276091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"1","Value":"49","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"397065","Value":"3276091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"388533","Value":"3213962","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"270","Value":"1994","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"1207959552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"2407","Value":"16512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"2526318","Value":"21131476","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"2507081","Value":"20960828","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"116531","Value":"963411","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"2608","Value":"17830","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":91,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8563802,"Value":67596963,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":2659,"Value":18318,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":2560551244,"Value":17454774224,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":65510,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":147,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":2818861184,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5077427,"Value":41943484,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":397065,"Value":3276091,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":825,"Value":6584,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":59939,"Value":474622,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8563802,"Executor Run Time":2659,"Executor CPU Time":2560551244,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":12,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5077427,"Total Records Read":397065},"Shuffle Write Metrics":{"Shuffle Bytes Written":825,"Shuffle Write Time":59939,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":430,"Index":40,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432730833,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"5138578","Value":"47082062","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"398664","Value":"3674755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"398664","Value":"3674755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"1","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"398664","Value":"3674755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"392344","Value":"3606306","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"49","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"281","Value":"2275","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"1358954496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"2442","Value":"18954","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"2612328","Value":"23743804","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"2589790","Value":"23550618","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"117338","Value":"1080749","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"2646","Value":"20476","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":101,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10133737,"Value":77730700,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":2699,"Value":21017,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":2549123517,"Value":20003897741,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":73699,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":159,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":3171218832,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":5138578,"Value":47082062,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":398664,"Value":3674755,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":825,"Value":7409,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":74482,"Value":549104,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":10133737,"Executor Run Time":2699,"Executor CPU Time":2549123517,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":12,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":5138578,"Total Records Read":398664},"Shuffle Write Metrics":{"Shuffle Bytes Written":825,"Shuffle Write Time":74482,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":390,"Index":0,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432731485,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"7603473","Value":"54685535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"566756","Value":"4241511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"566756","Value":"4241511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"8","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"566756","Value":"4241511","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"555159","Value":"4161465","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"53","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"277","Value":"2552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"150994944","Value":"1509949440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"3132","Value":"22086","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"3939232","Value":"27683036","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"3914157","Value":"27464775","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"166389","Value":"1247138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"3299","Value":"23775","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":112,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8523446,"Value":86254146,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":3349,"Value":24366,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":3098321581,"Value":23102219322,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":81888,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":30,"Value":189,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":352357648,"Value":3523576480,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":7603473,"Value":54685535,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":566756,"Value":4241511,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":8232,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":54866,"Value":603970,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8523446,"Executor Run Time":3349,"Executor CPU Time":3098321581,"Peak Execution Memory":352357648,"Result Size":8189,"JVM GC Time":30,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":7603473,"Total Records Read":566756},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":54866,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":415,"Index":31,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432732445,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"14179385","Value":"68864920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1069599","Value":"5311110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1069599","Value":"5311110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"4","Value":"62","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1069599","Value":"5311110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1050288","Value":"5211753","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"547","Value":"3099","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"1811939328","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"3981","Value":"26067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7084874","Value":"34767910","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7053239","Value":"34518014","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"315323","Value":"1562461","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"4249","Value":"28024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":122,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9196523,"Value":95450669,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":4317,"Value":28683,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":4169321458,"Value":27271540780,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":90076,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":226,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":4026929072,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14179385,"Value":68864920,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1069599,"Value":5311110,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":9054,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":57319,"Value":661289,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":9196523,"Executor Run Time":4317,"Executor CPU Time":4169321458,"Peak Execution Memory":503352592,"Result Size":8188,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14179385,"Total Records Read":1069599},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":57319,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":396,"Index":16,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432732505,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"15206714","Value":"84071634","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1185888","Value":"6496998","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1185888","Value":"6496998","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"5","Value":"67","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1185888","Value":"6496998","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1167901","Value":"6379654","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"625","Value":"3724","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"2113929216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"4000","Value":"30067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7540656","Value":"42308566","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7513451","Value":"42031465","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"350314","Value":"1912775","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"4330","Value":"32354","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":132,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8906090,"Value":104356759,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":4377,"Value":33060,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":4302385855,"Value":31573926635,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":98265,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":44,"Value":270,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":4530281664,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":15206714,"Value":84071634,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1185888,"Value":6496998,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":9877,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":47602,"Value":708891,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3683160272,"JVMOffHeapMemory":148981192,"OnHeapExecutionMemory":2013265920,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2286781017,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1661300,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":2460,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":8906090,"Executor Run Time":4377,"Executor CPU Time":4302385855,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":44,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":15206714,"Total Records Read":1185888},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":47602,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":395,"Index":11,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432732687,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"15063284","Value":"99134918","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1163966","Value":"7660964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1163966","Value":"7660964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"4","Value":"71","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1163966","Value":"7660964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1143328","Value":"7522982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"531","Value":"4255","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"2415919104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"4210","Value":"34277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7298870","Value":"49607436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7269686","Value":"49301151","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"343025","Value":"2255800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"4492","Value":"36846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":142,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9993557,"Value":114350316,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":4560,"Value":37620,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":4437605605,"Value":36011532240,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":106453,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":307,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":5033634256,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":15063284,"Value":99134918,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1163966,"Value":7660964,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":819,"Value":10696,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":52919,"Value":761810,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":9993557,"Executor Run Time":4560,"Executor CPU Time":4437605605,"Peak Execution Memory":503352592,"Result Size":8188,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":15063284,"Total Records Read":1163966},"Shuffle Write Metrics":{"Shuffle Bytes Written":819,"Shuffle Write Time":52919,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":399,"Index":12,"Attempt":0,"Launch Time":1646432728095,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432732731,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"17852704","Value":"116987622","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1354716","Value":"9015680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1354716","Value":"9015680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"5","Value":"76","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1354716","Value":"9015680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1335669","Value":"8858651","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"76","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"590","Value":"4845","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"2717908992","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"4222","Value":"38499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"8042908","Value":"57650344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"8012151","Value":"57313302","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"400822","Value":"2656622","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"4537","Value":"41383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8344253,"Value":122694569,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":4603,"Value":42223,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":4482400850,"Value":40493933090,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":114641,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":344,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":5536986848,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17852704,"Value":116987622,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1354716,"Value":9015680,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":11518,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":50753,"Value":812563,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":8344253,"Executor Run Time":4603,"Executor CPU Time":4482400850,"Peak Execution Memory":503352592,"Result Size":8188,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17852704,"Total Records Read":1354716},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":50753,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":420,"Index":37,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432732782,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"16812061","Value":"133799683","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1252080","Value":"10267760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1252080","Value":"10267760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"5","Value":"81","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1252080","Value":"10267760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1230187","Value":"10088838","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"649","Value":"5494","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"3019898880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"4163","Value":"42662","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7918049","Value":"65568393","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7886574","Value":"65199876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"368974","Value":"3025596","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"4606","Value":"45989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":163,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8203873,"Value":130898442,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":4655,"Value":46878,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":4457494511,"Value":44951427601,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":122830,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":122,"Value":466,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":6040339440,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16812061,"Value":133799683,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1252080,"Value":10267760,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":12341,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":49628,"Value":862191,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3683160272,"JVMOffHeapMemory":148981192,"OnHeapExecutionMemory":2013265920,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2286781017,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1661300,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":2460,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":8203873,"Executor Run Time":4655,"Executor CPU Time":4457494511,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":122,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16812061,"Total Records Read":1252080},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":49628,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":419,"Index":32,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432732789,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"16283262","Value":"150082945","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1222489","Value":"11490249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1222489","Value":"11490249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"6","Value":"87","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1222489","Value":"11490249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1207733","Value":"11296571","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"85","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"561","Value":"6055","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"3321888768","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"4302","Value":"46964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7750719","Value":"73319112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7719792","Value":"72919668","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"362810","Value":"3388406","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"4594","Value":"50583","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":175,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8742687,"Value":139641129,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":4661,"Value":51539,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":4509119007,"Value":49460546608,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":131018,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":503,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":6543692032,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16283262,"Value":150082945,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1222489,"Value":11490249,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":13164,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":52127,"Value":914318,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":8742687,"Executor Run Time":4661,"Executor CPU Time":4509119007,"Peak Execution Memory":503352592,"Result Size":8188,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16283262,"Total Records Read":1222489},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":52127,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":416,"Index":36,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432732936,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"14286711","Value":"164369656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1075303","Value":"12565552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1075303","Value":"12565552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"1","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1075303","Value":"12565552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1050388","Value":"12346959","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"89","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"654","Value":"6709","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"3623878656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"4453","Value":"51417","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7028804","Value":"80347916","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"6997233","Value":"79916901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"314974","Value":"3703380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"4761","Value":"55344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":185,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8141288,"Value":147782417,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":4809,"Value":56348,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":4612111195,"Value":54072657803,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":139207,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":122,"Value":625,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":7047044624,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14286711,"Value":164369656,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1075303,"Value":12565552,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":13986,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":50259,"Value":964577,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3683160272,"JVMOffHeapMemory":148981192,"OnHeapExecutionMemory":2013265920,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2286781017,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1661300,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":2460,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":8141288,"Executor Run Time":4809,"Executor CPU Time":4612111195,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":122,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14286711,"Total Records Read":1075303},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":50259,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":400,"Index":17,"Attempt":0,"Launch Time":1646432728095,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432732980,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"17279386","Value":"181649042","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1325325","Value":"13890877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1325325","Value":"13890877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"5","Value":"93","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1325325","Value":"13890877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1311802","Value":"13658761","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"93","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"666","Value":"7375","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"3925868544","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"4356","Value":"55773","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"8008126","Value":"88356042","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7980301","Value":"87897202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"392285","Value":"4095665","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"4805","Value":"60149","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":195,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9129581,"Value":156911998,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":4853,"Value":61201,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":4654385670,"Value":58727043473,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":147396,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":122,"Value":747,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":7550397216,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17279386,"Value":181649042,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1325325,"Value":13890877,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":820,"Value":14806,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":48407,"Value":1012984,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3683160272,"JVMOffHeapMemory":148981192,"OnHeapExecutionMemory":2013265920,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2286781017,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1661300,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":2460,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":9129581,"Executor Run Time":4853,"Executor CPU Time":4654385670,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":122,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17279386,"Total Records Read":1325325},"Shuffle Write Metrics":{"Shuffle Bytes Written":820,"Shuffle Write Time":48407,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":414,"Index":21,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432733225,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"14140207","Value":"195789249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1091142","Value":"14982019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1091142","Value":"14982019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"0","Value":"93","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1091142","Value":"14982019","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1068266","Value":"14727027","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"97","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"472","Value":"7847","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"4227858432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"4752","Value":"60525","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"6841900","Value":"95197942","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"6814278","Value":"94711480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"320767","Value":"4416432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"5039","Value":"65188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":206,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8261557,"Value":165173555,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":5089,"Value":66290,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":4917223015,"Value":63644266488,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":155585,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":38,"Value":785,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":8053749808,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14140207,"Value":195789249,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1091142,"Value":14982019,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":15629,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":55554,"Value":1068538,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8261557,"Executor Run Time":5089,"Executor CPU Time":4917223015,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":38,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14140207,"Total Records Read":1091142},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":55554,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":427,"Index":34,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432733610,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"20325382","Value":"216114631","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1507900","Value":"16489919","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1507900","Value":"16489919","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"5","Value":"98","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1507900","Value":"16489919","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1488908","Value":"16215935","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"103","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"705","Value":"8552","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"4630511616","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5059","Value":"65584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"10001165","Value":"105199107","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"9964616","Value":"104676096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"446743","Value":"4863175","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"5415","Value":"70603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":217,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8931413,"Value":174104968,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":5483,"Value":71773,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":5364674014,"Value":69008940502,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":163773,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":822,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":8657765696,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20325382,"Value":216114631,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1507900,"Value":16489919,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":16453,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":66845,"Value":1135383,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8931413,"Executor Run Time":5483,"Executor CPU Time":5364674014,"Peak Execution Memory":604015888,"Result Size":8188,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20325382,"Total Records Read":1507900},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":66845,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":423,"Index":33,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432733722,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"21849278","Value":"237963909","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1616064","Value":"18105983","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1616064","Value":"18105983","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"6","Value":"104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1616064","Value":"18105983","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1590300","Value":"17806235","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"108","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"771","Value":"9323","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"5033164800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5144","Value":"70728","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"11739245","Value":"116938352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"11710528","Value":"116386624","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"476596","Value":"5339771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"5527","Value":"76130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":228,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9083675,"Value":183188643,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":5594,"Value":77367,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":5523294117,"Value":74532234619,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":171961,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":859,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":9261781584,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21849278,"Value":237963909,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1616064,"Value":18105983,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":17276,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":51654,"Value":1187037,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":9083675,"Executor Run Time":5594,"Executor CPU Time":5523294117,"Peak Execution Memory":604015888,"Result Size":8188,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21849278,"Total Records Read":1616064},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":51654,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":424,"Index":38,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432733938,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"21789512","Value":"259753421","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1603400","Value":"19709383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1603400","Value":"19709383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"5","Value":"109","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1603400","Value":"19709383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1564114","Value":"19370349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"951","Value":"10274","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"5435817984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5314","Value":"76042","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"11511301","Value":"128449653","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"11482858","Value":"127869482","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"468047","Value":"5807818","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"5763","Value":"81893","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":238,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8188639,"Value":191377282,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":5811,"Value":83178,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":5615401052,"Value":80147635671,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":180150,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":122,"Value":981,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":9865797472,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21789512,"Value":259753421,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1603400,"Value":19709383,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":18100,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":91563,"Value":1278600,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3683160272,"JVMOffHeapMemory":148981192,"OnHeapExecutionMemory":2013265920,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2286781017,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1661300,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":2460,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":8188639,"Executor Run Time":5811,"Executor CPU Time":5615401052,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":122,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21789512,"Total Records Read":1603400},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":91563,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":407,"Index":14,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734097,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"22521225","Value":"282274646","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1704079","Value":"21413462","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1704079","Value":"21413462","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"4","Value":"113","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1704079","Value":"21413462","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1676732","Value":"21047081","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"764","Value":"11038","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"5838471168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5508","Value":"81550","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"10756664","Value":"139206317","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"10719584","Value":"138589066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"502476","Value":"6310294","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"5902","Value":"87795","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":249,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9137245,"Value":200514527,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":5969,"Value":89147,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":5846975068,"Value":85994610739,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":188338,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":1018,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":10469813360,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22521225,"Value":282274646,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1704079,"Value":21413462,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":18924,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":53888,"Value":1332488,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14979502832,"JVMOffHeapMemory":146808936,"OnHeapExecutionMemory":872415232,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1145930329,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1556358,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":99,"MinorGCTime":1784,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":9137245,"Executor Run Time":5969,"Executor CPU Time":5846975068,"Peak Execution Memory":604015888,"Result Size":8188,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22521225,"Total Records Read":1704079},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":53888,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":417,"Index":26,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734112,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"14350655","Value":"296625301","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1106404","Value":"22519866","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1106404","Value":"22519866","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"8","Value":"121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1106404","Value":"22519866","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1090201","Value":"22137282","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"7","Value":"124","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"801","Value":"11839","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"6140461056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5528","Value":"87078","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7013679","Value":"146219996","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"6983027","Value":"145572093","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"327203","Value":"6637497","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"5911","Value":"93706","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":267,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7730274,"Value":208244801,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":5972,"Value":95119,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":5724746425,"Value":91719357164,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":196527,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":100,"Value":1118,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":10973165952,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":14350655,"Value":296625301,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1106404,"Value":22519866,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":19746,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":57534,"Value":1390022,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":7730274,"Executor Run Time":5972,"Executor CPU Time":5724746425,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":100,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":14350655,"Total Records Read":1106404},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":57534,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":408,"Index":19,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734182,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"22027093","Value":"318652394","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1697367","Value":"24217233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1697367","Value":"24217233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"5","Value":"126","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1697367","Value":"24217233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1677876","Value":"23815158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"886","Value":"12725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"6543114240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5520","Value":"92598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"10836844","Value":"157056840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"10801236","Value":"156373329","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"502528","Value":"7140025","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"6007","Value":"99713","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":277,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8700711,"Value":216945512,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":6055,"Value":101174,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":5857208166,"Value":97576565330,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":204716,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":122,"Value":1240,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":11577181840,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22027093,"Value":318652394,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1697367,"Value":24217233,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":820,"Value":20566,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":49866,"Value":1439888,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3683160272,"JVMOffHeapMemory":148981192,"OnHeapExecutionMemory":2013265920,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2286781017,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1661300,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":2460,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":8700711,"Executor Run Time":6055,"Executor CPU Time":5857208166,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":122,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22027093,"Total Records Read":1697367},"Shuffle Write Metrics":{"Shuffle Bytes Written":820,"Shuffle Write Time":49866,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":428,"Index":39,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734308,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"26","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"20255703","Value":"338908097","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1503082","Value":"25720315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1503082","Value":"25720315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"5","Value":"131","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1503082","Value":"25720315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1476378","Value":"25291536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"799","Value":"13524","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"6945767424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5675","Value":"98273","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"9929413","Value":"166986253","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"9892652","Value":"166265981","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"442872","Value":"7582897","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"6133","Value":"105846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":287,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8797862,"Value":225743374,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":6180,"Value":107354,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":5985887015,"Value":103562452345,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":212905,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":122,"Value":1362,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":12181197728,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20255703,"Value":338908097,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1503082,"Value":25720315,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":21388,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":46694,"Value":1486582,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3683160272,"JVMOffHeapMemory":148981192,"OnHeapExecutionMemory":2013265920,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2286781017,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1661300,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":2460,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":8797862,"Executor Run Time":6180,"Executor CPU Time":5985887015,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":122,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20255703,"Total Records Read":1503082},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":46694,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":432,"Index":41,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734315,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"13740133","Value":"352648230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1035151","Value":"26755466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1035151","Value":"26755466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"1","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1035151","Value":"26755466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1016060","Value":"26307596","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"785","Value":"14309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"234881024","Value":"7180648448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5665","Value":"103938","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7046687","Value":"174032940","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7013519","Value":"173279500","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"304615","Value":"7887512","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"6129","Value":"111975","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":297,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9309574,"Value":235052948,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":6181,"Value":113535,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":6009416427,"Value":109571868772,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":221094,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":1409,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":436243728,"Value":12617441456,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":13740133,"Value":352648230,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1035151,"Value":26755466,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":818,"Value":22206,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":59680,"Value":1546262,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":9309574,"Executor Run Time":6181,"Executor CPU Time":6009416427,"Peak Execution Memory":436243728,"Result Size":8189,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":13740133,"Total Records Read":1035151},"Shuffle Write Metrics":{"Shuffle Bytes Written":818,"Shuffle Write Time":59680,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":403,"Index":13,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734407,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"23718187","Value":"376366417","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1791015","Value":"28546481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1791015","Value":"28546481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"6","Value":"138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1791015","Value":"28546481","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1758874","Value":"28066470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"143","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"786","Value":"15095","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"469762048","Value":"7650410496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5796","Value":"109734","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"11937551","Value":"185970491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"11907003","Value":"185186503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"527548","Value":"8415060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"6212","Value":"118187","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":13,"Value":310,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8449342,"Value":243502290,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":6277,"Value":119812,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":6158402424,"Value":115730271196,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8188,"Value":229282,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":1446,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":671124752,"Value":13288566208,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":23718187,"Value":376366417,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1791015,"Value":28546481,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":23028,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":55414,"Value":1601676,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":14979502832,"JVMOffHeapMemory":146808936,"OnHeapExecutionMemory":872415232,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":1145930329,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1556358,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":99,"MinorGCTime":1784,"MajorGCCount":7,"MajorGCTime":827},"Task Metrics":{"Executor Deserialize Time":13,"Executor Deserialize CPU Time":8449342,"Executor Run Time":6277,"Executor CPU Time":6158402424,"Peak Execution Memory":671124752,"Result Size":8188,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":23718187,"Total Records Read":1791015},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":55414,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":426,"Index":24,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734495,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"20356289","Value":"396722706","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1539649","Value":"30086130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1539649","Value":"30086130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"0","Value":"138","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1539649","Value":"30086130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1515507","Value":"29581977","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"148","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"743","Value":"15838","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"8053063680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"5910","Value":"115644","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"9818948","Value":"195789439","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"9784858","Value":"194971361","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"454481","Value":"8869541","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"6309","Value":"124496","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":322,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7563639,"Value":251065929,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":6359,"Value":126171,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":6242855879,"Value":121973127075,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":237471,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":1493,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":13892582096,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20356289,"Value":396722706,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1539649,"Value":30086130,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":23851,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":49521,"Value":1651197,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":7563639,"Executor Run Time":6359,"Executor CPU Time":6242855879,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20356289,"Total Records Read":1539649},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":49521,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":434,"Index":42,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734592,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"30","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"16633555","Value":"413356261","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1252125","Value":"31338255","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1252125","Value":"31338255","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"1","Value":"139","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1252125","Value":"31338255","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1237479","Value":"30819456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"153","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"568","Value":"16406","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"8355053568","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"6077","Value":"121721","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7978094","Value":"203767533","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7948557","Value":"202919918","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"370909","Value":"9240450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"6405","Value":"130901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":333,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8947587,"Value":260013516,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":6457,"Value":132628,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":6349174059,"Value":128322301134,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":245660,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":1540,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":14395934688,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16633555,"Value":413356261,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1252125,"Value":31338255,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":24674,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":47938,"Value":1699135,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8947587,"Executor Run Time":6457,"Executor CPU Time":6349174059,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16633555,"Total Records Read":1252125},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":47938,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":436,"Index":43,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734827,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"31","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"19771704","Value":"433127965","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1468326","Value":"32806581","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1468326","Value":"32806581","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"7","Value":"146","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1468326","Value":"32806581","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1447733","Value":"32267189","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"711","Value":"17117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"8757706752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"6245","Value":"127966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"10577490","Value":"214345023","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"10549548","Value":"213469466","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"434707","Value":"9675157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"6641","Value":"137542","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":345,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7834818,"Value":267848334,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":6691,"Value":139319,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":6549707651,"Value":134872008785,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":253849,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":1587,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":14999950576,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":31,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":19771704,"Value":433127965,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1468326,"Value":32806581,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":821,"Value":25495,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":31,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":49389,"Value":1748524,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":7834818,"Executor Run Time":6691,"Executor CPU Time":6549707651,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":19771704,"Total Records Read":1468326},"Shuffle Write Metrics":{"Shuffle Bytes Written":821,"Shuffle Write Time":49389,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":404,"Index":18,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432734882,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"23325788","Value":"456453753","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1776905","Value":"34583486","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1776905","Value":"34583486","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"1","Value":"147","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1776905","Value":"34583486","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1751997","Value":"34019186","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"162","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"923","Value":"18040","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"469762048","Value":"9227468800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"6203","Value":"134169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"11893096","Value":"226238119","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"11865777","Value":"225335243","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"525601","Value":"10200758","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"6706","Value":"144248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":354,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8318803,"Value":276167137,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":6755,"Value":146074,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":6567148539,"Value":141439157324,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":262038,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":122,"Value":1709,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":671124752,"Value":15671075328,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":23325788,"Value":456453753,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1776905,"Value":34583486,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":26319,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":47784,"Value":1796308,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":3683160272,"JVMOffHeapMemory":148981192,"OnHeapExecutionMemory":2013265920,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2286781017,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1661300,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":2460,"MajorGCCount":7,"MajorGCTime":844},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":8318803,"Executor Run Time":6755,"Executor CPU Time":6567148539,"Peak Execution Memory":671124752,"Result Size":8189,"JVM GC Time":122,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":23325788,"Total Records Read":1776905},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":47784,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":398,"Index":2,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432735192,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"21741695","Value":"478195448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1592591","Value":"36176077","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1592591","Value":"36176077","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"0","Value":"147","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1592591","Value":"36176077","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1570179","Value":"35589365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"166","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"738","Value":"18778","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"9630121984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"6587","Value":"140756","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"10699132","Value":"236937251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"10663812","Value":"235999055","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"471906","Value":"10672664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"7005","Value":"151253","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":364,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8838342,"Value":285005479,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":7057,"Value":153131,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":6919204818,"Value":148358362142,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":270227,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":1756,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":16275091216,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21741695,"Value":478195448,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1592591,"Value":36176077,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":27143,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":57714,"Value":1854022,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":18472397016,"JVMOffHeapMemory":147865040,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":1600,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":8838342,"Executor Run Time":7057,"Executor CPU Time":6919204818,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21741695,"Total Records Read":1592591},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":57714,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":394,"Index":1,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432735310,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"18420350","Value":"496615798","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1358796","Value":"37534873","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1358796","Value":"37534873","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"7","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1358796","Value":"37534873","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1331797","Value":"36921162","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"812","Value":"19590","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"9932111872","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"6759","Value":"147515","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"9392292","Value":"246329543","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"9357092","Value":"245356147","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"399070","Value":"11071734","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"7124","Value":"158377","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":374,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8731044,"Value":293736523,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":7175,"Value":160306,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":7052906643,"Value":155411268785,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":278416,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":47,"Value":1803,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":16778443808,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":18420350,"Value":496615798,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1358796,"Value":37534873,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":27965,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":60755,"Value":1914777,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":18472397016,"JVMOffHeapMemory":147865040,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":1600,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":8731044,"Executor Run Time":7175,"Executor CPU Time":7052906643,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":47,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":18420350,"Total Records Read":1358796},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":60755,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":418,"Index":22,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432735395,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"17648290","Value":"514264088","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1330648","Value":"38865521","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1330648","Value":"38865521","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"7","Value":"161","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1330648","Value":"38865521","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1310953","Value":"38232115","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"177","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"665","Value":"20255","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"10234101760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"6790","Value":"154305","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"8269956","Value":"254599499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"8241301","Value":"253597448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"393394","Value":"11465128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"7209","Value":"165586","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":383,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8642675,"Value":302379198,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":7261,"Value":167567,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":7021104657,"Value":162432373442,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":286605,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":113,"Value":1916,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":17281796400,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17648290,"Value":514264088,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1330648,"Value":38865521,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":28788,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":44993,"Value":1959770,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":18472397016,"JVMOffHeapMemory":147865040,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":1600,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":8642675,"Executor Run Time":7261,"Executor CPU Time":7021104657,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":113,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17648290,"Total Records Read":1330648},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":44993,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":433,"Index":46,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432735442,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"13508300","Value":"527772388","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1023135","Value":"39888656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1023135","Value":"39888656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"9","Value":"170","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1023135","Value":"39888656","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1009708","Value":"39241823","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"7","Value":"184","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"462","Value":"20717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"234881024","Value":"10468982784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"6974","Value":"161279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"6850760","Value":"261450259","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"6817396","Value":"260414844","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"302283","Value":"11767411","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"7242","Value":"172828","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":399,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7621045,"Value":310000243,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":7305,"Value":174872,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":7007548817,"Value":169439922259,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":294794,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":100,"Value":2016,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":436243728,"Value":17718040128,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":13508300,"Value":527772388,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1023135,"Value":39888656,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":29610,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":49366,"Value":2009136,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":7621045,"Executor Run Time":7305,"Executor CPU Time":7007548817,"Peak Execution Memory":436243728,"Result Size":8189,"JVM GC Time":100,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":13508300,"Total Records Read":1023135},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":49366,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":402,"Index":3,"Attempt":0,"Launch Time":1646432728096,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432735833,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"24749426","Value":"552521814","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1804783","Value":"41693439","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1804783","Value":"41693439","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"1","Value":"171","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1804783","Value":"41693439","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1778105","Value":"41019928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"190","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"880","Value":"21597","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"469762048","Value":"10938744832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"7106","Value":"168385","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"13318435","Value":"274768694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"13286263","Value":"273701107","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"533115","Value":"12300526","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"7646","Value":"180474","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":410,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8747901,"Value":318748144,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":7697,"Value":182569,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":7471642189,"Value":176911564448,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":302983,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":113,"Value":2129,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":671124752,"Value":18389164880,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":24749426,"Value":552521814,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1804783,"Value":41693439,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":30433,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":45997,"Value":2055133,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":18472397016,"JVMOffHeapMemory":147865040,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":1600,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8747901,"Executor Run Time":7697,"Executor CPU Time":7471642189,"Peak Execution Memory":671124752,"Result Size":8189,"JVM GC Time":113,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":24749426,"Total Records Read":1804783},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":45997,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":438,"Index":44,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432736023,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"20467942","Value":"572989756","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1520355","Value":"43213794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1520355","Value":"43213794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"7","Value":"178","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1520355","Value":"43213794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1500662","Value":"42520590","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"195","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"761","Value":"22358","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"11341398016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"7449","Value":"175834","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"10159321","Value":"284928015","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"10122495","Value":"283823602","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"450849","Value":"12751375","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"7836","Value":"188310","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":420,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9477291,"Value":328225435,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":7889,"Value":190458,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":7676348954,"Value":184587913402,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":311172,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":113,"Value":2242,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":18993180768,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20467942,"Value":572989756,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1520355,"Value":43213794,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":31257,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":70127,"Value":2125260,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":18472397016,"JVMOffHeapMemory":147865040,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":1600,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":9477291,"Executor Run Time":7889,"Executor CPU Time":7676348954,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":113,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20467942,"Total Records Read":1520355},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":70127,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":422,"Index":23,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432736155,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"22682377","Value":"595672133","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1708146","Value":"44921940","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1708146","Value":"44921940","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"7","Value":"185","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1708146","Value":"44921940","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1671692","Value":"44192282","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"5","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"828","Value":"23186","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"11744051200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"7535","Value":"183369","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"11677434","Value":"296605449","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"11650243","Value":"295473845","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"500801","Value":"13252176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"7969","Value":"196279","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":431,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8775562,"Value":337000997,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":8020,"Value":198478,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":7758888211,"Value":192346801613,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":319361,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":113,"Value":2355,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":19597196656,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22682377,"Value":595672133,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1708146,"Value":44921940,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":820,"Value":32077,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":47652,"Value":2172912,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":18472397016,"JVMOffHeapMemory":147865040,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":1600,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8775562,"Executor Run Time":8020,"Executor CPU Time":7758888211,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":113,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22682377,"Total Records Read":1708146},"Shuffle Write Metrics":{"Shuffle Bytes Written":820,"Shuffle Write Time":47652,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":406,"Index":4,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432737012,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"26091211","Value":"621763344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1890520","Value":"46812460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1890520","Value":"46812460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"8","Value":"193","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1890520","Value":"46812460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1860972","Value":"46053254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"4","Value":"204","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"894","Value":"24080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"469762048","Value":"12213813248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"8347","Value":"191716","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"13289107","Value":"309894556","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"13248488","Value":"308722333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"558546","Value":"13810722","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"8826","Value":"205105","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":442,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7823140,"Value":344824137,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":8877,"Value":207355,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":8544933356,"Value":200891734969,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":327550,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":113,"Value":2468,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":671124752,"Value":20268321408,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":26091211,"Value":621763344,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1890520,"Value":46812460,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":824,"Value":32901,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":48077,"Value":2220989,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":18472397016,"JVMOffHeapMemory":147865040,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1628926,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":107,"MinorGCTime":1600,"MajorGCCount":7,"MajorGCTime":660},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":7823140,"Executor Run Time":8877,"Executor CPU Time":8544933356,"Peak Execution Memory":671124752,"Result Size":8189,"JVM GC Time":113,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":26091211,"Total Records Read":1890520},"Shuffle Write Metrics":{"Shuffle Bytes Written":824,"Shuffle Write Time":48077,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":421,"Index":27,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432737866,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"41","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"17698934","Value":"639462278","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1333919","Value":"48146379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1333919","Value":"48146379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"14","Value":"207","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1333919","Value":"48146379","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1319111","Value":"47372365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"801","Value":"24881","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"12515803136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"9204","Value":"200920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"8355380","Value":"318249936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"8324284","Value":"317046617","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"394917","Value":"14205639","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"9664","Value":"214769","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":454,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7949172,"Value":352773309,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":9733,"Value":217088,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":9447429321,"Value":210339164290,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":335739,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":140,"Value":2608,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":20771674000,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":17698934,"Value":639462278,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1333919,"Value":48146379,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":821,"Value":33722,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":55722,"Value":2276711,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":7949172,"Executor Run Time":9733,"Executor CPU Time":9447429321,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":140,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":17698934,"Total Records Read":1333919},"Shuffle Write Metrics":{"Shuffle Bytes Written":821,"Shuffle Write Time":55722,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":397,"Index":6,"Attempt":0,"Launch Time":1646432728094,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432738583,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"15259337","Value":"654721615","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1154264","Value":"49300643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1154264","Value":"49300643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"10","Value":"217","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1154264","Value":"49300643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1136753","Value":"48509118","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"216","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"521","Value":"25402","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"12817793024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"10108","Value":"211028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7600974","Value":"325850910","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7568547","Value":"324615164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"341477","Value":"14547116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"10382","Value":"225151","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":465,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8398943,"Value":361172252,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":10453,"Value":227541,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":10190575140,"Value":220529739430,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":343928,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":140,"Value":2748,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":21275026592,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":15259337,"Value":654721615,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1154264,"Value":49300643,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":34545,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":47750,"Value":2324461,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8398943,"Executor Run Time":10453,"Executor CPU Time":10190575140,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":140,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":15259337,"Total Records Read":1154264},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":47750,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":437,"Index":48,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432739156,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"43","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"19877448","Value":"674599063","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1480083","Value":"50780726","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1480083","Value":"50780726","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"15","Value":"232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1480083","Value":"50780726","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1464397","Value":"49973515","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"684","Value":"26086","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"13220446208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"10602","Value":"221630","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"10574238","Value":"336425148","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"10546700","Value":"335161864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"439516","Value":"14986632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"10954","Value":"236105","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":476,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8906358,"Value":370078610,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":11024,"Value":238565,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":10751985545,"Value":231281724975,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":352117,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":140,"Value":2888,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":21879042480,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":19877448,"Value":674599063,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1480083,"Value":50780726,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":825,"Value":35370,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":49404,"Value":2373865,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8906358,"Executor Run Time":11024,"Executor CPU Time":10751985545,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":140,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":19877448,"Total Records Read":1480083},"Shuffle Write Metrics":{"Shuffle Bytes Written":825,"Shuffle Write Time":49404,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":401,"Index":7,"Attempt":0,"Launch Time":1646432728095,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432739209,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"18182549","Value":"692781612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1355043","Value":"52135769","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1355043","Value":"52135769","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"7","Value":"239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1355043","Value":"52135769","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1338255","Value":"51311770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"228","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"592","Value":"26678","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"13522436096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"10691","Value":"232321","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"8599656","Value":"345024804","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"8567580","Value":"343729444","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"400336","Value":"15386968","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"11006","Value":"247111","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":485,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7006423,"Value":377085033,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":11078,"Value":249643,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":10794250274,"Value":242075975249,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":360306,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":140,"Value":3028,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":22382395072,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":18182549,"Value":692781612,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1355043,"Value":52135769,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":36192,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":51380,"Value":2425245,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":7006423,"Executor Run Time":11078,"Executor CPU Time":10794250274,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":140,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":18182549,"Total Records Read":1355043},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":51380,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":435,"Index":47,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432739797,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"16286726","Value":"709068338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1228580","Value":"53364349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1228580","Value":"53364349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"11","Value":"250","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1228580","Value":"53364349","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1215094","Value":"52526864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"234","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"519","Value":"27197","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"301989888","Value":"13824425984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"11308","Value":"243629","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"7704937","Value":"352729741","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"7674636","Value":"351404080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"363684","Value":"15750652","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"11597","Value":"258708","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":496,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8949591,"Value":386034624,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":11666,"Value":261309,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":11398081446,"Value":253474056695,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":368495,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":140,"Value":3168,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":503352592,"Value":22885747664,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":16286726,"Value":709068338,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1228580,"Value":53364349,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":821,"Value":37013,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":47740,"Value":2472985,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8949591,"Executor Run Time":11666,"Executor CPU Time":11398081446,"Peak Execution Memory":503352592,"Result Size":8189,"JVM GC Time":140,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":16286726,"Total Records Read":1228580},"Shuffle Write Metrics":{"Shuffle Bytes Written":821,"Shuffle Write Time":47740,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":439,"Index":49,"Attempt":0,"Launch Time":1646432728100,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432741568,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"46","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"20138112","Value":"729206450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1498633","Value":"54862982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1498633","Value":"54862982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"11","Value":"261","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1498633","Value":"54862982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1483487","Value":"54010351","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"674","Value":"27871","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"14227079168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"12745","Value":"256374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"9849078","Value":"362578819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"9812593","Value":"361216673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"445199","Value":"16195851","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"13367","Value":"272075","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":507,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8173876,"Value":394208500,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":13437,"Value":274746,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":12915825180,"Value":266389881875,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":376684,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":390,"Value":3558,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":23489763552,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20138112,"Value":729206450,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1498633,"Value":54862982,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":37836,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":46496,"Value":2519481,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8173876,"Executor Run Time":13437,"Executor CPU Time":12915825180,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":390,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20138112,"Total Records Read":1498633},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":46496,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":425,"Index":28,"Attempt":0,"Launch Time":1646432728098,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432741600,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"47","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"22891115","Value":"752097565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1719119","Value":"56582101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1719119","Value":"56582101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"12","Value":"273","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1719119","Value":"56582101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1696792","Value":"55707143","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"768","Value":"28639","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"14629732352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"12736","Value":"269110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"12201936","Value":"374780755","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"12171849","Value":"373388522","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"508677","Value":"16704528","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"13399","Value":"285474","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":517,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7824625,"Value":402033125,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":13470,"Value":288216,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":12952614354,"Value":279342496229,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":384873,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":390,"Value":3948,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":24093779440,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22891115,"Value":752097565,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1719119,"Value":56582101,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":823,"Value":38659,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":56033,"Value":2575514,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":7824625,"Executor Run Time":13470,"Executor CPU Time":12952614354,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":390,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22891115,"Total Records Read":1719119},"Shuffle Write Metrics":{"Shuffle Bytes Written":823,"Shuffle Write Time":56033,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":429,"Index":29,"Attempt":0,"Launch Time":1646432728099,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432741636,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"20388677","Value":"772486242","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1540279","Value":"58122380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1540279","Value":"58122380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"8","Value":"281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1540279","Value":"58122380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1524417","Value":"57231560","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"935","Value":"29574","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"15032385536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"13078","Value":"282188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"10008625","Value":"384789380","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"9971320","Value":"383359842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"457322","Value":"17161850","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"13436","Value":"298910","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":19,"Value":536,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7885832,"Value":409918957,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":13497,"Value":301713,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":12917032381,"Value":292259528610,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":393062,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":390,"Value":4338,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":24697795328,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":20388677,"Value":772486242,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1540279,"Value":58122380,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":819,"Value":39478,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":49582,"Value":2625096,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":19,"Executor Deserialize CPU Time":7885832,"Executor Run Time":13497,"Executor CPU Time":12917032381,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":390,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":20388677,"Total Records Read":1540279},"Shuffle Write Metrics":{"Shuffle Bytes Written":819,"Shuffle Write Time":49582,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":409,"Index":9,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432742165,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"49","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"22864810","Value":"795351052","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1697113","Value":"59819493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1697113","Value":"59819493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"12","Value":"293","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1697113","Value":"59819493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1676385","Value":"58907945","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"6","Value":"258","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"1059","Value":"30633","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"15435038720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"13582","Value":"295770","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"11418581","Value":"396207961","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"11380207","Value":"394740049","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"504323","Value":"17666173","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"13965","Value":"312875","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":547,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8173916,"Value":418092873,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":14036,"Value":315749,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":13426976176,"Value":305686504786,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":401251,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":390,"Value":4728,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":25301811216,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":22864810,"Value":795351052,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1697113,"Value":59819493,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":821,"Value":40299,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":44911,"Value":2670007,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":8173916,"Executor Run Time":14036,"Executor CPU Time":13426976176,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":390,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":22864810,"Total Records Read":1697113},"Shuffle Write Metrics":{"Shuffle Bytes Written":821,"Shuffle Write Time":44911,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":92,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":405,"Index":8,"Attempt":0,"Launch Time":1646432728097,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432743167,"Failed":false,"Killed":false,"Accumulables":[{"ID":2562,"Name":"local blocks read","Update":"1","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Update":"23437384","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Update":"1737471","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Update":"1737471","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Update":"16","Value":"309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Update":"1737471","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Update":"1708343","Value":"60616288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Update":"7","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Update":"794","Value":"31427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Update":"402653184","Value":"15837691904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Update":"14573","Value":"310343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Update":"12578897","Value":"408786858","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Update":"12549967","Value":"407290016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Update":"512763","Value":"18178936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Update":"14967","Value":"327842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":563,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7453060,"Value":425545933,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Update":15030,"Value":330779,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Update":14433356253,"Value":320119861039,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Update":8189,"Value":409440,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Update":390,"Value":5118,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Update":604015888,"Value":25905827104,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":23437384,"Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1737471,"Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":822,"Value":41121,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Update":64429,"Value":2734436,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":17523345656,"JVMOffHeapMemory":146687512,"OnHeapExecutionMemory":2684354560,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":273515097,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":2957869657,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1493510,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":100,"MinorGCTime":1541,"MajorGCCount":8,"MajorGCTime":762},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":7453060,"Executor Run Time":15030,"Executor CPU Time":14433356253,"Peak Execution Memory":604015888,"Result Size":8189,"JVM GC Time":390,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":23437384,"Total Records Read":1737471},"Shuffle Write Metrics":{"Shuffle Bytes Written":822,"Shuffle Write Time":64429,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":92,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":50,"RDD Info":[{"RDD ID":215,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[214],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":212,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"352\",\"name\":\"map\"}","Callsite":"map at LinearRegression.scala:914","Parent IDs":[211],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":209,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10952\",\"name\":\"DeserializeToObject\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[208],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":208,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"spark_plan_10951\",\"name\":\"WholeStageCodegen\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[207],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":210,"Name":"SQLExecutionRDD","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[209],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":207,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"spark_plan_10925\",\"name\":\"CustomShuffleReader\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[183],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":214,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[213],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":211,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"351\",\"name\":\"mapPartitions\"}","Callsite":"rdd at LinearRegression.scala:912","Parent IDs":[210],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":213,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"353\",\"name\":\"map\"}","Callsite":"map at RegressionMetrics.scala:63","Parent IDs":[212],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":50,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[91],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432728088,"Completion Time":1646432743168,"Accumulables":[{"ID":2562,"Name":"local blocks read","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2565,"Name":"local bytes read","Value":"818788436","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2566,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2567,"Name":"records read","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2909,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2910,"Name":"time to read broadcast","Value":"309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2911,"Name":"number of output rows","Value":"61556964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"time to read broadcast","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"number of output rows","Value":"60616288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"time to read broadcast","Value":"265","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"sort time","Value":"31427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"peak memory","Value":"15837691904","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2917,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2918,"Name":"cumulative time","Value":"310343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2919,"Name":"spill write time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2920,"Name":"num prefix comparisons","Value":"408786858","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2921,"Name":"num record comparisons","Value":"407290016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2922,"Name":"number of output rows","Value":"18178936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2923,"Name":"duration","Value":"327842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2925,"Name":"internal.metrics.executorDeserializeTime","Value":563,"Internal":true,"Count Failed Values":true},{"ID":2926,"Name":"internal.metrics.executorDeserializeCpuTime","Value":425545933,"Internal":true,"Count Failed Values":true},{"ID":2927,"Name":"internal.metrics.executorRunTime","Value":330779,"Internal":true,"Count Failed Values":true},{"ID":2928,"Name":"internal.metrics.executorCpuTime","Value":320119861039,"Internal":true,"Count Failed Values":true},{"ID":2929,"Name":"internal.metrics.resultSize","Value":409440,"Internal":true,"Count Failed Values":true},{"ID":2930,"Name":"internal.metrics.jvmGCTime","Value":5118,"Internal":true,"Count Failed Values":true},{"ID":2932,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2933,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2934,"Name":"internal.metrics.peakExecutionMemory","Value":25905827104,"Internal":true,"Count Failed Values":true},{"ID":2938,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2939,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":50,"Internal":true,"Count Failed Values":true},{"ID":2940,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2941,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2942,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":818788436,"Internal":true,"Count Failed Values":true},{"ID":2943,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.read.recordsRead","Value":61556964,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":41121,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":50,"Internal":true,"Count Failed Values":true},{"ID":2947,"Name":"internal.metrics.shuffle.write.writeTime","Value":2734436,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":93,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":6,"RDD Info":[{"RDD ID":217,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[216],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":216,"Name":"ShuffledRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[215],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[92],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432743169,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.databricks.job.id":"703","spark.scheduler.pool":"4598586477831989067","spark.databricks.env":"databricks-python-repl","spark.job.interruptOnCancel":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.databricks.api.url":"https://nvirginia.cloud.databricks.com","spark.databricks.service.client.type":"python","spark.rdd.scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","spark.databricks.inherited.spark.databricks.api.url":"*********(redacted)","spark.rdd.scope.noOverride":"true","spark.job.description":"\nimport tempfile\nfile = tempfile.NamedTemporary...","spark.databricks.job.runId":"31144","spark.databricks.replId":"ReplId-3fd17-60988-17274-b","spark.jobGroup.id":"4598586477831989067_7525196874081667841_job-703-run-31144-action-4960839961566214","spark.databricks.isolationID":"1efbc00e-cb29-4e80-b07b-fe3ae8b92b02","spark.databricks.inherited.credentials.keys.spark.databricks.api.token":"*********(redacted)","spark.databricks.job.type":"python","spark.databricks.token":"*********(redacted)"}} +{"Event":"SparkListenerTaskStart","Stage ID":93,"Stage Attempt ID":0,"Task Info":{"Task ID":440,"Index":0,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":93,"Stage Attempt ID":0,"Task Info":{"Task ID":441,"Index":1,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":93,"Stage Attempt ID":0,"Task Info":{"Task ID":442,"Index":2,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":93,"Stage Attempt ID":0,"Task Info":{"Task ID":443,"Index":3,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":93,"Stage Attempt ID":0,"Task Info":{"Task ID":444,"Index":4,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":93,"Stage Attempt ID":0,"Task Info":{"Task ID":445,"Index":5,"Attempt":0,"Launch Time":1646432743177,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":93,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":445,"Index":5,"Attempt":0,"Launch Time":1646432743177,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432743208,"Failed":false,"Killed":false,"Accumulables":[{"ID":2955,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2956,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6319754,"Value":6319754,"Internal":true,"Count Failed Values":true},{"ID":2957,"Name":"internal.metrics.executorRunTime","Update":9,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2958,"Name":"internal.metrics.executorCpuTime","Update":5251696,"Value":5251696,"Internal":true,"Count Failed Values":true},{"ID":2959,"Name":"internal.metrics.resultSize","Update":9231,"Value":9231,"Internal":true,"Count Failed Values":true},{"ID":2962,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2963,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2964,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":2352,"Internal":true,"Count Failed Values":true},{"ID":2968,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2969,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2970,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":4919,"Value":4919,"Internal":true,"Count Failed Values":true},{"ID":2971,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2972,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1638,"Value":1638,"Internal":true,"Count Failed Values":true},{"ID":2973,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2974,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":8,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":6319754,"Executor Run Time":9,"Executor CPU Time":5251696,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":6,"Local Blocks Fetched":2,"Fetch Wait Time":2,"Remote Bytes Read":4919,"Remote Bytes Read To Disk":0,"Local Bytes Read":1638,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":93,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":441,"Index":1,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"0","Host":"10.77.255.181","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432743209,"Failed":false,"Killed":false,"Accumulables":[{"ID":2955,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2956,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8259184,"Value":14578938,"Internal":true,"Count Failed Values":true},{"ID":2957,"Name":"internal.metrics.executorRunTime","Update":8,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":2958,"Name":"internal.metrics.executorCpuTime","Update":5243802,"Value":10495498,"Internal":true,"Count Failed Values":true},{"ID":2959,"Name":"internal.metrics.resultSize","Update":9231,"Value":18462,"Internal":true,"Count Failed Values":true},{"ID":2962,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2963,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2964,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":4704,"Internal":true,"Count Failed Values":true},{"ID":2968,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2969,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2970,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":5755,"Value":10674,"Internal":true,"Count Failed Values":true},{"ID":2971,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2972,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1643,"Value":3281,"Internal":true,"Count Failed Values":true},{"ID":2973,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":2974,"Name":"internal.metrics.shuffle.read.recordsRead","Update":9,"Value":17,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":8259184,"Executor Run Time":8,"Executor CPU Time":5243802,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":2,"Fetch Wait Time":1,"Remote Bytes Read":5755,"Remote Bytes Read To Disk":0,"Local Bytes Read":1643,"Total Records Read":9},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":93,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":444,"Index":4,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432743210,"Failed":false,"Killed":false,"Accumulables":[{"ID":2955,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":2956,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6740830,"Value":21319768,"Internal":true,"Count Failed Values":true},{"ID":2957,"Name":"internal.metrics.executorRunTime","Update":8,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2958,"Name":"internal.metrics.executorCpuTime","Update":5116448,"Value":15611946,"Internal":true,"Count Failed Values":true},{"ID":2959,"Name":"internal.metrics.resultSize","Update":9231,"Value":27693,"Internal":true,"Count Failed Values":true},{"ID":2962,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2963,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2964,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":7056,"Internal":true,"Count Failed Values":true},{"ID":2968,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2969,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2970,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":5766,"Value":16440,"Internal":true,"Count Failed Values":true},{"ID":2971,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2972,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":823,"Value":4104,"Internal":true,"Count Failed Values":true},{"ID":2973,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":2974,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":25,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":6740830,"Executor Run Time":8,"Executor CPU Time":5116448,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":1,"Remote Bytes Read":5766,"Remote Bytes Read To Disk":0,"Local Bytes Read":823,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":93,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":440,"Index":0,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"2","Host":"10.77.253.227","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432743211,"Failed":false,"Killed":false,"Accumulables":[{"ID":2955,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2956,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6780652,"Value":28100420,"Internal":true,"Count Failed Values":true},{"ID":2957,"Name":"internal.metrics.executorRunTime","Update":9,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2958,"Name":"internal.metrics.executorCpuTime","Update":4936677,"Value":20548623,"Internal":true,"Count Failed Values":true},{"ID":2959,"Name":"internal.metrics.resultSize","Update":9231,"Value":36924,"Internal":true,"Count Failed Values":true},{"ID":2962,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2963,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2964,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":9408,"Internal":true,"Count Failed Values":true},{"ID":2968,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2969,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":2970,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":5762,"Value":22202,"Internal":true,"Count Failed Values":true},{"ID":2971,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2972,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1646,"Value":5750,"Internal":true,"Count Failed Values":true},{"ID":2973,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2974,"Name":"internal.metrics.shuffle.read.recordsRead","Update":9,"Value":34,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":6780652,"Executor Run Time":9,"Executor CPU Time":4936677,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":2,"Fetch Wait Time":1,"Remote Bytes Read":5762,"Remote Bytes Read To Disk":0,"Local Bytes Read":1646,"Total Records Read":9},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":93,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":443,"Index":3,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"3","Host":"10.77.234.35","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432743216,"Failed":false,"Killed":false,"Accumulables":[{"ID":2955,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2956,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7124817,"Value":35225237,"Internal":true,"Count Failed Values":true},{"ID":2957,"Name":"internal.metrics.executorRunTime","Update":16,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":2958,"Name":"internal.metrics.executorCpuTime","Update":5728827,"Value":26277450,"Internal":true,"Count Failed Values":true},{"ID":2959,"Name":"internal.metrics.resultSize","Update":9231,"Value":46155,"Internal":true,"Count Failed Values":true},{"ID":2962,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2963,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2964,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":11760,"Internal":true,"Count Failed Values":true},{"ID":2968,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2969,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":2970,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":5758,"Value":27960,"Internal":true,"Count Failed Values":true},{"ID":2971,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2972,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":823,"Value":6573,"Internal":true,"Count Failed Values":true},{"ID":2973,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2974,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":42,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":7124817,"Executor Run Time":16,"Executor CPU Time":5728827,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":1,"Remote Bytes Read":5758,"Remote Bytes Read To Disk":0,"Local Bytes Read":823,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":93,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":442,"Index":2,"Attempt":0,"Launch Time":1646432743176,"Executor ID":"1","Host":"10.77.224.250","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1646432743218,"Failed":false,"Killed":false,"Accumulables":[{"ID":2955,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2956,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7647709,"Value":42872946,"Internal":true,"Count Failed Values":true},{"ID":2957,"Name":"internal.metrics.executorRunTime","Update":16,"Value":66,"Internal":true,"Count Failed Values":true},{"ID":2958,"Name":"internal.metrics.executorCpuTime","Update":7103925,"Value":33381375,"Internal":true,"Count Failed Values":true},{"ID":2959,"Name":"internal.metrics.resultSize","Update":9231,"Value":55386,"Internal":true,"Count Failed Values":true},{"ID":2962,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2963,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2964,"Name":"internal.metrics.peakExecutionMemory","Update":2352,"Value":14112,"Internal":true,"Count Failed Values":true},{"ID":2968,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":6,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":2969,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2970,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":4944,"Value":32904,"Internal":true,"Count Failed Values":true},{"ID":2971,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2972,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1644,"Value":8217,"Internal":true,"Count Failed Values":true},{"ID":2973,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2974,"Name":"internal.metrics.shuffle.read.recordsRead","Update":8,"Value":50,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":7647709,"Executor Run Time":16,"Executor CPU Time":7103925,"Peak Execution Memory":2352,"Result Size":9231,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":6,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":4944,"Remote Bytes Read To Disk":0,"Local Bytes Read":1644,"Total Records Read":8},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":93,"Stage Attempt ID":0,"Stage Name":"treeAggregate at Statistics.scala:58","Number of Tasks":6,"RDD Info":[{"RDD ID":217,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[216],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":216,"Name":"ShuffledRDD","Scope":"{\"id\":\"354\",\"name\":\"treeAggregate\"}","Callsite":"treeAggregate at Statistics.scala:58","Parent IDs":[215],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":6,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[92],"Details":"org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1232)\norg.apache.spark.mllib.stat.Statistics$.colStats(Statistics.scala:58)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary$lzycompute(RegressionMetrics.scala:70)\norg.apache.spark.mllib.evaluation.RegressionMetrics.summary(RegressionMetrics.scala:62)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg$lzycompute(RegressionMetrics.scala:76)\norg.apache.spark.mllib.evaluation.RegressionMetrics.SSreg(RegressionMetrics.scala:77)\norg.apache.spark.mllib.evaluation.RegressionMetrics.explainedVariance(RegressionMetrics.scala:88)\norg.apache.spark.ml.regression.LinearRegressionSummary.(LinearRegression.scala:925)\norg.apache.spark.ml.regression.LinearRegressionModel.evaluate(LinearRegression.scala:718)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\npy4j.Gateway.invoke(Gateway.java:295)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.GatewayConnection.run(GatewayConnection.java:251)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1646432743169,"Completion Time":1646432743219,"Accumulables":[{"ID":2955,"Name":"internal.metrics.executorDeserializeTime","Value":41,"Internal":true,"Count Failed Values":true},{"ID":2956,"Name":"internal.metrics.executorDeserializeCpuTime","Value":42872946,"Internal":true,"Count Failed Values":true},{"ID":2957,"Name":"internal.metrics.executorRunTime","Value":66,"Internal":true,"Count Failed Values":true},{"ID":2958,"Name":"internal.metrics.executorCpuTime","Value":33381375,"Internal":true,"Count Failed Values":true},{"ID":2959,"Name":"internal.metrics.resultSize","Value":55386,"Internal":true,"Count Failed Values":true},{"ID":2962,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2963,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2964,"Name":"internal.metrics.peakExecutionMemory","Value":14112,"Internal":true,"Count Failed Values":true},{"ID":2968,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":40,"Internal":true,"Count Failed Values":true},{"ID":2969,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":10,"Internal":true,"Count Failed Values":true},{"ID":2970,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":32904,"Internal":true,"Count Failed Values":true},{"ID":2971,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2972,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":8217,"Internal":true,"Count Failed Values":true},{"ID":2973,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":2974,"Name":"internal.metrics.shuffle.read.recordsRead","Value":50,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":49,"Completion Time":1646432743219,"Job Result":{"Result":"JobSucceeded"}} diff --git a/tests/logs/databricks.zip b/tests/logs/databricks.zip new file mode 100644 index 0000000..16cf311 Binary files /dev/null and b/tests/logs/databricks.zip differ diff --git a/tests/logs/emr.zip b/tests/logs/emr.zip new file mode 100644 index 0000000..32781f4 Binary files /dev/null and b/tests/logs/emr.zip differ diff --git a/tests/test_bad_eventlog.py b/tests/test_bad_eventlog.py new file mode 100644 index 0000000..236db54 --- /dev/null +++ b/tests/test_bad_eventlog.py @@ -0,0 +1,44 @@ +import tempfile +import unittest +from pathlib import Path + +from spark_log_parser.eventlog import EventLogBuilder + + +class BadEventLog(unittest.TestCase): + def test_multiple_context_ids(self): + event_log = Path("tests", "logs", "bad", "non-unique-context-id.zip").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises( + ValueError, msg="Not all rollover files have the same Spark context ID" + ): + EventLogBuilder(event_log.as_uri(), temp_dir).build() + + def test_missing_dbc_event(self): + event_log = Path("tests", "logs", "bad", "missing-dbc-event.zip").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(ValueError, msg="Expected DBC event not found"): + EventLogBuilder(event_log.as_uri(), temp_dir).build() + + def test_duplicate_log_part(self): + event_log = Path("tests", "logs", "bad", "duplicate-part.tgz").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(ValueError, msg="Duplicate rollover file detected"): + EventLogBuilder(event_log.as_uri(), temp_dir).build() + + def test_missing_log_part(self): + event_log = Path("tests", "logs", "bad", "missing-part.zip").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(ValueError, msg="Rollover file appears to be missing"): + EventLogBuilder(event_log.as_uri(), temp_dir).build() + + def test_missing_first_part(self): + event_log = Path("tests", "logs", "bad", "missing-first-part.zip").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(ValueError, msg="Rollover file appears to be missing"): + EventLogBuilder(event_log.as_uri(), temp_dir).build() diff --git a/tests/test_eventlog.py b/tests/test_eventlog.py new file mode 100644 index 0000000..9691ea9 --- /dev/null +++ b/tests/test_eventlog.py @@ -0,0 +1,102 @@ +import json +import tempfile +import unittest +import zipfile +from pathlib import Path + +from spark_log_parser import eventlog + + +def test_simple_emr_log(): + event_log_path = Path("tests", "logs", "emr.zip").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + event_log = eventlog.EventLogBuilder( + source_url=event_log_path.as_uri(), work_dir=temp_dir + ).build() + + with open(event_log) as log_fobj: + event = json.loads(log_fobj.readline()) + + assert all(key in event for key in ["Event", "Spark Version"]), "All keys are present" + + assert event["Event"] == "SparkListenerLogStart", "Expected first event is present" + + +def test_simple_databricks_log(): + event_log_path = Path("tests", "logs", "databricks.zip").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + event_log = eventlog.EventLogBuilder(event_log_path.as_uri(), temp_dir).build() + + with open(event_log) as log_fobj: + event = json.loads(log_fobj.readline()) + + assert all( + key in event + for key in ["Event", "Spark Version", "Timestamp", "Rollover Number", "SparkContext Id"] + ), "All keys are present" + + +def test_raw_databricks_log(): + event_log_path = Path("tests", "logs", "databricks.json").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + event_log = eventlog.EventLogBuilder(event_log_path.as_uri(), temp_dir).build() + + with open(event_log) as log_fobj: + event = json.loads(log_fobj.readline()) + + assert all( + key in event + for key in ["Event", "Spark Version", "Timestamp", "Rollover Number", "SparkContext Id"] + ), "All keys are present" + + assert event["Event"] == "DBCEventLoggingListenerMetadata", "Expected first event is present" + + +class RolloverLog(unittest.TestCase): + def test_databricks_rollover_log(self): + self.validate_log(Path("tests", "logs", "databricks-rollover.zip").resolve(), 3, 16945) + + def test_databricks_messy_rollover_log_archive(self): + self.validate_log( + Path("tests", "logs", "databricks-rollover-messy.zip").resolve(), 3, 16945 + ) + + def test_databricks_messy_rollover_log_dir(self): + with tempfile.TemporaryDirectory() as temp_dir: + with zipfile.ZipFile(Path("tests", "logs", "databricks-rollover-messy.zip")) as zfile: + temp_dir_path = Path(temp_dir) + zfile.extractall(temp_dir_path) + self.validate_log(temp_dir_path, 3, 16945) + + def validate_log(self, event_log: Path, log_file_total: int, log_entry_total: int): + with tempfile.TemporaryDirectory() as temp_dir: + event_log = eventlog.EventLogBuilder(event_log.as_uri(), temp_dir).build() + + with open(event_log) as log_fobj: + event = json.loads(log_fobj.readline()) + log_fobj.seek(0) + + rollover_count = 0 + for i, event_str in enumerate(log_fobj): + event = json.loads(event_str) + if "Rollover Number" in event: + assert all( + key in event + for key in [ + "Event", + "Spark Version", + "Timestamp", + "Rollover Number", + "SparkContext Id", + ] + ), "All keys are present" + assert ( + rollover_count == event["Rollover Number"] + ), "Contiguous monotonically increasing IDs" + rollover_count += 1 + + assert rollover_count == log_file_total, "All log parts are present" + assert i + 1 == log_entry_total, "All events are present" diff --git a/tests/test_eventlog_s3.py b/tests/test_eventlog_s3.py new file mode 100644 index 0000000..ff67e85 --- /dev/null +++ b/tests/test_eventlog_s3.py @@ -0,0 +1,168 @@ +import io +import json +import os +import tempfile +from pathlib import Path +from urllib.parse import urlparse +from zipfile import ZipFile + +import boto3 as boto +import pytest +from botocore.stub import ANY, Stubber + +from spark_log_parser import eventlog + + +@pytest.mark.parametrize( + "event_log_url", + [ + "s3://sync-test-artifacts/airlinedelay/jb-42K1E16/emr.zip", + "s3://sync-test-artifacts/airlinedelay/jb-42K1E16/", + ], +) +@pytest.mark.parametrize( + "event_log_file_archive,event_log_s3_dir", + [(Path("tests", "logs", "emr.zip"), "airlinedelay/jb-42K1E16/")], +) +def test_emr_log_from_s3(event_log_url, event_log_file_archive, event_log_s3_dir): + parsed_event_log_url = urlparse(event_log_url) + s3_bucket = parsed_event_log_url.netloc + s3_prefix = parsed_event_log_url.path.lstrip("/") + + s3 = boto.client("s3") + stubber = Stubber(s3) + s3_key = event_log_s3_dir + event_log_file_archive.name + s3_object_size = event_log_file_archive.stat().st_size + stubber.add_response( + "list_objects_v2", + {"Contents": [{"Key": s3_key, "Size": s3_object_size}]}, + {"Bucket": s3_bucket, "Prefix": s3_prefix}, + ) + stubber.add_response( + "head_object", + {"ContentLength": s3_object_size, "ContentType": "application/octet-stream"}, + {"Bucket": s3_bucket, "Key": s3_key}, + ) + + with open(event_log_file_archive, "rb") as fobj: + chunk_size = boto.s3.transfer.TransferConfig().multipart_chunksize + chunk = fobj.read(chunk_size) + while chunk: + if s3_object_size > chunk_size: + expected_params = {"Bucket": s3_bucket, "Key": s3_key, "Range": ANY} + else: + expected_params = {"Bucket": s3_bucket, "Key": s3_key} + stubber.add_response( + "get_object", + { + "ContentLength": len(chunk), + "ContentType": "application/octet-stream", + "Body": io.BytesIO(chunk), + }, + expected_params, + ) + + chunk = fobj.read(chunk_size) + + with tempfile.TemporaryDirectory() as temp_dir, stubber: + event_log = eventlog.EventLogBuilder(event_log_url, temp_dir, s3).build() + + with open(event_log) as log_fobj: + event = json.loads(log_fobj.readline()) + + assert all(key in event for key in ["Event", "Spark Version"]), "All keys are present" + + assert event["Event"] == "SparkListenerLogStart", "Expected first event is present" + + +@pytest.mark.parametrize( + "event_log_url", + ["s3://sync-test-artifacts/airlinedelay/jb-42K1E16/", "s3://sync-test-artifacts/airlinedelay"], +) +@pytest.mark.parametrize( + "event_log_file_archive,event_log_s3_dir", + [(Path("tests", "logs", "databricks-rollover-messy.zip"), "airlinedelay/jb-42K1E16/")], +) +def test_databricks_log_from_s3_dir(event_log_url, event_log_file_archive, event_log_s3_dir): + parsed_event_log_url = urlparse(event_log_url) + s3_bucket = parsed_event_log_url.netloc + s3_prefix = parsed_event_log_url.path.lstrip("/") + + with tempfile.TemporaryDirectory() as temp_dir: + s3 = boto.client("s3") + stubber = Stubber(s3) + with ZipFile(event_log_file_archive) as zfile: + contents = [ + (event_log_s3_dir + "/".join(zinfo.filename.split(os.sep)), zinfo) + for zinfo in zfile.infolist() + if not zinfo.is_dir() + ] + stubber.add_response( + "list_objects_v2", + {"Contents": [{"Key": key, "Size": zinfo.file_size} for key, zinfo in contents]}, + {"Bucket": s3_bucket, "Prefix": s3_prefix}, + ) + + for s3_key, zinfo in contents: + with zfile.open(zinfo) as zobj: + stubber.add_response( + "head_object", + { + "ContentLength": zinfo.file_size, + "ContentType": "application/octet-stream", + }, + {"Bucket": s3_bucket, "Key": s3_key}, + ) + + chunk_size = boto.s3.transfer.TransferConfig().multipart_chunksize + chunk = zobj.read(chunk_size) + while chunk: + if zinfo.file_size > chunk_size: + expected_params = {"Bucket": s3_bucket, "Key": s3_key, "Range": ANY} + else: + expected_params = {"Bucket": s3_bucket, "Key": s3_key} + stubber.add_response( + "get_object", + { + "ContentLength": len(chunk), + "ContentType": "application/octet-stream", + "Body": io.BytesIO(chunk), + }, + expected_params, + ) + + chunk = zobj.read(chunk_size) + + with stubber: + event_log = eventlog.EventLogBuilder(event_log_url, temp_dir, s3).build() + + stubber.assert_no_pending_responses() + + with open(event_log) as log_fobj: + event = json.loads(log_fobj.readline()) + log_fobj.seek(0) + + rollover_count = 0 + for i, event_str in enumerate(log_fobj): + try: + event = json.loads(event_str) + if "Rollover Number" in event: + assert all( + key in event + for key in [ + "Event", + "Spark Version", + "Timestamp", + "Rollover Number", + "SparkContext Id", + ] + ), "All keys are present" + assert ( + rollover_count == event["Rollover Number"] + ), "Contiguous monotonically increasing IDs" + rollover_count += 1 + except Exception as exc: + raise ValueError("Problem with line %d: %s" % (i, event_str), exc) + + assert rollover_count == 3, "All log parts are present" + assert i + 1 == 16945, "All events are present" diff --git a/tests/test_parse.py b/tests/test_parse.py new file mode 100644 index 0000000..99a5c01 --- /dev/null +++ b/tests/test_parse.py @@ -0,0 +1,66 @@ +import json +import tempfile +from pathlib import Path + +from spark_log_parser import eventlog +from spark_log_parser.parsing_models.application_model_v2 import sparkApplication + + +def test_simple_databricks_log(): + event_log_path = Path("tests", "logs", "databricks.zip").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + event_log = eventlog.EventLogBuilder(event_log_path.as_uri(), temp_dir).build() + + result_path = str(Path(temp_dir, "result")) + sparkApplication(eventlog=str(event_log)).save(result_path) + + with open(result_path + ".json") as result_fobj: + parsed = json.load(result_fobj) + + assert all( + key in parsed + for key in [ + "accumData", + "executors", + "jobData", + "metadata", + "sqlData", + "stageData", + "taskData", + ] + ), "All keys are present" + + assert ( + parsed["metadata"]["application_info"]["name"] == "Databricks Shell" + ), "Name is as expected" + + +def test_simple_emr_log(): + event_log_path = Path("tests", "logs", "emr.zip").resolve() + + with tempfile.TemporaryDirectory() as temp_dir: + event_log = eventlog.EventLogBuilder(event_log_path.as_uri(), temp_dir).build() + + result_path = str(Path(temp_dir, "result")) + sparkApplication(eventlog=str(event_log)).save(result_path) + + with open(str(result_path) + ".json") as result_fobj: + parsed = json.load(result_fobj) + + assert all( + key in parsed + for key in [ + "accumData", + "executors", + "jobData", + "metadata", + "sqlData", + "stageData", + "taskData", + ] + ), "All keys are present" + + assert ( + parsed["metadata"]["application_info"]["name"] == "Text Similarity" + ), "Name is as expected"