/
app-20180510051218-0000
executable file
·53 lines (53 loc) · 132 KB
/
app-20180510051218-0000
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
{"Event":"SparkListenerLogStart","Spark Version":"2.2.1"}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"172.18.0.6","Port":39403},"Maximum Memory":384093388,"Timestamp":1525929139100,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-oracle/jre","Java Version":"1.8.0_171 (Oracle Corporation)","Scala Version":"version 2.11.8"},"Spark Properties":{"spark.driver.host":"172.18.0.6","spark.history.fs.logDirectory":"file:///opt/spark-events","spark.eventLog.enabled":"true","spark.driver.port":"45347","spark.jars":"file:/opt/Docker_WordCount_Spark-1.0.jar","spark.driver.supervise":"false","spark.app.name":"Docker_Word_Count","spark.scheduler.mode":"FIFO","spark.executor.id":"driver","spark.submit.deployMode":"cluster","spark.master":"spark://08ff0c3256e7:7077","spark.eventLog.dir":"file:///opt/spark-events","spark.app.id":"app-20180510051218-0000"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.vm.specification.version":"1.8","user.home":"*********(redacted)","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-oracle/jre/lib/amd64","user.dir":"*********(redacted)","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.171-b11","java.endorsed.dirs":"/usr/lib/jvm/java-8-oracle/jre/lib/endorsed","java.runtime.version":"1.8.0_171-b11","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-oracle/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"Java(TM) SE Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-oracle/jre/lib/resources.jar:/usr/lib/jvm/java-8-oracle/jre/lib/rt.jar:/usr/lib/jvm/java-8-oracle/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jsse.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jce.jar:/usr/lib/jvm/java-8-oracle/jre/lib/charsets.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jfr.jar:/usr/lib/jvm/java-8-oracle/jre/classes","file.encoding":"ANSI_X3.4-1968","user.timezone":"*********(redacted)","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"4.9.87-linuxkit-aufs","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"*********(redacted)","sun.jnu.encoding":"ANSI_X3.4-1968","user.language":"*********(redacted)","java.vendor.url":"*********(redacted)","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"*********(redacted)","user.name":"*********(redacted)","java.vm.name":"Java HotSpot(TM) 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.worker.DriverWrapper spark://Worker@172.18.0.6:43521 /opt/spark/work/driver-20180510051213-0000/Docker_WordCount_Spark-1.0.jar com.pavanpkulkarni.dockerwordcount.DockerWordCount /opt/sample.txt /opt/output/op1","java.home":"/usr/lib/jvm/java-8-oracle/jre","java.version":"1.8.0_171","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/opt/spark/jars/hive-metastore-1.2.1.spark2.jar":"System Classpath","/opt/spark/jars/hadoop-yarn-server-common-2.7.3.jar":"System Classpath","/opt/spark/jars/xz-1.0.jar":"System Classpath","/opt/spark/jars/zookeeper-3.4.6.jar":"System Classpath","/opt/spark/jars/javax.annotation-api-1.2.jar":"System Classpath","/opt/spark/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/opt/spark/jars/datanucleus-core-3.2.10.jar":"System Classpath","/opt/spark/jars/compress-lzf-1.0.3.jar":"System Classpath","/opt/spark/jars/protobuf-java-2.5.0.jar":"System Classpath","/opt/spark/jars/httpclient-4.5.2.jar":"System Classpath","/opt/spark/jars/jersey-container-servlet-2.22.2.jar":"System Classpath","/opt/spark/jars/jackson-databind-2.6.5.jar":"System Classpath","/opt/spark/jars/leveldbjni-all-1.8.jar":"System Classpath","/opt/spark/jars/lz4-1.3.0.jar":"System Classpath","/opt/spark/jars/metrics-core-3.1.2.jar":"System Classpath","/opt/spark/jars/super-csv-2.2.0.jar":"System Classpath","/opt/spark/jars/javolution-5.5.1.jar":"System Classpath","/opt/spark/jars/datanucleus-api-jdo-3.2.6.jar":"System Classpath","/opt/spark/jars/metrics-json-3.1.2.jar":"System Classpath","/opt/spark/jars/netty-all-4.0.43.Final.jar":"System Classpath","/opt/spark/jars/jtransforms-2.4.0.jar":"System Classpath","/opt/spark/jars/gson-2.2.4.jar":"System Classpath","/opt/spark/jars/xercesImpl-2.9.1.jar":"System Classpath","/opt/spark/jars/ivy-2.4.0.jar":"System Classpath","/opt/spark/jars/calcite-linq4j-1.2.0-incubating.jar":"System Classpath","/opt/spark/jars/scala-parser-combinators_2.11-1.0.4.jar":"System Classpath","/opt/spark/jars/slf4j-log4j12-1.7.16.jar":"System Classpath","/opt/spark/jars/jersey-client-2.22.2.jar":"System Classpath","/opt/spark/jars/jline-2.12.1.jar":"System Classpath","/opt/spark/jars/pmml-model-1.2.15.jar":"System Classpath","/opt/spark/jars/jaxb-api-2.2.2.jar":"System Classpath","/opt/spark/jars/commons-dbcp-1.4.jar":"System Classpath","/opt/spark/jars/hadoop-auth-2.7.3.jar":"System Classpath","/opt/spark/jars/stringtemplate-3.2.1.jar":"System Classpath","/opt/spark/jars/mesos-1.0.0-shaded-protobuf.jar":"System Classpath","/opt/spark/jars/activation-1.1.1.jar":"System Classpath","/opt/spark/jars/base64-2.3.8.jar":"System Classpath","/opt/spark/jars/py4j-0.10.4.jar":"System Classpath","/opt/spark/jars/json4s-core_2.11-3.2.11.jar":"System Classpath","/opt/spark/jars/commons-configuration-1.6.jar":"System Classpath","/opt/spark/jars/spark-unsafe_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/spark-graphx_2.11-2.2.1.jar":"System Classpath","spark://172.18.0.6:45347/jars/Docker_WordCount_Spark-1.0.jar":"*********(redacted)","/opt/spark/jars/parquet-hadoop-1.8.2.jar":"System Classpath","/opt/spark/jars/jetty-util-6.1.26.jar":"System Classpath","/opt/spark/jars/spark-network-shuffle_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/commons-codec-1.10.jar":"System Classpath","/opt/spark/jars/scala-compiler-2.11.8.jar":"System Classpath","/opt/spark/jars/breeze-macros_2.11-0.13.2.jar":"System Classpath","/opt/spark/jars/joda-time-2.9.3.jar":"System Classpath","/opt/spark/jars/commons-compiler-3.0.0.jar":"System Classpath","/opt/spark/jars/spark-mllib_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/pyrolite-4.13.jar":"System Classpath","/opt/spark/jars/calcite-avatica-1.2.0-incubating.jar":"System Classpath","/opt/spark/jars/commons-collections-3.2.2.jar":"System Classpath","/opt/spark/jars/oro-2.0.8.jar":"System Classpath","/opt/spark/jars/htrace-core-3.1.0-incubating.jar":"System Classpath","/opt/spark/jars/hk2-api-2.4.0-b34.jar":"System Classpath","/opt/spark/jars/jcl-over-slf4j-1.7.16.jar":"System Classpath","/opt/spark/jars/httpcore-4.4.4.jar":"System Classpath","/opt/spark/jars/hk2-locator-2.4.0-b34.jar":"System Classpath","/opt/spark/jars/jersey-container-servlet-core-2.22.2.jar":"System Classpath","/opt/spark/jars/hadoop-client-2.7.3.jar":"System Classpath","/opt/spark/jars/commons-pool-1.5.4.jar":"System Classpath","/opt/spark/jars/avro-1.7.7.jar":"System Classpath","/opt/spark/jars/hk2-utils-2.4.0-b34.jar":"System Classpath","/opt/spark/jars/hive-cli-1.2.1.spark2.jar":"System Classpath","/opt/spark/jars/commons-lang-2.6.jar":"System Classpath","/opt/spark/jars/apacheds-kerberos-codec-2.0.0-M15.jar":"System Classpath","/opt/spark/jars/jackson-annotations-2.6.5.jar":"System Classpath","/opt/spark/jars/parquet-format-2.3.1.jar":"System Classpath","/opt/spark/jars/javax.inject-2.4.0-b34.jar":"System Classpath","/opt/spark/jars/mail-1.4.7.jar":"System Classpath","/opt/spark/jars/shapeless_2.11-2.3.2.jar":"System Classpath","/opt/spark/jars/hadoop-annotations-2.7.3.jar":"System Classpath","/opt/spark/jars/jackson-module-paranamer-2.6.5.jar":"System Classpath","/opt/spark/jars/pmml-schema-1.2.15.jar":"System Classpath","/opt/spark/jars/jta-1.1.jar":"System Classpath","/opt/spark/jars/curator-framework-2.6.0.jar":"System Classpath","/opt/spark/jars/spark-sketch_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/jetty-6.1.26.jar":"System Classpath","/opt/spark/jars/jackson-jaxrs-1.9.13.jar":"System Classpath","/opt/spark/jars/spark-sql_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/log4j-1.2.17.jar":"System Classpath","/opt/spark/jars/hadoop-yarn-common-2.7.3.jar":"System Classpath","/opt/spark/jars/guava-14.0.1.jar":"System Classpath","/opt/spark/jars/spark-network-common_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/api-asn1-api-1.0.0-M20.jar":"System Classpath","/opt/spark/jars/parquet-common-1.8.2.jar":"System Classpath","/opt/spark/jars/avro-mapred-1.7.7-hadoop2.jar":"System Classpath","/opt/spark/jars/jsp-api-2.1.jar":"System Classpath","/opt/spark/jars/commons-cli-1.2.jar":"System Classpath","/opt/spark/jars/spark-hive-thriftserver_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/jackson-module-scala_2.11-2.6.5.jar":"System Classpath","/opt/spark/jars/apacheds-i18n-2.0.0-M15.jar":"System Classpath","/opt/spark/jars/commons-beanutils-core-1.8.0.jar":"System Classpath","/opt/spark/jars/jackson-core-2.6.5.jar":"System Classpath","/opt/spark/jars/parquet-hadoop-bundle-1.6.0.jar":"System Classpath","/opt/spark/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/opt/spark/jars/api-util-1.0.0-M20.jar":"System Classpath","/opt/spark/jars/JavaEWAH-0.3.2.jar":"System Classpath","/opt/spark/jars/spire_2.11-0.13.0.jar":"System Classpath","/opt/spark/jars/paranamer-2.6.jar":"System Classpath","/opt/spark/jars/spark-tags_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/commons-lang3-3.5.jar":"System Classpath","/opt/spark/jars/commons-digester-1.8.jar":"System Classpath","/opt/spark/jars/jdo-api-3.0.1.jar":"System Classpath","/opt/spark/jars/commons-httpclient-3.1.jar":"System Classpath","/opt/spark/jars/datanucleus-rdbms-3.2.9.jar":"System Classpath","/opt/spark/jars/javax.inject-1.jar":"System Classpath","/opt/spark/jars/minlog-1.3.0.jar":"System Classpath","/opt/spark/jars/machinist_2.11-0.6.1.jar":"System Classpath","/opt/spark/jars/spark-launcher_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/scala-library-2.11.8.jar":"System Classpath","/opt/spark/jars/spark-yarn_2.11-2.2.1.jar":"System Classpath","/opt/spark/conf/":"System Classpath","/opt/spark/jars/xbean-asm5-shaded-4.4.jar":"System Classpath","/opt/spark/jars/spark-mllib-local_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/hadoop-mapreduce-client-shuffle-2.7.3.jar":"System Classpath","/opt/spark/jars/parquet-encoding-1.8.2.jar":"System Classpath","/opt/spark/jars/hadoop-yarn-api-2.7.3.jar":"System Classpath","/opt/spark/jars/spark-hive_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/scala-xml_2.11-1.0.2.jar":"System Classpath","/opt/spark/jars/eigenbase-properties-1.1.5.jar":"System Classpath","/opt/spark/jars/stax-api-1.0-2.jar":"System Classpath","/opt/spark/jars/chill_2.11-0.8.0.jar":"System Classpath","/opt/spark/jars/spire-macros_2.11-0.13.0.jar":"System Classpath","/opt/spark/jars/macro-compat_2.11-1.1.1.jar":"System Classpath","/opt/spark/jars/antlr-runtime-3.4.jar":"System Classpath","/opt/spark/jars/hive-exec-1.2.1.spark2.jar":"System Classpath","/opt/spark/jars/aopalliance-1.0.jar":"System Classpath","/opt/spark/jars/mx4j-3.0.2.jar":"System Classpath","/opt/spark/jars/commons-io-2.4.jar":"System Classpath","/opt/spark/jars/hadoop-common-2.7.3.jar":"System Classpath","/opt/spark/jars/jodd-core-3.5.2.jar":"System Classpath","/opt/spark/jars/curator-client-2.6.0.jar":"System Classpath","/opt/spark/jars/jackson-xc-1.9.13.jar":"System Classpath","/opt/spark/jars/curator-recipes-2.6.0.jar":"System Classpath","/opt/spark/jars/janino-3.0.0.jar":"System Classpath","/opt/spark/jars/core-1.1.2.jar":"System Classpath","/opt/spark/jars/hadoop-yarn-server-web-proxy-2.7.3.jar":"System Classpath","/opt/spark/jars/jersey-media-jaxb-2.22.2.jar":"System Classpath","/opt/spark/jars/jersey-guava-2.22.2.jar":"System Classpath","/opt/spark/jars/antlr4-runtime-4.5.3.jar":"System Classpath","/opt/spark/jars/spark-core_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/jersey-common-2.22.2.jar":"System Classpath","/opt/spark/jars/stream-2.7.0.jar":"System Classpath","/opt/spark/jars/parquet-column-1.8.2.jar":"System Classpath","/opt/spark/jars/stax-api-1.0.1.jar":"System Classpath","/opt/spark/jars/guice-3.0.jar":"System Classpath","/opt/spark/jars/commons-compress-1.4.1.jar":"System Classpath","/opt/spark/jars/javax.ws.rs-api-2.0.1.jar":"System Classpath","/opt/spark/jars/aopalliance-repackaged-2.4.0-b34.jar":"System Classpath","/opt/spark/jars/RoaringBitmap-0.5.11.jar":"System Classpath","/opt/spark/jars/commons-logging-1.1.3.jar":"System Classpath","/opt/spark/jars/apache-log4j-extras-1.2.17.jar":"System Classpath","/opt/spark/jars/hadoop-mapreduce-client-app-2.7.3.jar":"System Classpath","/opt/spark/jars/hive-beeline-1.2.1.spark2.jar":"System Classpath","/opt/spark/jars/json4s-ast_2.11-3.2.11.jar":"System Classpath","/opt/spark/jars/spark-streaming_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/commons-crypto-1.0.0.jar":"System Classpath","/opt/spark/jars/snappy-0.2.jar":"System Classpath","/opt/spark/jars/json4s-jackson_2.11-3.2.11.jar":"System Classpath","/opt/spark/jars/univocity-parsers-2.2.1.jar":"System Classpath","/opt/spark/jars/arpack_combined_all-0.1.jar":"System Classpath","/opt/spark/jars/jets3t-0.9.3.jar":"System Classpath","/opt/spark/jars/scalap-2.11.8.jar":"System Classpath","/opt/spark/jars/javax.servlet-api-3.1.0.jar":"System Classpath","/opt/spark/jars/spark-repl_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/jpam-1.1.jar":"System Classpath","/opt/spark/jars/commons-beanutils-1.7.0.jar":"System Classpath","/opt/spark/jars/jersey-server-2.22.2.jar":"System Classpath","/opt/spark/jars/opencsv-2.3.jar":"System Classpath","/opt/spark/jars/netty-3.9.9.Final.jar":"System Classpath","/opt/spark/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/opt/spark/jars/antlr-2.7.7.jar":"System Classpath","/opt/spark/jars/slf4j-api-1.7.16.jar":"System Classpath","/opt/spark/jars/javassist-3.18.1-GA.jar":"System Classpath","/opt/spark/jars/hadoop-hdfs-2.7.3.jar":"System Classpath","/opt/spark/jars/xmlenc-0.52.jar":"System Classpath","/opt/spark/jars/metrics-jvm-3.1.2.jar":"System Classpath","/opt/spark/jars/spark-catalyst_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/snappy-java-1.1.2.6.jar":"System Classpath","/opt/spark/jars/ST4-4.0.4.jar":"System Classpath","/opt/spark/jars/commons-net-2.2.jar":"System Classpath","/opt/spark/jars/calcite-core-1.2.0-incubating.jar":"System Classpath","/opt/spark/jars/metrics-graphite-3.1.2.jar":"System Classpath","/opt/spark/jars/chill-java-0.8.0.jar":"System Classpath","/opt/spark/jars/hadoop-mapreduce-client-core-2.7.3.jar":"System Classpath","/opt/spark/jars/java-xmlbuilder-1.0.jar":"System Classpath","/opt/spark/jars/objenesis-2.1.jar":"System Classpath","/opt/spark/jars/commons-math3-3.4.1.jar":"System Classpath","/opt/spark/jars/jsr305-1.3.9.jar":"System Classpath","/opt/spark/jars/parquet-jackson-1.8.2.jar":"System Classpath","/opt/spark/jars/avro-ipc-1.7.7.jar":"System Classpath","/opt/spark/jars/osgi-resource-locator-1.0.1.jar":"System Classpath","/opt/spark/jars/hadoop-yarn-client-2.7.3.jar":"System Classpath","/opt/spark/jars/bcprov-jdk15on-1.51.jar":"System Classpath","/opt/spark/jars/libfb303-0.9.3.jar":"System Classpath","/opt/spark/jars/guice-servlet-3.0.jar":"System Classpath","/opt/spark/jars/derby-10.12.1.1.jar":"System Classpath","/opt/spark/jars/hadoop-mapreduce-client-jobclient-2.7.3.jar":"System Classpath","/opt/spark/jars/validation-api-1.1.0.Final.jar":"System Classpath","/opt/spark/jars/kryo-shaded-3.0.3.jar":"System Classpath","/opt/spark/jars/spark-mesos_2.11-2.2.1.jar":"System Classpath","/opt/spark/jars/breeze_2.11-0.13.2.jar":"System Classpath","/opt/spark/jars/hive-jdbc-1.2.1.spark2.jar":"System Classpath","/opt/spark/jars/scala-reflect-2.11.8.jar":"System Classpath","/opt/spark/jars/jul-to-slf4j-1.7.16.jar":"System Classpath","/opt/spark/jars/libthrift-0.9.3.jar":"System Classpath","/opt/spark/jars/hadoop-mapreduce-client-common-2.7.3.jar":"System Classpath"}}
{"Event":"SparkListenerApplicationStart","App Name":"Docker_Word_Count","App ID":"app-20180510051218-0000","Timestamp":1525929137986,"User":"root"}
{"Event":"SparkListenerExecutorAdded","Timestamp":1525929144703,"Executor ID":"1","Executor Info":{"Host":"172.18.0.4","Total Cores":2,"Log Urls":{"stdout":"http://172.18.0.4:8081/logPage/?appId=app-20180510051218-0000&executorId=1&logType=stdout","stderr":"http://172.18.0.4:8081/logPage/?appId=app-20180510051218-0000&executorId=1&logType=stderr"}}}
{"Event":"SparkListenerExecutorAdded","Timestamp":1525929144852,"Executor ID":"0","Executor Info":{"Host":"172.18.0.5","Total Cores":2,"Log Urls":{"stdout":"http://172.18.0.5:8081/logPage/?appId=app-20180510051218-0000&executorId=0&logType=stdout","stderr":"http://172.18.0.5:8081/logPage/?appId=app-20180510051218-0000&executorId=0&logType=stderr"}}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"172.18.0.4","Port":39193},"Maximum Memory":384093388,"Timestamp":1525929144984,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"172.18.0.5","Port":46171},"Maximum Memory":384093388,"Timestamp":1525929145000,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"show at DockerWordCount.scala:28","details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","physicalPlanDescription":"== Parsed Logical Plan ==\nGlobalLimit 21\n+- LocalLimit 21\n +- Aggregate [value#11], [value#11, count(1) AS count(1)#15L]\n +- AppendColumns <function1>, class java.lang.String, [StructField(value,StringType,true)], cast(value#8 as string).toString, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]\n +- SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8]\n +- MapPartitions <function1>, obj#7: java.lang.String\n +- DeserializeToObject cast(value#0 as string).toString, obj#6: java.lang.String\n +- Relation[value#0] text\n\n== Analyzed Logical Plan ==\nvalue: string, count(1): bigint\nGlobalLimit 21\n+- LocalLimit 21\n +- Aggregate [value#11], [value#11, count(1) AS count(1)#15L]\n +- AppendColumns <function1>, class java.lang.String, [StructField(value,StringType,true)], cast(value#8 as string).toString, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]\n +- SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8]\n +- MapPartitions <function1>, obj#7: java.lang.String\n +- DeserializeToObject cast(value#0 as string).toString, obj#6: java.lang.String\n +- Relation[value#0] text\n\n== Optimized Logical Plan ==\nGlobalLimit 21\n+- LocalLimit 21\n +- Aggregate [value#11], [value#11, count(1) AS count(1)#15L]\n +- Project [value#11]\n +- AppendColumnsWithObject <function1>, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8], [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]\n +- MapPartitions <function1>, obj#7: java.lang.String\n +- DeserializeToObject value#0.toString, obj#6: java.lang.String\n +- Relation[value#0] text\n\n== Physical Plan ==\nCollectLimit 21\n+- *HashAggregate(keys=[value#11], functions=[count(1)], output=[value#11, count(1)#15L])\n +- Exchange hashpartitioning(value#11, 200)\n +- *HashAggregate(keys=[value#11], functions=[partial_count(1)], output=[value#11, count#22L])\n +- *Project [value#11]\n +- AppendColumnsWithObject <function1>, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8], [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]\n +- MapPartitions <function1>, obj#7: java.lang.String\n +- DeserializeToObject value#0.toString, obj#6: java.lang.String\n +- *FileScan text [value#0] Batched: false, Format: Text, Location: InMemoryFileIndex[file:/opt/sample.txt], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<value:string>","sparkPlanInfo":{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#11], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#11, 200)","children":[{"nodeName":"WholeStageCodegen","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#11], functions=[partial_count(1)])","children":[{"nodeName":"Project","simpleString":"Project [value#11]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AppendColumnsWithObject","simpleString":"AppendColumnsWithObject <function1>, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8], [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions <function1>, obj#7: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#0.toString, obj#6: java.lang.String","children":[{"nodeName":"WholeStageCodegen","simpleString":"WholeStageCodegen","children":[{"nodeName":"Scan text ","simpleString":"FileScan text [value#0] Batched: false, Format: Text, Location: InMemoryFileIndex[file:/opt/sample.txt], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<value:string>","children":[],"metadata":{"Location":"InMemoryFileIndex[file:/opt/sample.txt]","ReadSchema":"struct<value:string>","Format":"Text","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]"},"metrics":[{"name":"number of output rows","accumulatorId":13,"metricType":"sum"},{"name":"number of files","accumulatorId":14,"metricType":"sum"},{"name":"metadata time (ms)","accumulatorId":15,"metricType":"sum"},{"name":"scan time total (min, med, max)","accumulatorId":16,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"duration total (min, med, max)","accumulatorId":12,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":8,"metricType":"sum"},{"name":"peak memory total (min, med, max)","accumulatorId":9,"metricType":"size"},{"name":"spill size total (min, med, max)","accumulatorId":10,"metricType":"size"},{"name":"aggregate time total (min, med, max)","accumulatorId":11,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"duration total (min, med, max)","accumulatorId":7,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"data size total (min, med, max)","accumulatorId":1,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3,"metricType":"sum"},{"name":"peak memory total (min, med, max)","accumulatorId":4,"metricType":"size"},{"name":"spill size total (min, med, max)","accumulatorId":5,"metricType":"size"},{"name":"aggregate time total (min, med, max)","accumulatorId":6,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"duration total (min, med, max)","accumulatorId":2,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1525929147557}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[14,1],[15,3]]}
{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1525929148998,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"MapPartitions\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"DeserializeToObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"AppendColumnsWithObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[0],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]}],"Stage IDs":[0,1],"Properties":{"spark.sql.execution.id":"0"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"MapPartitions\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"DeserializeToObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"AppendColumnsWithObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},"Properties":{"spark.sql.execution.id":"0"}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1525929149126,"Executor ID":"1","Host":"172.18.0.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1525929149126,"Executor ID":"1","Host":"172.18.0.4","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1525929153615,"Failed":false,"Killed":false,"Accumulables":[{"ID":1,"Name":"data size total (min, med, max)","Update":"38931","Value":"38930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":11,"Name":"aggregate time total (min, med, max)","Update":"1047","Value":"1046","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":9,"Name":"peak memory total (min, med, max)","Update":"262143","Value":"262142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"1007","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"duration total (min, med, max)","Update":"1216","Value":"1215","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":13,"Name":"number of output rows","Update":"79232","Value":"79232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":12,"Name":"duration total (min, med, max)","Update":"2862","Value":"2861","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"internal.metrics.input.recordsRead","Update":79232,"Value":79232,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.input.bytesRead","Update":2583212,"Value":2583212,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.shuffle.write.writeTime","Update":49639900,"Value":49639900,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1007,"Value":1007,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":33781,"Value":33781,"Internal":true,"Count Failed Values":true},{"ID":26,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":23,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":22,"Name":"internal.metrics.jvmGCTime","Update":123,"Value":123,"Internal":true,"Count Failed Values":true},{"ID":21,"Name":"internal.metrics.resultSize","Update":3295,"Value":3295,"Internal":true,"Count Failed Values":true},{"ID":20,"Name":"internal.metrics.executorCpuTime","Update":2826684513,"Value":2826684513,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.executorRunTime","Update":3484,"Value":3484,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.executorDeserializeCpuTime","Update":559990457,"Value":559990457,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.executorDeserializeTime","Update":908,"Value":908,"Internal":true,"Count Failed Values":true}]},"Task Metrics":{"Executor Deserialize Time":908,"Executor Deserialize CPU Time":559990457,"Executor Run Time":3484,"Executor CPU Time":2826684513,"Result Size":3295,"JVM GC Time":123,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":33781,"Shuffle Write Time":49639900,"Shuffle Records Written":1007},"Input Metrics":{"Bytes Read":2583212,"Records Read":79232},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[{"Block ID":"broadcast_1_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":12251,"Disk Size":0}},{"Block ID":"broadcast_1","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":27608,"Disk Size":0}},{"Block ID":"broadcast_0_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":24029,"Disk Size":0}},{"Block ID":"broadcast_0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":335512,"Disk Size":0}}]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"MapPartitions\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"DeserializeToObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"AppendColumnsWithObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Submission Time":1525929149120,"Completion Time":1525929153635,"Accumulables":[{"ID":23,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"number of output rows","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":17,"Name":"internal.metrics.executorDeserializeTime","Value":908,"Internal":true,"Count Failed Values":true},{"ID":26,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1007,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"aggregate time total (min, med, max)","Value":"1046","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"internal.metrics.input.recordsRead","Value":79232,"Internal":true,"Count Failed Values":true},{"ID":20,"Name":"internal.metrics.executorCpuTime","Value":2826684513,"Internal":true,"Count Failed Values":true},{"ID":13,"Name":"number of output rows","Value":"79232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":22,"Name":"internal.metrics.jvmGCTime","Value":123,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"duration total (min, med, max)","Value":"1215","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":33781,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"data size total (min, med, max)","Value":"38930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":37,"Name":"internal.metrics.input.bytesRead","Value":2583212,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.executorRunTime","Value":3484,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"peak memory total (min, med, max)","Value":"262142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":18,"Name":"internal.metrics.executorDeserializeCpuTime","Value":559990457,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.shuffle.write.writeTime","Value":49639900,"Internal":true,"Count Failed Values":true},{"ID":12,"Name":"duration total (min, med, max)","Value":"2861","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":21,"Name":"internal.metrics.resultSize","Value":3295,"Internal":true,"Count Failed Values":true}]}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[0],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Submission Time":1525929153723,"Accumulables":[]},"Properties":{"spark.sql.execution.id":"0"}}
{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1525929153740,"Executor ID":"1","Host":"172.18.0.4","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Launch Time":1525929153740,"Executor ID":"1","Host":"172.18.0.4","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1525929153981,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"aggregate time total (min, med, max)","Update":"14","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4,"Name":"peak memory total (min, med, max)","Update":"8650751","Value":"8650750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2,"Name":"duration total (min, med, max)","Update":"15","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":81,"Name":"internal.metrics.shuffle.read.recordsRead","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":278,"Value":278,"Internal":true,"Count Failed Values":true},{"ID":78,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":77,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":76,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Update":8650752,"Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Update":3571,"Value":3571,"Internal":true,"Count Failed Values":true},{"ID":68,"Name":"internal.metrics.executorCpuTime","Update":101336812,"Value":101336812,"Internal":true,"Count Failed Values":true},{"ID":67,"Name":"internal.metrics.executorRunTime","Update":168,"Value":168,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Update":25342838,"Value":25342838,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Metrics":{"Executor Deserialize Time":57,"Executor Deserialize CPU Time":25342838,"Executor Run Time":168,"Executor CPU Time":101336812,"Result Size":3571,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Local Bytes Read":278,"Total Records Read":11},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[{"Block ID":"broadcast_2_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":11356,"Disk Size":0}},{"Block ID":"broadcast_2","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":23736,"Disk Size":0}}]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[0],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Submission Time":1525929153723,"Completion Time":1525929153983,"Accumulables":[{"ID":68,"Name":"internal.metrics.executorCpuTime","Value":101336812,"Internal":true,"Count Failed Values":true},{"ID":77,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.peakExecutionMemory","Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"duration total (min, med, max)","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"internal.metrics.executorDeserializeTime","Value":57,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"peak memory total (min, med, max)","Value":"8650750","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"internal.metrics.executorRunTime","Value":168,"Internal":true,"Count Failed Values":true},{"ID":76,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":278,"Internal":true,"Count Failed Values":true},{"ID":69,"Name":"internal.metrics.resultSize","Value":3571,"Internal":true,"Count Failed Values":true},{"ID":78,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.shuffle.read.recordsRead","Value":11,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"internal.metrics.executorDeserializeCpuTime","Value":25342838,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"aggregate time total (min, med, max)","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"}]}}
{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1525929153990,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1525929154065,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"MapPartitions\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"DeserializeToObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"AppendColumnsWithObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[2],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]}],"Stage IDs":[2,3],"Properties":{"spark.sql.execution.id":"0"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[2],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},"Properties":{"spark.sql.execution.id":"0"}}
{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Launch Time":1525929154129,"Executor ID":"1","Host":"172.18.0.4","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Launch Time":1525929154129,"Executor ID":"1","Host":"172.18.0.4","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1525929154202,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"aggregate time total (min, med, max)","Update":"2","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4,"Name":"peak memory total (min, med, max)","Update":"8650751","Value":"17301501","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3,"Name":"number of output rows","Update":"5","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2,"Name":"duration total (min, med, max)","Update":"2","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":105,"Name":"internal.metrics.shuffle.read.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":103,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":154,"Value":154,"Internal":true,"Count Failed Values":true},{"ID":102,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":101,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":98,"Name":"internal.metrics.peakExecutionMemory","Update":8650752,"Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":93,"Name":"internal.metrics.resultSize","Update":3453,"Value":3453,"Internal":true,"Count Failed Values":true},{"ID":92,"Name":"internal.metrics.executorCpuTime","Update":5374747,"Value":5374747,"Internal":true,"Count Failed Values":true},{"ID":91,"Name":"internal.metrics.executorRunTime","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":90,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7589071,"Value":7589071,"Internal":true,"Count Failed Values":true},{"ID":89,"Name":"internal.metrics.executorDeserializeTime","Update":39,"Value":39,"Internal":true,"Count Failed Values":true}]},"Task Metrics":{"Executor Deserialize Time":39,"Executor Deserialize CPU Time":7589071,"Executor Run Time":5,"Executor CPU Time":5374747,"Result Size":3453,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Local Bytes Read":154,"Total Records Read":5},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[{"Block ID":"broadcast_3_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":11350,"Disk Size":0}},{"Block ID":"broadcast_3","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":23736,"Disk Size":0}}]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[2],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Submission Time":1525929154124,"Completion Time":1525929154203,"Accumulables":[{"ID":92,"Name":"internal.metrics.executorCpuTime","Value":5374747,"Internal":true,"Count Failed Values":true},{"ID":101,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":89,"Name":"internal.metrics.executorDeserializeTime","Value":39,"Internal":true,"Count Failed Values":true},{"ID":98,"Name":"internal.metrics.peakExecutionMemory","Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"duration total (min, med, max)","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":91,"Name":"internal.metrics.executorRunTime","Value":5,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"peak memory total (min, med, max)","Value":"17301501","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":154,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.shuffle.read.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":90,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7589071,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"number of output rows","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":93,"Name":"internal.metrics.resultSize","Value":3453,"Internal":true,"Count Failed Values":true},{"ID":102,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"aggregate time total (min, med, max)","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"}]}}
{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1525929154203,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1525929154223,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"MapPartitions\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"DeserializeToObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"AppendColumnsWithObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]}],"Stage IDs":[5,4],"Properties":{"spark.sql.execution.id":"0"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},"Properties":{"spark.sql.execution.id":"0"}}
{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Launch Time":1525929154236,"Executor ID":"1","Host":"172.18.0.4","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Launch Time":1525929154236,"Executor ID":"1","Host":"172.18.0.4","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1525929154301,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"aggregate time total (min, med, max)","Update":"0","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4,"Name":"peak memory total (min, med, max)","Update":"8650751","Value":"25952252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3,"Name":"number of output rows","Update":"4","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2,"Name":"duration total (min, med, max)","Update":"1","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"internal.metrics.shuffle.read.recordsRead","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":128,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":127,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":141,"Value":141,"Internal":true,"Count Failed Values":true},{"ID":126,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.peakExecutionMemory","Update":8650752,"Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":117,"Name":"internal.metrics.resultSize","Update":3446,"Value":3446,"Internal":true,"Count Failed Values":true},{"ID":116,"Name":"internal.metrics.executorCpuTime","Update":4791879,"Value":4791879,"Internal":true,"Count Failed Values":true},{"ID":115,"Name":"internal.metrics.executorRunTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7438355,"Value":7438355,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.executorDeserializeTime","Update":30,"Value":30,"Internal":true,"Count Failed Values":true}]},"Task Metrics":{"Executor Deserialize Time":30,"Executor Deserialize CPU Time":7438355,"Executor Run Time":12,"Executor CPU Time":4791879,"Result Size":3446,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Local Bytes Read":141,"Total Records Read":4},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[{"Block ID":"broadcast_4_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":11349,"Disk Size":0}},{"Block ID":"broadcast_4","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":23736,"Disk Size":0}}]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Submission Time":1525929154231,"Completion Time":1525929154303,"Accumulables":[{"ID":128,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.peakExecutionMemory","Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":113,"Name":"internal.metrics.executorDeserializeTime","Value":30,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":116,"Name":"internal.metrics.executorCpuTime","Value":4791879,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"duration total (min, med, max)","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":124,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":115,"Name":"internal.metrics.executorRunTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":127,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":141,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"peak memory total (min, med, max)","Value":"25952252","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":129,"Name":"internal.metrics.shuffle.read.recordsRead","Value":4,"Internal":true,"Count Failed Values":true},{"ID":114,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7438355,"Internal":true,"Count Failed Values":true},{"ID":117,"Name":"internal.metrics.resultSize","Value":3446,"Internal":true,"Count Failed Values":true},{"ID":126,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"aggregate time total (min, med, max)","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"}]}}
{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1525929154303,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1525929154308,"Stage Infos":[{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"MapPartitions\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"DeserializeToObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"11\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"AppendColumnsWithObject\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]}],"Stage IDs":[6,7],"Properties":{"spark.sql.execution.id":"0"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},"Properties":{"spark.sql.execution.id":"0"}}
{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Launch Time":1525929154316,"Executor ID":"1","Host":"172.18.0.4","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Launch Time":1525929154316,"Executor ID":"1","Host":"172.18.0.4","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1525929154360,"Failed":false,"Killed":false,"Accumulables":[{"ID":6,"Name":"aggregate time total (min, med, max)","Update":"0","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4,"Name":"peak memory total (min, med, max)","Update":"8650751","Value":"34603003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3,"Name":"number of output rows","Update":"4","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2,"Name":"duration total (min, med, max)","Update":"1","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.shuffle.read.recordsRead","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":152,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":151,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":155,"Value":155,"Internal":true,"Count Failed Values":true},{"ID":150,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.peakExecutionMemory","Update":8650752,"Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.resultSize","Update":3464,"Value":3464,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.executorCpuTime","Update":3477883,"Value":3477883,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.executorRunTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6107315,"Value":6107315,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.executorDeserializeTime","Update":26,"Value":26,"Internal":true,"Count Failed Values":true}]},"Task Metrics":{"Executor Deserialize Time":26,"Executor Deserialize CPU Time":6107315,"Executor Run Time":4,"Executor CPU Time":3477883,"Result Size":3464,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Local Bytes Read":155,"Total Records Read":4},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[{"Block ID":"broadcast_5_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":11349,"Disk Size":0}},{"Block ID":"broadcast_5","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":23736,"Disk Size":0}}]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"show at DockerWordCount.scala:28","Number of Tasks":1,"RDD Info":[{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"mapPartitionsInternal\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen\"}","Callsite":"show at DockerWordCount.scala:28","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.Dataset.show(Dataset.scala:605)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:28)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Submission Time":1525929154316,"Completion Time":1525929154366,"Accumulables":[{"ID":137,"Name":"internal.metrics.executorDeserializeTime","Value":26,"Internal":true,"Count Failed Values":true},{"ID":146,"Name":"internal.metrics.peakExecutionMemory","Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.executorCpuTime","Value":3477883,"Internal":true,"Count Failed Values":true},{"ID":149,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"duration total (min, med, max)","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":151,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":155,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"peak memory total (min, med, max)","Value":"34603003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":139,"Name":"internal.metrics.executorRunTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":148,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":138,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6107315,"Internal":true,"Count Failed Values":true},{"ID":150,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.resultSize","Value":3464,"Internal":true,"Count Failed Values":true},{"ID":153,"Name":"internal.metrics.shuffle.read.recordsRead","Value":4,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"number of output rows","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":6,"Name":"aggregate time total (min, med, max)","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":152,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true}]}}
{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1525929154366,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1525929154371}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"csv at DockerWordCount.scala:30","details":"org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:597)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:30)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","physicalPlanDescription":"== Parsed Logical Plan ==\nRepartition 1, false\n+- Aggregate [value#11], [value#11, count(1) AS count(1)#15L]\n +- AppendColumns <function1>, class java.lang.String, [StructField(value,StringType,true)], cast(value#8 as string).toString, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]\n +- SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8]\n +- MapPartitions <function1>, obj#7: java.lang.String\n +- DeserializeToObject cast(value#0 as string).toString, obj#6: java.lang.String\n +- Relation[value#0] text\n\n== Analyzed Logical Plan ==\nvalue: string, count(1): bigint\nRepartition 1, false\n+- Aggregate [value#11], [value#11, count(1) AS count(1)#15L]\n +- AppendColumns <function1>, class java.lang.String, [StructField(value,StringType,true)], cast(value#8 as string).toString, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]\n +- SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8]\n +- MapPartitions <function1>, obj#7: java.lang.String\n +- DeserializeToObject cast(value#0 as string).toString, obj#6: java.lang.String\n +- Relation[value#0] text\n\n== Optimized Logical Plan ==\nRepartition 1, false\n+- Aggregate [value#11], [value#11, count(1) AS count(1)#15L]\n +- Project [value#11]\n +- AppendColumnsWithObject <function1>, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8], [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]\n +- MapPartitions <function1>, obj#7: java.lang.String\n +- DeserializeToObject value#0.toString, obj#6: java.lang.String\n +- Relation[value#0] text\n\n== Physical Plan ==\nCoalesce 1\n+- *HashAggregate(keys=[value#11], functions=[count(1)], output=[value#11, count(1)#15L])\n +- Exchange hashpartitioning(value#11, 200)\n +- *HashAggregate(keys=[value#11], functions=[partial_count(1)], output=[value#11, count#22L])\n +- *Project [value#11]\n +- AppendColumnsWithObject <function1>, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8], [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]\n +- MapPartitions <function1>, obj#7: java.lang.String\n +- DeserializeToObject value#0.toString, obj#6: java.lang.String\n +- *FileScan text [value#0] Batched: false, Format: Text, Location: InMemoryFileIndex[file:/opt/sample.txt], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<value:string>","sparkPlanInfo":{"nodeName":"Coalesce","simpleString":"Coalesce 1","children":[{"nodeName":"WholeStageCodegen","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#11], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(value#11, 200)","children":[{"nodeName":"WholeStageCodegen","simpleString":"WholeStageCodegen","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[value#11], functions=[partial_count(1)])","children":[{"nodeName":"Project","simpleString":"Project [value#11]","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AppendColumnsWithObject","simpleString":"AppendColumnsWithObject <function1>, [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#8], [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true) AS value#11]","children":[{"nodeName":"MapPartitions","simpleString":"MapPartitions <function1>, obj#7: java.lang.String","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject value#0.toString, obj#6: java.lang.String","children":[{"nodeName":"WholeStageCodegen","simpleString":"WholeStageCodegen","children":[{"nodeName":"Scan text ","simpleString":"FileScan text [value#0] Batched: false, Format: Text, Location: InMemoryFileIndex[file:/opt/sample.txt], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<value:string>","children":[],"metadata":{"Location":"InMemoryFileIndex[file:/opt/sample.txt]","ReadSchema":"struct<value:string>","Format":"Text","Batched":"false","PartitionFilters":"[]","PushedFilters":"[]"},"metrics":[{"name":"number of output rows","accumulatorId":174,"metricType":"sum"},{"name":"number of files","accumulatorId":175,"metricType":"sum"},{"name":"metadata time (ms)","accumulatorId":176,"metricType":"sum"},{"name":"scan time total (min, med, max)","accumulatorId":177,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"duration total (min, med, max)","accumulatorId":173,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":169,"metricType":"sum"},{"name":"peak memory total (min, med, max)","accumulatorId":170,"metricType":"size"},{"name":"spill size total (min, med, max)","accumulatorId":171,"metricType":"size"},{"name":"aggregate time total (min, med, max)","accumulatorId":172,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"duration total (min, med, max)","accumulatorId":168,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"data size total (min, med, max)","accumulatorId":162,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":164,"metricType":"sum"},{"name":"peak memory total (min, med, max)","accumulatorId":165,"metricType":"size"},{"name":"spill size total (min, med, max)","accumulatorId":166,"metricType":"size"},{"name":"aggregate time total (min, med, max)","accumulatorId":167,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"duration total (min, med, max)","accumulatorId":163,"metricType":"timing"}]}],"metadata":{},"metrics":[]},"time":1525929154589}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[175,1],[176,0]]}
{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1525929154935,"Stage Infos":[{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"csv at DockerWordCount.scala:30","Number of Tasks":1,"RDD Info":[{"RDD ID":19,"Name":"CoalescedRDD","Scope":"{\"id\":\"29\",\"name\":\"Coalesce\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":17,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"33\",\"name\":\"Exchange\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[8],"Details":"org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:597)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:30)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"csv at DockerWordCount.scala:30","Number of Tasks":1,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"33\",\"name\":\"Exchange\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"FileScanRDD","Scope":"{\"id\":\"41\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"38\",\"name\":\"AppendColumnsWithObject\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"DeserializeToObject\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"34\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"MapPartitions\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:597)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:30)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]}],"Stage IDs":[9,8],"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"28\",\"name\":\"ExecutedCommand\"}","spark.sql.execution.id":"1"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"csv at DockerWordCount.scala:30","Number of Tasks":1,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"33\",\"name\":\"Exchange\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"FileScanRDD","Scope":"{\"id\":\"41\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"38\",\"name\":\"AppendColumnsWithObject\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"DeserializeToObject\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"34\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"MapPartitions\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:597)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:30)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Submission Time":1525929154941,"Accumulables":[]},"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"28\",\"name\":\"ExecutedCommand\"}","spark.sql.execution.id":"1"}}
{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Launch Time":1525929154941,"Executor ID":"0","Host":"172.18.0.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Launch Time":1525929154941,"Executor ID":"0","Host":"172.18.0.5","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1525929159770,"Failed":false,"Killed":false,"Accumulables":[{"ID":162,"Name":"data size total (min, med, max)","Update":"38931","Value":"38930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":172,"Name":"aggregate time total (min, med, max)","Update":"1077","Value":"1076","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"peak memory total (min, med, max)","Update":"262143","Value":"262142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":169,"Name":"number of output rows","Update":"1007","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":168,"Name":"duration total (min, med, max)","Update":"1292","Value":"1291","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":174,"Name":"number of output rows","Update":"79232","Value":"79232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":173,"Name":"duration total (min, med, max)","Update":"2913","Value":"2912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":199,"Name":"internal.metrics.input.recordsRead","Update":79232,"Value":79232,"Internal":true,"Count Failed Values":true},{"ID":198,"Name":"internal.metrics.input.bytesRead","Update":2583212,"Value":2583212,"Internal":true,"Count Failed Values":true},{"ID":197,"Name":"internal.metrics.shuffle.write.writeTime","Update":74137400,"Value":74137400,"Internal":true,"Count Failed Values":true},{"ID":196,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1007,"Value":1007,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":33781,"Value":33781,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Update":282,"Value":282,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":3295,"Value":3295,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":2942865521,"Value":2942865521,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":3813,"Value":3813,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":594081549,"Value":594081549,"Internal":true,"Count Failed Values":true},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":943,"Value":943,"Internal":true,"Count Failed Values":true}]},"Task Metrics":{"Executor Deserialize Time":943,"Executor Deserialize CPU Time":594081549,"Executor Run Time":3813,"Executor CPU Time":2942865521,"Result Size":3295,"JVM GC Time":282,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":33781,"Shuffle Write Time":74137400,"Shuffle Records Written":1007},"Input Metrics":{"Bytes Read":2583212,"Records Read":79232},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[{"Block ID":"broadcast_7_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":12253,"Disk Size":0}},{"Block ID":"broadcast_7","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":27608,"Disk Size":0}},{"Block ID":"broadcast_6_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":24029,"Disk Size":0}},{"Block ID":"broadcast_6","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":335512,"Disk Size":0}}]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"csv at DockerWordCount.scala:30","Number of Tasks":1,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"33\",\"name\":\"Exchange\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"FileScanRDD","Scope":"{\"id\":\"41\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"38\",\"name\":\"AppendColumnsWithObject\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"DeserializeToObject\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"34\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"MapPartitions\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"41\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:597)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:30)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Submission Time":1525929154941,"Completion Time":1525929159777,"Accumulables":[{"ID":173,"Name":"duration total (min, med, max)","Value":"2912","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":182,"Name":"internal.metrics.resultSize","Value":3295,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"number of output rows","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Value":943,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":196,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1007,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"aggregate time total (min, med, max)","Value":"1076","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":181,"Name":"internal.metrics.executorCpuTime","Value":2942865521,"Internal":true,"Count Failed Values":true},{"ID":199,"Name":"internal.metrics.input.recordsRead","Value":79232,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Value":3813,"Internal":true,"Count Failed Values":true},{"ID":198,"Name":"internal.metrics.input.bytesRead","Value":2583212,"Internal":true,"Count Failed Values":true},{"ID":174,"Name":"number of output rows","Value":"79232","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":183,"Name":"internal.metrics.jvmGCTime","Value":282,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"duration total (min, med, max)","Value":"1291","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":195,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":33781,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"data size total (min, med, max)","Value":"38930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":170,"Name":"peak memory total (min, med, max)","Value":"262142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Value":594081549,"Internal":true,"Count Failed Values":true},{"ID":197,"Name":"internal.metrics.shuffle.write.writeTime","Value":74137400,"Internal":true,"Count Failed Values":true}]}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"csv at DockerWordCount.scala:30","Number of Tasks":1,"RDD Info":[{"RDD ID":19,"Name":"CoalescedRDD","Scope":"{\"id\":\"29\",\"name\":\"Coalesce\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":17,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"33\",\"name\":\"Exchange\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[8],"Details":"org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:597)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:30)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Accumulables":[]},"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"28\",\"name\":\"ExecutedCommand\"}","spark.sql.execution.id":"1"}}
{"Event":"SparkListenerTaskStart","Stage ID":9,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Launch Time":1525929159869,"Executor ID":"0","Host":"172.18.0.5","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":9,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Launch Time":1525929159869,"Executor ID":"0","Host":"172.18.0.5","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1525929161261,"Failed":false,"Killed":false,"Accumulables":[{"ID":167,"Name":"aggregate time total (min, med, max)","Update":"404","Value":"403","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":165,"Name":"peak memory total (min, med, max)","Update":"1730150399","Value":"1730150398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":164,"Name":"number of output rows","Update":"1007","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":163,"Name":"duration total (min, med, max)","Update":"509","Value":"508","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":218,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1007,"Value":1007,"Internal":true,"Count Failed Values":true},{"ID":217,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":216,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":33781,"Value":33781,"Internal":true,"Count Failed Values":true},{"ID":215,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":214,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":200,"Value":200,"Internal":true,"Count Failed Values":true},{"ID":213,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":211,"Name":"internal.metrics.peakExecutionMemory","Update":1730150400,"Value":1730150400,"Internal":true,"Count Failed Values":true},{"ID":208,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.jvmGCTime","Update":28,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.resultSize","Update":3827,"Value":3827,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.executorCpuTime","Update":642040066,"Value":642040066,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.executorRunTime","Update":1236,"Value":1236,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.executorDeserializeCpuTime","Update":88488150,"Value":88488150,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.executorDeserializeTime","Update":138,"Value":138,"Internal":true,"Count Failed Values":true}]},"Task Metrics":{"Executor Deserialize Time":138,"Executor Deserialize CPU Time":88488150,"Executor Run Time":1236,"Executor CPU Time":642040066,"Result Size":3827,"JVM GC Time":28,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":200,"Fetch Wait Time":2,"Remote Bytes Read":0,"Local Bytes Read":33781,"Total Records Read":1007},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[{"Block ID":"broadcast_8_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":37574,"Disk Size":0}},{"Block ID":"broadcast_8","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":97912,"Disk Size":0}}]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"csv at DockerWordCount.scala:30","Number of Tasks":1,"RDD Info":[{"RDD ID":19,"Name":"CoalescedRDD","Scope":"{\"id\":\"29\",\"name\":\"Coalesce\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":17,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"33\",\"name\":\"Exchange\"}","Callsite":"csv at DockerWordCount.scala:30","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Number of Partitions":200,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[8],"Details":"org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:597)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount$.main(DockerWordCount.scala:30)\ncom.pavanpkulkarni.dockerwordcount.DockerWordCount.main(DockerWordCount.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.worker.DriverWrapper$.main(DriverWrapper.scala:58)\norg.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala)","Submission Time":1525929159871,"Completion Time":1525929161262,"Accumulables":[{"ID":164,"Name":"number of output rows","Value":"1007","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":218,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1007,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"aggregate time total (min, med, max)","Value":"403","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":202,"Name":"internal.metrics.executorDeserializeTime","Value":138,"Internal":true,"Count Failed Values":true},{"ID":211,"Name":"internal.metrics.peakExecutionMemory","Value":1730150400,"Internal":true,"Count Failed Values":true},{"ID":205,"Name":"internal.metrics.executorCpuTime","Value":642040066,"Internal":true,"Count Failed Values":true},{"ID":214,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":200,"Internal":true,"Count Failed Values":true},{"ID":163,"Name":"duration total (min, med, max)","Value":"508","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":217,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":208,"Name":"internal.metrics.resultSerializationTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":207,"Name":"internal.metrics.jvmGCTime","Value":28,"Internal":true,"Count Failed Values":true},{"ID":216,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":33781,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"peak memory total (min, med, max)","Value":"1730150398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":213,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":204,"Name":"internal.metrics.executorRunTime","Value":1236,"Internal":true,"Count Failed Values":true},{"ID":203,"Name":"internal.metrics.executorDeserializeCpuTime","Value":88488150,"Internal":true,"Count Failed Values":true},{"ID":215,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":206,"Name":"internal.metrics.resultSize","Value":3827,"Internal":true,"Count Failed Values":true}]}}
{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1525929161262,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1525929161323}
{"Event":"SparkListenerApplicationEnd","Timestamp":1525929161342}