08_CML_Hive_Integration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5383f489-d054-46e0-947c-b9c3232cc6cb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24/01/22 17:54:38 WARN SparkConf: The configuration key 'spark.yarn.access.hadoopFileSystems' has been deprecated as of Spark 3.0 and may be removed in the future. Please use the new key 'spark.kerberos.access.hadoopFileSystems' instead.\n",
      "24/01/22 17:54:38 WARN SparkConf: The configuration key 'spark.yarn.access.hadoopFileSystems' has been deprecated as of Spark 3.0 and may be removed in the future. Please use the new key 'spark.kerberos.access.hadoopFileSystems' instead.\n",
      "Setting spark.hadoop.yarn.resourcemanager.principal to user050\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "24/01/22 17:54:39 WARN SparkConf: The configuration key 'spark.yarn.access.hadoopFileSystems' has been deprecated as of Spark 3.0 and may be removed in the future. Please use the new key 'spark.kerberos.access.hadoopFileSystems' instead.\n",
      "24/01/22 17:54:39 WARN SparkConf: The configuration key 'spark.yarn.access.hadoopFileSystems' has been deprecated as of Spark 3.0 and may be removed in the future. Please use the new key 'spark.kerberos.access.hadoopFileSystems' instead.\n",
      "24/01/22 17:54:39 WARN SparkConf: The configuration key 'spark.yarn.access.hadoopFileSystems' has been deprecated as of Spark 3.0 and may be removed in the future. Please use the new key 'spark.kerberos.access.hadoopFileSystems' instead.\n",
      "24/01/22 17:54:41 WARN JettyUtils: GET /jobs/ failed: java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:41 WARN HttpChannel: /jobs/\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:42 WARN MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-s3a-file-system.properties,hadoop-metrics2.properties\n",
      "24/01/22 17:54:42 WARN JettyUtils: GET /jobs/ failed: java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)\n",
      "\tat org.sparkproject.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:42 WARN HttpChannel: /jobs/\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)\n",
      "\tat org.sparkproject.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:43 WARN JettyUtils: GET /jobs/ failed: java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)\n",
      "\tat org.sparkproject.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:43 WARN HttpChannel: /jobs/\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)\n",
      "\tat org.sparkproject.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:44 WARN HiveServer2CredentialProvider: Failed to get HS2 delegation token\n",
      "java.util.NoSuchElementException: spark.sql.hive.hiveserver2.jdbc.url\n",
      "\tat org.apache.spark.SparkConf.$anonfun$get$1(SparkConf.scala:245)\n",
      "\tat scala.Option.getOrElse(Option.scala:189)\n",
      "\tat org.apache.spark.SparkConf.get(SparkConf.scala:245)\n",
      "\tat com.hortonworks.spark.deploy.yarn.security.HiveServer2CredentialProvider.obtainDelegationTokens(HiveServer2CredentialProvider.scala:64)\n",
      "\tat org.apache.spark.deploy.security.HadoopDelegationTokenManager.$anonfun$obtainDelegationTokens$2(HadoopDelegationTokenManager.scala:164)\n",
      "\tat scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245)\n",
      "\tat scala.collection.Iterator.foreach(Iterator.scala:941)\n",
      "\tat scala.collection.Iterator.foreach$(Iterator.scala:941)\n",
      "\tat scala.collection.AbstractIterator.foreach(Iterator.scala:1429)\n",
      "\tat scala.collection.MapLike$DefaultValuesIterable.foreach(MapLike.scala:213)\n",
      "\tat scala.collection.TraversableLike.flatMap(TraversableLike.scala:245)\n",
      "\tat scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242)\n",
      "\tat scala.collection.AbstractTraversable.flatMap(Traversable.scala:108)\n",
      "\tat org.apache.spark.deploy.security.HadoopDelegationTokenManager.org$apache$spark$deploy$security$HadoopDelegationTokenManager$$obtainDelegationTokens(HadoopDelegationTokenManager.scala:162)\n",
      "\tat org.apache.spark.deploy.security.HadoopDelegationTokenManager$$anon$4.run(HadoopDelegationTokenManager.scala:226)\n",
      "\tat org.apache.spark.deploy.security.HadoopDelegationTokenManager$$anon$4.run(HadoopDelegationTokenManager.scala:224)\n",
      "\tat java.security.AccessController.doPrivileged(Native Method)\n",
      "\tat javax.security.auth.Subject.doAs(Subject.java:422)\n",
      "\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)\n",
      "\tat org.apache.spark.deploy.security.HadoopDelegationTokenManager.obtainTokensAndScheduleRenewal(HadoopDelegationTokenManager.scala:224)\n",
      "\tat org.apache.spark.deploy.security.HadoopDelegationTokenManager.org$apache$spark$deploy$security$HadoopDelegationTokenManager$$updateTokensTask(HadoopDelegationTokenManager.scala:198)\n",
      "\tat org.apache.spark.deploy.security.HadoopDelegationTokenManager.start(HadoopDelegationTokenManager.scala:123)\n",
      "\tat org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.$anonfun$start$1(CoarseGrainedSchedulerBackend.scala:552)\n",
      "\tat org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.$anonfun$start$1$adapted(CoarseGrainedSchedulerBackend.scala:549)\n",
      "\tat scala.Option.foreach(Option.scala:407)\n",
      "\tat org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.start(CoarseGrainedSchedulerBackend.scala:549)\n",
      "\tat org.apache.spark.scheduler.cluster.k8s.KubernetesClusterSchedulerBackend.start(KubernetesClusterSchedulerBackend.scala:95)\n",
      "\tat org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:220)\n",
      "\tat org.apache.spark.SparkContext.<init>(SparkContext.scala:581)\n",
      "\tat org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)\n",
      "\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n",
      "\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n",
      "\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n",
      "\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n",
      "\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)\n",
      "\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n",
      "\tat py4j.Gateway.invoke(Gateway.java:238)\n",
      "\tat py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)\n",
      "\tat py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)\n",
      "\tat py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\n",
      "\tat py4j.ClientServerConnection.run(ClientServerConnection.java:106)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:44 WARN JettyUtils: GET /jobs/ failed: java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)\n",
      "\tat org.sparkproject.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:44 WARN HttpChannel: /jobs/\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)\n",
      "\tat org.sparkproject.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:45 WARN ExecutorAllocationManager: Dynamic allocation without a shuffle service is an experimental feature.\n",
      "24/01/22 17:54:45 WARN JettyUtils: GET /jobs/ failed: java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)\n",
      "\tat org.sparkproject.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:45 WARN HttpChannel: /jobs/\n",
      "java.util.NoSuchElementException: Failed to get the application information. If you are starting up Spark, please wait a while until it's ready.\n",
      "\tat org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:51)\n",
      "\tat org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:276)\n",
      "\tat org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:90)\n",
      "\tat org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:81)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:687)\n",
      "\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:791)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)\n",
      "\tat org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)\n",
      "\tat org.sparkproject.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1435)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n",
      "\tat org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1350)\n",
      "\tat org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n",
      "\tat org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)\n",
      "\tat org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.apache.spark.ui.ProxyRedirectHandler.handle(JettyUtils.scala:581)\n",
      "\tat org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n",
      "\tat org.sparkproject.jetty.server.Server.handle(Server.java:516)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)\n",
      "\tat org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:380)\n",
      "\tat org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n",
      "\tat org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n",
      "\tat org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:105)\n",
      "\tat org.sparkproject.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)\n",
      "\tat org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)\n",
      "\tat org.sparkproject.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)\n",
      "\tat org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "24/01/22 17:54:49 WARN HiveClientImpl: Detected HiveConf hive.execution.engine is 'tez' and will be reset to 'mr' to disable useless hive logic\n",
      "Hive Session ID = fc93a843-278a-4325-8cb1-1704dbe5876f\n",
      "24/01/22 17:54:51 WARN SparkConf: The configuration key 'spark.yarn.access.hadoopFileSystems' has been deprecated as of Spark 3.0 and may be removed in the future. Please use the new key 'spark.kerberos.access.hadoopFileSystems' instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+\n",
      "|         namespace|\n",
      "+------------------+\n",
      "|           default|\n",
      "|information_schema|\n",
      "|               sys|\n",
      "+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import cml.data_v1 as cmldata\n",
    "\n",
    "# Sample in-code customization of spark configurations\n",
    "#from pyspark import SparkContext\n",
    "#SparkContext.setSystemProperty('spark.executor.cores', '1')\n",
    "#SparkContext.setSystemProperty('spark.executor.memory', '2g')\n",
    "\n",
    "CONNECTION_NAME = \"sonaemc-dl\"\n",
    "conn = cmldata.get_connection(CONNECTION_NAME)\n",
    "spark = conn.get_spark_session()\n",
    "\n",
    "# Sample usage to run query through spark\n",
    "EXAMPLE_SQL_QUERY = \"show databases\"\n",
    "spark.sql(EXAMPLE_SQL_QUERY).show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bdb14a26-b93c-4688-8f81-2c41a1eff972",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 0:>                                                          (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------------+-------------+--------------+---------------+-----------+------------+----------------+---------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------+----------------+--------------------+--------------+------------+-------------+\n",
      "|multiplelines|paperlessbilling|       gender|onlinesecurity|internetservice|techsupport|    contract|           churn|  seniorcitizen|   deviceprotection|        streamingtv|    streamingmovies|       totalcharges|            partner|     monthlycharges|    customerid|      dependents|        onlinebackup|  phoneservice|      tenure|paymentmethod|\n",
      "+-------------+----------------+-------------+--------------+---------------+-----------+------------+----------------+---------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------+----------------+--------------------+--------------+------------+-------------+\n",
      "|   customerID|          gender|SeniorCitizen|       Partner|     Dependents|     tenure|PhoneService|   MultipleLines|InternetService|     OnlineSecurity|       OnlineBackup|   DeviceProtection|        TechSupport|        StreamingTV|    StreamingMovies|      Contract|PaperlessBilling|       PaymentMethod|MonthlyCharges|TotalCharges|        Churn|\n",
      "|   7590-VHVEG|          Female|            0|           Yes|             No|          1|          No|No phone service|            DSL|                 No|                Yes|                 No|                 No|                 No|                 No|Month-to-month|             Yes|    Electronic check|         29.85|       29.85|           No|\n",
      "|   5575-GNVDE|            Male|            0|            No|             No|         34|         Yes|              No|            DSL|                Yes|                 No|                Yes|                 No|                 No|                 No|      One year|              No|        Mailed check|         56.95|      1889.5|           No|\n",
      "|   3668-QPYBK|            Male|            0|            No|             No|          2|         Yes|              No|            DSL|                Yes|                Yes|                 No|                 No|                 No|                 No|Month-to-month|             Yes|        Mailed check|         53.85|      108.15|          Yes|\n",
      "|   7795-CFOCW|            Male|            0|            No|             No|         45|          No|No phone service|            DSL|                Yes|                 No|                Yes|                Yes|                 No|                 No|      One year|              No|Bank transfer (au...|          42.3|     1840.75|           No|\n",
      "|   9237-HQITU|          Female|            0|            No|             No|          2|         Yes|              No|    Fiber optic|                 No|                 No|                 No|                 No|                 No|                 No|Month-to-month|             Yes|    Electronic check|          70.7|      151.65|          Yes|\n",
      "|   9305-CDSKC|          Female|            0|            No|             No|          8|         Yes|             Yes|    Fiber optic|                 No|                 No|                Yes|                 No|                Yes|                Yes|Month-to-month|             Yes|    Electronic check|         99.65|       820.5|          Yes|\n",
      "|   1452-KIOVK|            Male|            0|            No|            Yes|         22|         Yes|             Yes|    Fiber optic|                 No|                Yes|                 No|                 No|                Yes|                 No|Month-to-month|             Yes|Credit card (auto...|          89.1|      1949.4|           No|\n",
      "|   6713-OKOMC|          Female|            0|            No|             No|         10|          No|No phone service|            DSL|                Yes|                 No|                 No|                 No|                 No|                 No|Month-to-month|              No|        Mailed check|         29.75|       301.9|           No|\n",
      "|   7892-POOKP|          Female|            0|           Yes|             No|         28|         Yes|             Yes|    Fiber optic|                 No|                 No|                Yes|                Yes|                Yes|                Yes|Month-to-month|             Yes|    Electronic check|         104.8|     3046.05|          Yes|\n",
      "|   6388-TABGU|            Male|            0|            No|            Yes|         62|         Yes|              No|            DSL|                Yes|                Yes|                 No|                 No|                 No|                 No|      One year|              No|Bank transfer (au...|         56.15|     3487.95|           No|\n",
      "|   9763-GRSKD|            Male|            0|           Yes|            Yes|         13|         Yes|              No|            DSL|                Yes|                 No|                 No|                 No|                 No|                 No|Month-to-month|             Yes|        Mailed check|         49.95|      587.45|           No|\n",
      "|   7469-LKBCI|            Male|            0|            No|             No|         16|         Yes|              No|             No|No internet service|No internet service|No internet service|No internet service|No internet service|No internet service|      Two year|              No|Credit card (auto...|         18.95|       326.8|           No|\n",
      "|   8091-TTVAX|            Male|            0|           Yes|             No|         58|         Yes|             Yes|    Fiber optic|                 No|                 No|                Yes|                 No|                Yes|                Yes|      One year|              No|Credit card (auto...|        100.35|      5681.1|           No|\n",
      "|   0280-XJGEX|            Male|            0|            No|             No|         49|         Yes|             Yes|    Fiber optic|                 No|                Yes|                Yes|                 No|                Yes|                Yes|Month-to-month|             Yes|Bank transfer (au...|         103.7|      5036.3|          Yes|\n",
      "|   5129-JLPIS|            Male|            0|            No|             No|         25|         Yes|              No|    Fiber optic|                Yes|                 No|                Yes|                Yes|                Yes|                Yes|Month-to-month|             Yes|    Electronic check|         105.5|     2686.05|           No|\n",
      "|   3655-SNQYZ|          Female|            0|           Yes|            Yes|         69|         Yes|             Yes|    Fiber optic|                Yes|                Yes|                Yes|                Yes|                Yes|                Yes|      Two year|              No|Credit card (auto...|        113.25|     7895.15|           No|\n",
      "|   8191-XWSZG|          Female|            0|            No|             No|         52|         Yes|              No|             No|No internet service|No internet service|No internet service|No internet service|No internet service|No internet service|      One year|              No|        Mailed check|         20.65|     1022.95|           No|\n",
      "|   9959-WOFKT|            Male|            0|            No|            Yes|         71|         Yes|             Yes|    Fiber optic|                Yes|                 No|                Yes|                 No|                Yes|                Yes|      Two year|              No|Bank transfer (au...|         106.7|     7382.25|           No|\n",
      "|   4190-MFLUW|          Female|            0|           Yes|            Yes|         10|         Yes|              No|            DSL|                 No|                 No|                Yes|                Yes|                 No|                 No|Month-to-month|              No|Credit card (auto...|          55.2|      528.35|          Yes|\n",
      "+-------------+----------------+-------------+--------------+---------------+-----------+------------+----------------+---------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+--------------+----------------+--------------------+--------------+------------+-------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df2=spark.sql(\"select * from default.curated\")\n",
    "df2.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b3d102f-afd9-4d7f-8053-2e127d8fca07",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample usage to run query through spark\n",
    "EXAMPLE_SQL_QUERY = \"SELECT contract, count(*) as cant  FROM default.curated GROUP BY contract\"\n",
    "temp_df = spark.sql(EXAMPLE_SQL_QUERY)\n",
    "temp_df.write.mode(\"overwrite\").saveAsTable(\"default.curated_0101\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39f06d89-294d-4bef-b50b-a98bf3ee33ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24/01/22 17:55:45 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n",
      "24/01/22 17:56:00 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n",
      "24/01/22 17:56:15 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n",
      "24/01/22 17:56:30 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n",
      "24/01/22 17:56:45 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n",
      "24/01/22 17:57:00 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n",
      "24/01/22 17:57:15 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n",
      "24/01/22 17:57:30 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n",
      "24/01/22 17:57:45 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n",
      "24/01/22 17:58:00 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources\n"
     ]
    }
   ],
   "source": [
    "df2=spark.sql(\"show tables\")\n",
    "df2.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86c7415a-70ce-4513-9cde-42226ca3a194",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2=spark.sql(\"select * from default.curated_0101 limit 5\")\n",
    "df2.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}