From 32be4ca8629cc87903356e604beaae2cd21e4333 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 26 Mar 2015 15:33:56 +0900 Subject: [PATCH 1/6] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Added maven-shade-plugin - Removed copy-dependencies --- spark/pom.xml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/spark/pom.xml b/spark/pom.xml index 49b4d082a01..2e7f546f8b6 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -306,11 +306,25 @@ + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + + + maven-dependency-plugin 2.8 - + copy-artifact package @@ -335,7 +349,6 @@ false false true - runtime ${project.groupId} From 91066c41b07b087187edfff69aae708d5a1d057e Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 26 Mar 2015 16:50:53 +0900 Subject: [PATCH 2/6] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Fixed errors while running --- notebook/2A94M5J1Z/note.json | 112 +++++++++++++++++++++++++++-------- spark/pom.xml | 19 +++++- 2 files changed, 106 insertions(+), 25 deletions(-) diff --git a/notebook/2A94M5J1Z/note.json b/notebook/2A94M5J1Z/note.json index 03e9fa3a0de..7432bca2607 100644 --- a/notebook/2A94M5J1Z/note.json +++ b/notebook/2A94M5J1Z/note.json @@ -10,7 +10,8 @@ "optionOpen": false, "keys": [], "values": [], - "groups": [] + "groups": [], + "scatter": {} }, "editorHide": false }, @@ -26,8 +27,8 @@ "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run codes by yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" }, "dateCreated": "Feb 13, 2015 11:16:21 PM", - "dateStarted": "Feb 13, 2015 11:23:27 PM", - "dateFinished": "Feb 13, 2015 11:23:27 PM", + "dateStarted": "Mar 26, 2015 4:49:50 PM", + "dateFinished": "Mar 26, 2015 4:49:51 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -42,7 +43,8 @@ "optionOpen": false, "keys": [], "values": [], - "groups": [] + "groups": [], + "scatter": {} }, "title": true }, @@ -55,11 +57,11 @@ "result": { "code": "SUCCESS", "type": "TEXT", - "msg": "import sys.process._\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres49: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres50: Int \u003d 1\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres51: Int \u003d 1\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres52: Int \u003d 0\n" + "msg": "import sys.process._\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres4: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres5: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres6: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres7: Int \u003d 0\n" }, "dateCreated": "Dec 4, 2014 10:28:55 AM", - "dateStarted": "Feb 13, 2015 11:22:01 PM", - "dateFinished": "Feb 13, 2015 11:22:09 PM", + "dateStarted": "Mar 26, 2015 4:49:52 PM", + "dateFinished": "Mar 26, 2015 4:49:56 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -74,7 +76,8 @@ "optionOpen": false, "keys": [], "values": [], - "groups": [] + "groups": [], + "scatter": {} }, "title": true }, @@ -85,13 +88,13 @@ "jobName": "paragraph_1423500779206_-1502780787", "id": "20150210-015259_1403135953", "result": { - "code": "SUCCESS", + "code": "ERROR", "type": "TEXT", - "msg": "warning: there were 1 feature warning(s); re-run with -feature for details\nzeppelinHome: String \u003d /Users/swkim/Documents/zeppelin\nbankText: org.apache.spark.rdd.RDD[String] \u003d /Users/swkim/Documents/zeppelin/data/bank-full.csv MappedRDD[147] at textFile at \u003cconsole\u003e:24\ndefined class Bank\nbank: org.apache.spark.rdd.RDD[Bank] \u003d MappedRDD[150] at map at \u003cconsole\u003e:29\n" + "msg": "warning: there were 1 feature warning(s); re-run with -feature for details\nzeppelinHome: String \u003d /Users/jl/local/src/g/incubator-zeppelin\nbankText: org.apache.spark.rdd.RDD[String] \u003d /Users/jl/local/src/g/incubator-zeppelin/data/bank-full.csv MapPartitionsRDD[3] at textFile at \u003cconsole\u003e:28\ndefined class Bank\nbank: org.apache.spark.rdd.RDD[Bank] \u003d MapPartitionsRDD[6] at map at \u003cconsole\u003e:33\n\u003cconsole\u003e:35: error: value registerTempTable is not a member of org.apache.spark.rdd.RDD[Bank]\n bank.registerTempTable(\"bank\")\n ^\n" }, "dateCreated": "Feb 10, 2015 1:52:59 AM", - "dateStarted": "Feb 13, 2015 11:15:38 PM", - "dateFinished": "Feb 13, 2015 11:15:42 PM", + "dateStarted": "Mar 26, 2015 4:50:06 PM", + "dateFinished": "Mar 26, 2015 4:50:07 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -117,7 +120,8 @@ "aggr": "sum" } ], - "groups": [] + "groups": [], + "scatter": {} } }, "settings": { @@ -126,15 +130,12 @@ }, "jobName": "paragraph_1423500782552_-1439281894", "id": "20150210-015302_1492795503", - "result": { - "code": "SUCCESS", - "type": "TABLE", - "msg": "age\tvalue\n18\t12\n19\t35\n20\t50\n21\t79\n22\t129\n23\t202\n24\t302\n25\t527\n26\t805\n27\t909\n28\t1038\n29\t1185\n" - }, + "result": "org.apache.thrift.TApplicationException: Internal error processing interpret", "dateCreated": "Feb 10, 2015 1:53:02 AM", - "dateStarted": "Feb 13, 2015 11:15:45 PM", - "dateFinished": "Feb 13, 2015 11:15:47 PM", + "dateStarted": "Mar 26, 2015 4:38:14 PM", + "dateFinished": "Mar 26, 2015 4:38:16 PM", "status": "FINISHED", + "errorMessage": "com.nflabs.zeppelin.interpreter.remote.RemoteInterpreter.interpret(RemoteInterpreter.java:203)\ncom.nflabs.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:76)\ncom.nflabs.zeppelin.notebook.Paragraph.jobRun(Paragraph.java:183)\ncom.nflabs.zeppelin.scheduler.Job.run(Job.java:152)\ncom.nflabs.zeppelin.scheduler.RemoteScheduler$JobRunner.run(RemoteScheduler.java:237)\njava.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\njava.util.concurrent.FutureTask.run(FutureTask.java:262)\njava.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)\njava.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\njava.lang.Thread.run(Thread.java:745)\n", "progressUpdateIntervalMs": 500 }, { @@ -258,7 +259,8 @@ "optionOpen": false, "keys": [], "values": [], - "groups": [] + "groups": [], + "scatter": {} } }, "settings": { @@ -273,20 +275,82 @@ "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n" }, "dateCreated": "Feb 13, 2015 11:04:28 PM", - "dateStarted": "Feb 13, 2015 11:23:16 PM", - "dateFinished": "Feb 13, 2015 11:23:16 PM", + "dateStarted": "Mar 26, 2015 4:38:20 PM", + "dateFinished": "Mar 26, 2015 4:38:20 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, { - "config": {}, + "text": "sc.version", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + }, "settings": { "params": {}, "forms": {} }, "jobName": "paragraph_1423836471379_293523076", "id": "20150213-230751_1038476812", + "result": { + "code": "SUCCESS", + "type": "TEXT", + "msg": "res0: String \u003d 1.3.0\n" + }, "dateCreated": "Feb 13, 2015 11:07:51 PM", + "dateStarted": "Mar 26, 2015 4:48:41 PM", + "dateFinished": "Mar 26, 2015 4:48:50 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "sc.textFile(\"/data/pickat/tsv/app/2015/03/03\").take(10).foreach(println)", + "config": { + "colWidth": 12.0, + "graph": { + "mode": "table", + "height": 300.0, + "optionOpen": false, + "keys": [], + "values": [], + "groups": [], + "scatter": {} + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1427355508701_-1807073905", + "id": "20150326-163828_2000051303", + "result": { + "code": "ERROR", + "type": "TEXT", + "msg": "org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: file:/data/pickat/tsv/app/2015/03/03\n\tat org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:285)\n\tat org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:228)\n\tat org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:313)\n\tat org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:203)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:217)\n\tat org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:32)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:217)\n\tat org.apache.spark.rdd.RDD.take(RDD.scala:1156)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:24)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:29)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:31)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:33)\n\tat $iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:35)\n\tat $iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:37)\n\tat $iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:39)\n\tat $iwC.\u003cinit\u003e(\u003cconsole\u003e:41)\n\tat \u003cinit\u003e(\u003cconsole\u003e:43)\n\tat .\u003cinit\u003e(\u003cconsole\u003e:47)\n\tat .\u003cclinit\u003e(\u003cconsole\u003e)\n\tat .\u003cinit\u003e(\u003cconsole\u003e:7)\n\tat .\u003cclinit\u003e(\u003cconsole\u003e)\n\tat $print(\u003cconsole\u003e)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:606)\n\tat org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065)\n\tat org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1338)\n\tat org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpretInput(SparkInterpreter.java:541)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:517)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:510)\n\tat com.nflabs.zeppelin.interpreter.ClassloaderInterpreter.interpret(ClassloaderInterpreter.java:40)\n\tat com.nflabs.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:76)\n\tat com.nflabs.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:247)\n\tat com.nflabs.zeppelin.scheduler.Job.run(Job.java:152)\n\tat com.nflabs.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:101)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\n" + }, + "dateCreated": "Mar 26, 2015 4:38:28 PM", + "dateStarted": "Mar 26, 2015 4:49:23 PM", + "dateFinished": "Mar 26, 2015 4:49:24 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "jobName": "paragraph_1427356163194_1561954386", + "id": "20150326-164923_2131128923", + "dateCreated": "Mar 26, 2015 4:49:23 PM", "status": "READY", "progressUpdateIntervalMs": 500 } diff --git a/spark/pom.xml b/spark/pom.xml index 2e7f546f8b6..d19c0fe50a8 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -310,6 +310,23 @@ org.apache.maven.plugins maven-shade-plugin 2.3 + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + reference.conf + + + package @@ -339,7 +356,7 @@ --> - copy-artifact + package copy From 29ea46bf620b8d14a192b9a73c793c6020eadea9 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 26 Mar 2015 17:15:16 +0900 Subject: [PATCH 3/6] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Removed unused codes --- notebook/2A94M5J1Z/note.json | 8 ++++---- spark/pom.xml | 16 +--------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/notebook/2A94M5J1Z/note.json b/notebook/2A94M5J1Z/note.json index 7432bca2607..ffda9853f37 100644 --- a/notebook/2A94M5J1Z/note.json +++ b/notebook/2A94M5J1Z/note.json @@ -67,7 +67,7 @@ }, { "title": "Load data into table", - "text": "val zeppelinHome \u003d (\"pwd\" !!).replace(\"\\n\", \"\")\nval bankText \u003d sc.textFile(s\"$zeppelinHome/data/bank-full.csv\")\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n)\nbank.registerTempTable(\"bank\")", + "text": "val zeppelinHome \u003d (\"pwd\" !!).replace(\"\\n\", \"\")\nval bankText \u003d sc.textFile(s\"$zeppelinHome/data/bank-full.csv\")\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n)\nbank.toDF.registerTempTable(\"bank\")", "config": { "colWidth": 12.0, "graph": { @@ -90,11 +90,11 @@ "result": { "code": "ERROR", "type": "TEXT", - "msg": "warning: there were 1 feature warning(s); re-run with -feature for details\nzeppelinHome: String \u003d /Users/jl/local/src/g/incubator-zeppelin\nbankText: org.apache.spark.rdd.RDD[String] \u003d /Users/jl/local/src/g/incubator-zeppelin/data/bank-full.csv MapPartitionsRDD[3] at textFile at \u003cconsole\u003e:28\ndefined class Bank\nbank: org.apache.spark.rdd.RDD[Bank] \u003d MapPartitionsRDD[6] at map at \u003cconsole\u003e:33\n\u003cconsole\u003e:35: error: value registerTempTable is not a member of org.apache.spark.rdd.RDD[Bank]\n bank.registerTempTable(\"bank\")\n ^\n" + "msg": "warning: there were 1 feature warning(s); re-run with -feature for details\nzeppelinHome: String \u003d /Users/jl/local/src/g/incubator-zeppelin\nbankText: org.apache.spark.rdd.RDD[String] \u003d /Users/jl/local/src/g/incubator-zeppelin/data/bank-full.csv MapPartitionsRDD[8] at textFile at \u003cconsole\u003e:28\ndefined class Bank\nbank: org.apache.spark.rdd.RDD[Bank] \u003d MapPartitionsRDD[11] at map at \u003cconsole\u003e:33\njava.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.metastore.HiveMetaStoreClient\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:346)\n\tat org.apache.spark.sql.hive.HiveContext.sessionState$lzycompute(HiveContext.scala:229)\n\tat org.apache.spark.sql.hive.HiveContext.sessionState(HiveContext.scala:225)\n\tat org.apache.spark.sql.hive.HiveContext$QueryExecution.\u003cinit\u003e(HiveContext.scala:373)\n\tat org.apache.spark.sql.hive.HiveContext.executePlan(HiveContext.scala:80)\n\tat org.apache.spark.sql.hive.HiveContext.executePlan(HiveContext.scala:49)\n\tat org.apache.spark.sql.DataFrame.\u003cinit\u003e(DataFrame.scala:131)\n\tat org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:51)\n\tat org.apache.spark.sql.SQLContext.createDataFrame(SQLContext.scala:319)\n\tat org.apache.spark.sql.SQLContext$implicits$.rddToDataFrameHolder(SQLContext.scala:254)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:35)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:40)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:42)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:44)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:46)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:48)\n\tat $iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:50)\n\tat $iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:52)\n\tat $iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:54)\n\tat $iwC.\u003cinit\u003e(\u003cconsole\u003e:56)\n\tat \u003cinit\u003e(\u003cconsole\u003e:58)\n\tat .\u003cinit\u003e(\u003cconsole\u003e:62)\n\tat .\u003cclinit\u003e(\u003cconsole\u003e)\n\tat .\u003cinit\u003e(\u003cconsole\u003e:7)\n\tat .\u003cclinit\u003e(\u003cconsole\u003e)\n\tat $print(\u003cconsole\u003e)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:606)\n\tat org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065)\n\tat org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1338)\n\tat org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpretInput(SparkInterpreter.java:541)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:517)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:510)\n\tat com.nflabs.zeppelin.interpreter.ClassloaderInterpreter.interpret(ClassloaderInterpreter.java:40)\n\tat com.nflabs.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:76)\n\tat com.nflabs.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:247)\n\tat com.nflabs.zeppelin.scheduler.Job.run(Job.java:152)\n\tat com.nflabs.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:101)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\nCaused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.metastore.HiveMetaStoreClient\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1412)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.\u003cinit\u003e(RetryingMetaStoreClient.java:62)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:72)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:2453)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:2465)\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:340)\n\t... 49 more\nCaused by: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:526)\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1410)\n\t... 54 more\nCaused by: javax.jdo.JDOFatalInternalException: Unexpected exception caught.\nNestedThrowables:\njava.lang.reflect.InvocationTargetException\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1193)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:310)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:339)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:248)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:223)\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73)\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.\u003cinit\u003e(RawStoreProxy.java:58)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:67)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:497)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:475)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:523)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:397)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.\u003cinit\u003e(HiveMetaStore.java:356)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.\u003cinit\u003e(RetryingHMSHandler.java:54)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:59)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newHMSHandler(HiveMetaStore.java:4944)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.\u003cinit\u003e(HiveMetaStoreClient.java:171)\n\t... 59 more\nCaused by: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:606)\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\n\t... 78 more\nCaused by: org.datanucleus.exceptions.NucleusUserException: Persistence process has been specified to use a ClassLoaderResolver of name \"datanucleus\" yet this has not been found by the DataNucleus plugin mechanism. Please check your CLASSPATH and plugin specification.\n\tat org.datanucleus.NucleusContext.\u003cinit\u003e(NucleusContext.java:283)\n\tat org.datanucleus.NucleusContext.\u003cinit\u003e(NucleusContext.java:247)\n\tat org.datanucleus.NucleusContext.\u003cinit\u003e(NucleusContext.java:225)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.\u003cinit\u003e(JDOPersistenceManagerFactory.java:416)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:301)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\n\t... 86 more\n\n" }, "dateCreated": "Feb 10, 2015 1:52:59 AM", - "dateStarted": "Mar 26, 2015 4:50:06 PM", - "dateFinished": "Mar 26, 2015 4:50:07 PM", + "dateStarted": "Mar 26, 2015 4:52:11 PM", + "dateFinished": "Mar 26, 2015 4:52:13 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, diff --git a/spark/pom.xml b/spark/pom.xml index d19c0fe50a8..292be88c671 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -322,6 +322,7 @@ + reference.conf @@ -341,22 +342,7 @@ maven-dependency-plugin 2.8 - - package copy From b366d16096d8036aab2feb16221ff2601815852f Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 26 Mar 2015 17:16:12 +0900 Subject: [PATCH 4/6] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Fixed 2 spaces indentations --- spark/pom.xml | 72 +++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/spark/pom.xml b/spark/pom.xml index 292be88c671..fcecc1bc3e9 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 @@ -9,7 +10,6 @@ 0.5.0-SNAPSHOT - com.nflabs.zeppelin zeppelin-spark jar 0.5.0-SNAPSHOT @@ -67,7 +67,7 @@ org.scala-lang scala-library - + org.scala-lang scala-compiler @@ -77,7 +77,7 @@ org.scala-lang scala-reflect - + org.eclipse.jetty.aggregate jetty-all-server @@ -94,27 +94,27 @@ ${spark.version} - + org.apache.spark spark-sql_2.10 ${spark.version} - + org.apache.spark spark-hive_2.10 ${spark.version} - + org.apache.spark spark-streaming_2.10 ${spark.version} - + org.apache.spark spark-streaming-twitter_2.10 ${spark.version} @@ -125,7 +125,7 @@ spark-catalyst_${scala.binary.version} ${spark.version} - + org.apache.maven @@ -139,12 +139,12 @@ org.sonatype.sisu sisu-inject-plexus - + org.apache.maven maven-model - - + + org.sonatype.aether @@ -161,7 +161,7 @@ aether-impl 1.12 - + org.apache.maven maven-aether-provider @@ -170,7 +170,7 @@ org.sonatype.aether aether-api - + org.sonatype.aether aether-spi @@ -178,24 +178,24 @@ org.sonatype.aether aether-util - + org.sonatype.aether aether-impl - + org.codehaus.plexus plexus-utils - - + + - + org.sonatype.aether aether-connector-file 1.12 - + org.sonatype.aether aether-connector-wagon @@ -205,9 +205,9 @@ org.apache.maven.wagon wagon-provider-api - + - + org.apache.maven.wagon wagon-provider-api @@ -216,10 +216,10 @@ org.codehaus.plexus plexus-utils - - + + - + org.apache.maven.wagon wagon-http-lightweight @@ -230,7 +230,7 @@ wagon-http-shared - + org.apache.maven.wagon @@ -259,7 +259,7 @@ test - + yarn @@ -272,7 +272,7 @@ - + @@ -286,11 +286,11 @@ maven-enforcer-plugin - 1.3.1 - - - enforce - none + 1.3.1 + + + enforce + none @@ -337,7 +337,7 @@ - + maven-dependency-plugin 2.8 @@ -359,11 +359,11 @@ ${project.version} ${project.packaging} - + - + From 7f84a5a8ffd9487a9856ad884626cc51d548148d Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 26 Mar 2015 19:22:41 +0900 Subject: [PATCH 5/6] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Reverted to initial commit --- notebook/2A94M5J1Z/note.json | 114 ++++++++--------------------------- 1 file changed, 25 insertions(+), 89 deletions(-) diff --git a/notebook/2A94M5J1Z/note.json b/notebook/2A94M5J1Z/note.json index ffda9853f37..03e9fa3a0de 100644 --- a/notebook/2A94M5J1Z/note.json +++ b/notebook/2A94M5J1Z/note.json @@ -10,8 +10,7 @@ "optionOpen": false, "keys": [], "values": [], - "groups": [], - "scatter": {} + "groups": [] }, "editorHide": false }, @@ -27,8 +26,8 @@ "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run codes by yourself. (Shift-Enter to Run)\u003c/h5\u003e\n" }, "dateCreated": "Feb 13, 2015 11:16:21 PM", - "dateStarted": "Mar 26, 2015 4:49:50 PM", - "dateFinished": "Mar 26, 2015 4:49:51 PM", + "dateStarted": "Feb 13, 2015 11:23:27 PM", + "dateFinished": "Feb 13, 2015 11:23:27 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -43,8 +42,7 @@ "optionOpen": false, "keys": [], "values": [], - "groups": [], - "scatter": {} + "groups": [] }, "title": true }, @@ -57,17 +55,17 @@ "result": { "code": "SUCCESS", "type": "TEXT", - "msg": "import sys.process._\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres4: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres5: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres6: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres7: Int \u003d 0\n" + "msg": "import sys.process._\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres49: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres50: Int \u003d 1\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres51: Int \u003d 1\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres52: Int \u003d 0\n" }, "dateCreated": "Dec 4, 2014 10:28:55 AM", - "dateStarted": "Mar 26, 2015 4:49:52 PM", - "dateFinished": "Mar 26, 2015 4:49:56 PM", + "dateStarted": "Feb 13, 2015 11:22:01 PM", + "dateFinished": "Feb 13, 2015 11:22:09 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, { "title": "Load data into table", - "text": "val zeppelinHome \u003d (\"pwd\" !!).replace(\"\\n\", \"\")\nval bankText \u003d sc.textFile(s\"$zeppelinHome/data/bank-full.csv\")\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n)\nbank.toDF.registerTempTable(\"bank\")", + "text": "val zeppelinHome \u003d (\"pwd\" !!).replace(\"\\n\", \"\")\nval bankText \u003d sc.textFile(s\"$zeppelinHome/data/bank-full.csv\")\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n)\nbank.registerTempTable(\"bank\")", "config": { "colWidth": 12.0, "graph": { @@ -76,8 +74,7 @@ "optionOpen": false, "keys": [], "values": [], - "groups": [], - "scatter": {} + "groups": [] }, "title": true }, @@ -88,13 +85,13 @@ "jobName": "paragraph_1423500779206_-1502780787", "id": "20150210-015259_1403135953", "result": { - "code": "ERROR", + "code": "SUCCESS", "type": "TEXT", - "msg": "warning: there were 1 feature warning(s); re-run with -feature for details\nzeppelinHome: String \u003d /Users/jl/local/src/g/incubator-zeppelin\nbankText: org.apache.spark.rdd.RDD[String] \u003d /Users/jl/local/src/g/incubator-zeppelin/data/bank-full.csv MapPartitionsRDD[8] at textFile at \u003cconsole\u003e:28\ndefined class Bank\nbank: org.apache.spark.rdd.RDD[Bank] \u003d MapPartitionsRDD[11] at map at \u003cconsole\u003e:33\njava.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.metastore.HiveMetaStoreClient\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:346)\n\tat org.apache.spark.sql.hive.HiveContext.sessionState$lzycompute(HiveContext.scala:229)\n\tat org.apache.spark.sql.hive.HiveContext.sessionState(HiveContext.scala:225)\n\tat org.apache.spark.sql.hive.HiveContext$QueryExecution.\u003cinit\u003e(HiveContext.scala:373)\n\tat org.apache.spark.sql.hive.HiveContext.executePlan(HiveContext.scala:80)\n\tat org.apache.spark.sql.hive.HiveContext.executePlan(HiveContext.scala:49)\n\tat org.apache.spark.sql.DataFrame.\u003cinit\u003e(DataFrame.scala:131)\n\tat org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:51)\n\tat org.apache.spark.sql.SQLContext.createDataFrame(SQLContext.scala:319)\n\tat org.apache.spark.sql.SQLContext$implicits$.rddToDataFrameHolder(SQLContext.scala:254)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:35)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:40)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:42)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:44)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:46)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:48)\n\tat $iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:50)\n\tat $iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:52)\n\tat $iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:54)\n\tat $iwC.\u003cinit\u003e(\u003cconsole\u003e:56)\n\tat \u003cinit\u003e(\u003cconsole\u003e:58)\n\tat .\u003cinit\u003e(\u003cconsole\u003e:62)\n\tat .\u003cclinit\u003e(\u003cconsole\u003e)\n\tat .\u003cinit\u003e(\u003cconsole\u003e:7)\n\tat .\u003cclinit\u003e(\u003cconsole\u003e)\n\tat $print(\u003cconsole\u003e)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:606)\n\tat org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065)\n\tat org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1338)\n\tat org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpretInput(SparkInterpreter.java:541)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:517)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:510)\n\tat com.nflabs.zeppelin.interpreter.ClassloaderInterpreter.interpret(ClassloaderInterpreter.java:40)\n\tat com.nflabs.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:76)\n\tat com.nflabs.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:247)\n\tat com.nflabs.zeppelin.scheduler.Job.run(Job.java:152)\n\tat com.nflabs.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:101)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\nCaused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.metastore.HiveMetaStoreClient\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1412)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.\u003cinit\u003e(RetryingMetaStoreClient.java:62)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:72)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:2453)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:2465)\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:340)\n\t... 49 more\nCaused by: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:526)\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1410)\n\t... 54 more\nCaused by: javax.jdo.JDOFatalInternalException: Unexpected exception caught.\nNestedThrowables:\njava.lang.reflect.InvocationTargetException\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1193)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:310)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:339)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:248)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:223)\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73)\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.\u003cinit\u003e(RawStoreProxy.java:58)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:67)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:497)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:475)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:523)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:397)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.\u003cinit\u003e(HiveMetaStore.java:356)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.\u003cinit\u003e(RetryingHMSHandler.java:54)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:59)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newHMSHandler(HiveMetaStore.java:4944)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.\u003cinit\u003e(HiveMetaStoreClient.java:171)\n\t... 59 more\nCaused by: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:606)\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\n\t... 78 more\nCaused by: org.datanucleus.exceptions.NucleusUserException: Persistence process has been specified to use a ClassLoaderResolver of name \"datanucleus\" yet this has not been found by the DataNucleus plugin mechanism. Please check your CLASSPATH and plugin specification.\n\tat org.datanucleus.NucleusContext.\u003cinit\u003e(NucleusContext.java:283)\n\tat org.datanucleus.NucleusContext.\u003cinit\u003e(NucleusContext.java:247)\n\tat org.datanucleus.NucleusContext.\u003cinit\u003e(NucleusContext.java:225)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.\u003cinit\u003e(JDOPersistenceManagerFactory.java:416)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:301)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\n\t... 86 more\n\n" + "msg": "warning: there were 1 feature warning(s); re-run with -feature for details\nzeppelinHome: String \u003d /Users/swkim/Documents/zeppelin\nbankText: org.apache.spark.rdd.RDD[String] \u003d /Users/swkim/Documents/zeppelin/data/bank-full.csv MappedRDD[147] at textFile at \u003cconsole\u003e:24\ndefined class Bank\nbank: org.apache.spark.rdd.RDD[Bank] \u003d MappedRDD[150] at map at \u003cconsole\u003e:29\n" }, "dateCreated": "Feb 10, 2015 1:52:59 AM", - "dateStarted": "Mar 26, 2015 4:52:11 PM", - "dateFinished": "Mar 26, 2015 4:52:13 PM", + "dateStarted": "Feb 13, 2015 11:15:38 PM", + "dateFinished": "Feb 13, 2015 11:15:42 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, @@ -120,8 +117,7 @@ "aggr": "sum" } ], - "groups": [], - "scatter": {} + "groups": [] } }, "settings": { @@ -130,12 +126,15 @@ }, "jobName": "paragraph_1423500782552_-1439281894", "id": "20150210-015302_1492795503", - "result": "org.apache.thrift.TApplicationException: Internal error processing interpret", + "result": { + "code": "SUCCESS", + "type": "TABLE", + "msg": "age\tvalue\n18\t12\n19\t35\n20\t50\n21\t79\n22\t129\n23\t202\n24\t302\n25\t527\n26\t805\n27\t909\n28\t1038\n29\t1185\n" + }, "dateCreated": "Feb 10, 2015 1:53:02 AM", - "dateStarted": "Mar 26, 2015 4:38:14 PM", - "dateFinished": "Mar 26, 2015 4:38:16 PM", + "dateStarted": "Feb 13, 2015 11:15:45 PM", + "dateFinished": "Feb 13, 2015 11:15:47 PM", "status": "FINISHED", - "errorMessage": "com.nflabs.zeppelin.interpreter.remote.RemoteInterpreter.interpret(RemoteInterpreter.java:203)\ncom.nflabs.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:76)\ncom.nflabs.zeppelin.notebook.Paragraph.jobRun(Paragraph.java:183)\ncom.nflabs.zeppelin.scheduler.Job.run(Job.java:152)\ncom.nflabs.zeppelin.scheduler.RemoteScheduler$JobRunner.run(RemoteScheduler.java:237)\njava.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\njava.util.concurrent.FutureTask.run(FutureTask.java:262)\njava.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)\njava.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\njava.lang.Thread.run(Thread.java:745)\n", "progressUpdateIntervalMs": 500 }, { @@ -259,8 +258,7 @@ "optionOpen": false, "keys": [], "values": [], - "groups": [], - "scatter": {} + "groups": [] } }, "settings": { @@ -275,82 +273,20 @@ "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n" }, "dateCreated": "Feb 13, 2015 11:04:28 PM", - "dateStarted": "Mar 26, 2015 4:38:20 PM", - "dateFinished": "Mar 26, 2015 4:38:20 PM", + "dateStarted": "Feb 13, 2015 11:23:16 PM", + "dateFinished": "Feb 13, 2015 11:23:16 PM", "status": "FINISHED", "progressUpdateIntervalMs": 500 }, { - "text": "sc.version", - "config": { - "colWidth": 12.0, - "graph": { - "mode": "table", - "height": 300.0, - "optionOpen": false, - "keys": [], - "values": [], - "groups": [], - "scatter": {} - } - }, + "config": {}, "settings": { "params": {}, "forms": {} }, "jobName": "paragraph_1423836471379_293523076", "id": "20150213-230751_1038476812", - "result": { - "code": "SUCCESS", - "type": "TEXT", - "msg": "res0: String \u003d 1.3.0\n" - }, "dateCreated": "Feb 13, 2015 11:07:51 PM", - "dateStarted": "Mar 26, 2015 4:48:41 PM", - "dateFinished": "Mar 26, 2015 4:48:50 PM", - "status": "FINISHED", - "progressUpdateIntervalMs": 500 - }, - { - "text": "sc.textFile(\"/data/pickat/tsv/app/2015/03/03\").take(10).foreach(println)", - "config": { - "colWidth": 12.0, - "graph": { - "mode": "table", - "height": 300.0, - "optionOpen": false, - "keys": [], - "values": [], - "groups": [], - "scatter": {} - } - }, - "settings": { - "params": {}, - "forms": {} - }, - "jobName": "paragraph_1427355508701_-1807073905", - "id": "20150326-163828_2000051303", - "result": { - "code": "ERROR", - "type": "TEXT", - "msg": "org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: file:/data/pickat/tsv/app/2015/03/03\n\tat org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:285)\n\tat org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:228)\n\tat org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:313)\n\tat org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:203)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:217)\n\tat org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:32)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:217)\n\tat org.apache.spark.rdd.RDD.take(RDD.scala:1156)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:24)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:29)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:31)\n\tat $iwC$$iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:33)\n\tat $iwC$$iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:35)\n\tat $iwC$$iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:37)\n\tat $iwC$$iwC.\u003cinit\u003e(\u003cconsole\u003e:39)\n\tat $iwC.\u003cinit\u003e(\u003cconsole\u003e:41)\n\tat \u003cinit\u003e(\u003cconsole\u003e:43)\n\tat .\u003cinit\u003e(\u003cconsole\u003e:47)\n\tat .\u003cclinit\u003e(\u003cconsole\u003e)\n\tat .\u003cinit\u003e(\u003cconsole\u003e:7)\n\tat .\u003cclinit\u003e(\u003cconsole\u003e)\n\tat $print(\u003cconsole\u003e)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:606)\n\tat org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065)\n\tat org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1338)\n\tat org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpretInput(SparkInterpreter.java:541)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:517)\n\tat com.nflabs.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:510)\n\tat com.nflabs.zeppelin.interpreter.ClassloaderInterpreter.interpret(ClassloaderInterpreter.java:40)\n\tat com.nflabs.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:76)\n\tat com.nflabs.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:247)\n\tat com.nflabs.zeppelin.scheduler.Job.run(Job.java:152)\n\tat com.nflabs.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:101)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:262)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\n" - }, - "dateCreated": "Mar 26, 2015 4:38:28 PM", - "dateStarted": "Mar 26, 2015 4:49:23 PM", - "dateFinished": "Mar 26, 2015 4:49:24 PM", - "status": "FINISHED", - "progressUpdateIntervalMs": 500 - }, - { - "config": {}, - "settings": { - "params": {}, - "forms": {} - }, - "jobName": "paragraph_1427356163194_1561954386", - "id": "20150326-164923_2131128923", - "dateCreated": "Mar 26, 2015 4:49:23 PM", "status": "READY", "progressUpdateIntervalMs": 500 } From df697c12f672148a52534a4ab60f64b0fd9c0b72 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Fri, 27 Mar 2015 11:34:40 +0900 Subject: [PATCH 6/6] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Updated README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 93df3121745..46e188ca09b 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,8 @@ If you set `SPARK_HOME`, you should deploy spark binary on the same location to Yarn # ./conf/zeppelin-env.sh - export SPARK_YARN_JAR=/path/to/spark-assembly-*.jar export HADOOP_CONF_DIR=/path/to/hadoop_conf_dir - -`SPARK_YARN_JAR` is deployed for running executor, this could be a local path or HDFS. HDFS allows YARN to cache it on nodes so that it doesn't need to be distributed each time an application runs. To point to a jar on HDFS, for example, set this configuration to "hdfs:///some/path". + `HADOOP_CONF_DIR` should contains yarn-site.xml and core-site.xml. ### Run