Compatibility with Spark 2 jar loading

Kotlin · Oct 2, 2018 · c8d698f · c8d698f
1 parent 1a7c19e
commit c8d698f
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 25 deletions.
diff --git a/bin/kshell.sh b/bin/kshell.sh
@@ -21,4 +21,6 @@ KSHELL=$HOME/.m2/repository/sparklin/kshell/${SPARKLIN_VERSION}/kshell-${SPARKLI
 
 JARS=${KOTLIN_COMPILER}:${KOTLIN_STDLIB}:${KOTLIN_REFLECT}:${KOTLIN_SCRIPT_RUNTIME}:${KSHELL_REPL_API}
 
+echo $JARS
+
 java -classpath ${JARS}:${KSHELL} com.github.khud.sparklin.kshell.KotlinShell $@
diff --git a/bin/sparklin-spark2x.sh b/bin/sparklin-spark2x.sh
@@ -8,6 +8,7 @@ SPARKLIN_VERSION=0.2.5
 # Kotlin libraries
 KOTLIN_STDLIB=${MAVEN_REPO_BASE}/kotlin-stdlib/${KOTLIN_VERSION}/kotlin-stdlib-${KOTLIN_VERSION}.jar
 KOTLIN_COMPILER=${MAVEN_REPO_BASE}/kotlin-compiler/${KOTLIN_VERSION}/kotlin-compiler-${KOTLIN_VERSION}.jar
+KOTLIN_COMPILER_SHADED=kotlin-compiler-shaded/target/kotlin-compiler-shaded-${KOTLIN_VERSION}.jar
 KOTLIN_SCRIPT_RUNTIME=${MAVEN_REPO_BASE}/kotlin-script-runtime/${KOTLIN_VERSION}/kotlin-script-runtime-${KOTLIN_VERSION}.jar
 KOTLIN_REFLECT=${MAVEN_REPO_BASE}/kotlin-reflect/${KOTLIN_VERSION}/kotlin-reflect-${KOTLIN_VERSION}.jar
 
@@ -32,7 +33,7 @@ SPARK_HOME=$HOME/Programs/spark-2.3.1-bin-hadoop2.7
 
 JARS=${KOTLIN_COMPILER},${KOTLIN_STDLIB},${KOTLIN_REFLECT},${KOTLIN_SCRIPT_RUNTIME},${KSHELL_REPL_API},${SPARKLIN_PLUGIN},${HDFS_BROWSER_PLUGIN}
 
-${SPARK_HOME}/bin/spark-submit --jars=${JARS} --conf "spark.driver.extraJavaOptions=-Dconfig.path=${SPARKLIN_CONFIG}" \
+${SPARK_HOME}/bin/spark-submit --driver-class-path $KOTLIN_COMPILER_SHADED --conf "spark.driver.extraJavaOptions=-Dconfig.path=${SPARKLIN_CONFIG}" \
         --master "local[2]" --class "com.github.khud.sparklin.kshell.KotlinShell" \
-		${KSHELL} \
+		${SPARKLIN_PLUGIN} \
 		$@
diff --git a/hdfs-browser/pom.xml b/hdfs-browser/pom.xml
@@ -19,16 +19,6 @@
             <artifactId>kshell</artifactId>
             <version>${sparklin.version}</version>
         </dependency>
-        <dependency>
-            <groupId>sparklin</groupId>
-            <artifactId>sparklin-spark-1.x</artifactId>
-            <version>${sparklin.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>sparklin</groupId>
-            <artifactId>sparklin-spark-2.x</artifactId>
-            <version>${sparklin.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.jetbrains.kotlin</groupId>
             <artifactId>kotlin-stdlib-jdk8</artifactId>

diff --git a/kshell/pom.xml b/kshell/pom.xml
@@ -46,7 +46,7 @@
         <dependency>
             <groupId>com.github.khud</groupId>
             <artifactId>kshell-repl-api</artifactId>
-            <version>0.2.4-1.2.60</version>
+            <version>0.2.5-1.2.60</version>
         </dependency>
         <dependency>
             <groupId>sparklin</groupId>

diff --git a/pom.xml b/pom.xml
@@ -16,11 +16,11 @@
 	<packaging>pom</packaging>
 
 	<modules>
-        <module>jline3-shaded</module>
+        <!--<module>jline3-shaded</module>-->
         <module>kshell</module>
+        <module>hdfs-browser</module>
         <module>sparklin-spark-1.x</module>
         <module>sparklin-spark-2.x</module>
-        <module>hdfs-browser</module>
         <!--<module>zeppelin-interpreter</module>-->
     </modules>
 </project>
diff --git a/sparklin-spark-2.x/pom.xml b/sparklin-spark-2.x/pom.xml
@@ -19,6 +19,11 @@
             <artifactId>kshell</artifactId>
             <version>${sparklin.version}</version>
         </dependency>
+        <dependency>
+            <groupId>sparklin</groupId>
+            <artifactId>hdfs-browser</artifactId>
+            <version>${sparklin.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.jetbrains.kotlin</groupId>
             <artifactId>kotlin-stdlib-jdk8</artifactId>
@@ -34,6 +39,11 @@
             <artifactId>kotlin-compiler</artifactId>
             <version>${kotlin.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-script-runtime</artifactId>
+            <version>${kotlin.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_2.11</artifactId>
@@ -140,17 +150,53 @@
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-jar-plugin</artifactId>
-                <version>2.6</version>
-                <configuration>
-                    <archive>
-                        <manifest>
-                            <!-- addClasspath>true</addClasspath> -->
-                            <mainClass>sparklin.spark1x.Main</mainClass>
-                        </manifest>
-                    </archive>
-                </configuration>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.1.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <artifactSet>
+                                <includes>
+                                    <include>sparklin:kshell</include>
+                                    <include>sparklin:hdfs-browser</include>
+                                    <include>com.github.khud:kshell-repl-api</include>
+                                    <!--<include>org.jetbrains.kotlin:kotlin-compiler</include>-->
+                                    <include>org.jetbrains.kotlin:kotlin-stdlib-jdk8</include>
+                                    <include>org.jetbrains.kotlin:kotlin-reflect</include>
+                                    <include>org.jetbrains.kotlin:kotlin-script-runtime</include>
+                                </includes>
+                            </artifactSet>
+                            <!--<relocations>-->
+                                <!--<relocation>-->
+                                    <!--<pattern>org.apache.log4j</pattern>-->
+                                    <!--<shadedPattern>org.jetbrains.kotlin.shaded.log4j</shadedPattern>-->
+                                <!--</relocation>-->
+                                <!--<relocation>-->
+                                    <!--<pattern>com.google</pattern>-->
+                                    <!--<shadedPattern>org.jetbrains.kotlin.shaded.com.google</shadedPattern>-->
+                                <!--</relocation>-->
+                            <!--</relocations>-->
+                        </configuration>
+                    </execution>
+                </executions>
             </plugin>
+            <!--<plugin>-->
+                <!--<groupId>org.apache.maven.plugins</groupId>-->
+                <!--<artifactId>maven-jar-plugin</artifactId>-->
+                <!--<version>2.6</version>-->
+                <!--<configuration>-->
+                    <!--<archive>-->
+                        <!--<manifest>-->
+                            <!--&lt;!&ndash; addClasspath>true</addClasspath> &ndash;&gt;-->
+                            <!--<mainClass>sparklin.spark1x.Main</mainClass>-->
+                        <!--</manifest>-->
+                    <!--</archive>-->
+                <!--</configuration>-->
+            <!--</plugin>-->
         </plugins>
     </build>
 </project>