Perlito5 - java - Spark example - wip

fglock · Nov 28, 2019 · 39cd180 · 39cd180
1 parent 1ad317e
commit 39cd180
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 11 deletions.
diff --git a/misc/Spark/spark.pl b/misc/Spark/spark.pl
@@ -29,8 +29,13 @@ package List { import => "java.util.ArrayList<PlObject>" }
 
 my $dataSet = $jsc->parallelize($list, $slices);
 
-
 my $count;
+$count = $dataSet->count();
+print "Count: $count\n";
+
+# TODO
+# my $sum = $dataSet->reduce( sub ($$) { $_[0] + $_[1] } );
+# print "Sum: $sum\n";
 
 # TODO
 # $count = $dataSet->map(
@@ -42,8 +47,7 @@ package List { import => "java.util.ArrayList<PlObject>" }
 #     )->reduce(
 #         sub ($$) { $_[0] + $_[1] }
 #     );
-
-print "Pi is roughly ", 4.0 * $count / $n, "\n";
+# print "Pi is roughly ", 4.0 * $count / $n, "\n";
 
 $spark->stop();
 
diff --git a/misc/Spark/spark2.pl b/misc/Spark/spark2.pl
@@ -1,20 +1,25 @@
 
-package JavaRDD::PlObject { import => "org.apache.spark.api.java.JavaRDD<PlObject>" }
+package JavaRDD::String { import => "org.apache.spark.api.java.JavaRDD<String>" }
 
 package JavaSparkContext { import => "org.apache.spark.api.java.JavaSparkContext" }
 
 package SparkSession { import => "org.apache.spark.sql.SparkSession" }
 
 package Integer { import => "java.lang.Integer" }
 
-package List { import => "java.util.ArrayList<PlObject>" }
+package List { import => "java.util.ArrayList<String>" }
 
 use strict;
 use warnings;
 use Java;
 
 my SparkSession $spark =
-  SparkSession->builder()->appName("JavaSparkPi")->config( "spark.master", "local" )->getOrCreate();
+  SparkSession
+  ->builder()
+  ->appName("JavaSparkPi")
+  ->config( "spark.master", "local" )
+  # ->config( "spark.jars", "./perlito5.jar,./spark-2.4.4-bin-hadoop2.7/jars/*" )
+  ->getOrCreate();
 
 my JavaSparkContext $jsc = JavaSparkContext->new( $spark->sparkContext() );
 
@@ -25,10 +30,10 @@ package List { import => "java.util.ArrayList<PlObject>" }
 
 my List $list = List->new();
 for ( 0 .. $n - 1 ) {
-    $list->add($_);
+    $list->add("$_");
 }
 
-my JavaRDD::PlObject $dataSet = $jsc->parallelize( $list, $slices );
+my JavaRDD::String $dataSet = $jsc->parallelize( $list, $slices );
 
 my $count;
 
@@ -49,16 +54,16 @@ package List { import => "java.util.ArrayList<PlObject>" }
         }
     )
 
-)->reduce(
+  )->reduce(
 
     Java::inline q(
         (arg1, arg2) -> {
             // TODO use $reduce
-            return new PlLvalue(arg1.add(arg2));
+            return new PlString(arg1).add(new PlString(arg2)).toString();
         }
     )
 
-);
+  );
 
 print "Pi is roughly ", 4.0 * $count / $n, "\n";
 

diff --git a/misc/Spark/spark3.pl b/misc/Spark/spark3.pl
@@ -0,0 +1,32 @@
+
+package JavaRDD::String { import => "org.apache.spark.api.java.JavaRDD<String>" }
+
+package JavaSparkContext { import => "org.apache.spark.api.java.JavaSparkContext" }
+
+package SparkSession { import => "org.apache.spark.sql.SparkSession" }
+
+package Integer { import => "java.lang.Integer" }
+
+package List { import => "java.util.ArrayList<String>" }
+
+use strict;
+use warnings;
+use Java;
+
+my SparkSession $spark =
+  SparkSession
+  ->builder()
+  ->appName("JavaSparkPi")
+  ->config( "spark.master", "local" )
+  # ->config( "spark.jars", "./perlito5.jar,./spark-2.4.4-bin-hadoop2.7/jars/*" )
+  ->getOrCreate();
+
+my JavaSparkContext $sc = JavaSparkContext->new( $spark->sparkContext() );
+
+my JavaRDD::String $dataSet = $sc->textFile( "test.txt" );
+
+my $v = $dataSet->first();
+print "got [$v]\n";
+
+$spark->stop();
+