first draft

nicolaferraro · Feb 15, 2017 · b14d838 · b14d838
commit b14d838
Show file tree

Hide file tree

Showing 10 changed files with 288 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,5 @@
+.project
+target
+.metadata
+.idea
+*.iml
diff --git a/pom.xml b/pom.xml
@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>me.nicolaferraro.spark</groupId>
+    <artifactId>spark-kubernetes-quickstart</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <!-- Source config -->
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+
+        <!-- Dependency versions -->
+        <spark.version>2.0.1</spark.version>
+
+        <!-- Plugin versions -->
+        <!--<fabric8.maven.plugin.version>3.2.20</fabric8.maven.plugin.version>-->
+        <fabric8.maven.plugin.version>3.2-SNAPSHOT</fabric8.maven.plugin.version>
+        <maven.shade.plugin.version>3.0.0</maven.shade.plugin.version>
+        <scala.maven.plugin.version>3.2.1</scala.maven.plugin.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_2.11</artifactId>
+            <version>${spark.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <sourceDirectory>src/main/scala</sourceDirectory>
+        <plugins>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>${scala.maven.plugin.version}</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <args>
+                        <!-- work-around for https://issues.scala-lang.org/browse/SI-8358 -->
+                        <arg>-nobootcp</arg>
+                    </args>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>${maven.shade.plugin.version}</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <createDependencyReducedPom>false</createDependencyReducedPom>
+                    <transformers>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                            <mainClass>spark.kubernetes.quickstart.Hello</mainClass>
+                        </transformer>
+                    </transformers>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>io.fabric8</groupId>
+                <artifactId>fabric8-maven-plugin</artifactId>
+                <version>${fabric8.maven.plugin.version}</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>resource</goal>
+                            <goal>build</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <generator>
+                        <config>
+                            <java-exec>
+                                <from>radanalyticsio/radanalytics-java-spark</from>
+                            </java-exec>
+                        </config>
+                    </generator>
+                </configuration>
+            </plugin>
+
+
+        </plugins>
+    </build>
+
+</project>
diff --git a/src/main/fabric8/deployment.yaml b/src/main/fabric8/deployment.yaml
@@ -0,0 +1,7 @@
+spec:
+  template:
+    spec:
+      containers:
+      - env:
+        - name: APP_FILE
+          value: spark-kubernetes-quickstart-1.0-SNAPSHOT.jar
diff --git a/src/main/fabric8/raw/oshinko-cm.yml b/src/main/fabric8/raw/oshinko-cm.yml
@@ -0,0 +1,2 @@
+metadata:
+  name: oshinko-spark-driver-config
diff --git a/src/main/fabric8/raw/oshinko-deployment.yml b/src/main/fabric8/raw/oshinko-deployment.yml
@@ -0,0 +1,93 @@
+apiVersion: "extensions/v1beta1"
+kind: "Deployment"
+metadata:
+  labels:
+    provider: "fabric8"
+    group: "io.fabric8.ipaas.apps"
+  name: "oshinko"
+spec:
+  replicas: 1
+  selector:
+    name: oshinko-rest
+    matchLabels:
+      provider: "fabric8"
+      project: "oshinko-rest"
+      group: "io.fabric8.ipaas.apps"
+  template:
+    metadata:
+      labels:
+        name: oshinko-rest
+        provider: "fabric8"
+        project: "oshinko-rest"
+        group: "io.fabric8.ipaas.apps"
+    spec:
+      containers:
+      - env:
+        - name: OSHINKO_SERVER_PORT
+          value: "8081"
+        - name: OSHINKO_REST_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: OSHINKO_CLUSTER_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: OSHINKO_CLUSTER_IMAGE
+          value: radanalyticsio/openshift-spark
+        - name: OSHINKO_WEB_NAME
+          value: oshinko-web
+        image: radanalyticsio/oshinko-rest
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /
+            port: 8081
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        name: oshinko-rest
+        ports:
+        - containerPort: 8081
+          name: o-rest-port
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /
+            port: 8081
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+      - env:
+        - name: OPENSHIFT_OSHINKO_REST
+          value: 127.0.0.1
+        - name: OPENSHIFT_OSHINKO_REST_PORT
+          value: "8081"
+        image: radanalyticsio/oshinko-webui
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /
+            port: 8080
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        name: oshinko-web
+        ports:
+        - containerPort: 8080
+          name: o-web-port
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /
+            port: 8080
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+      serviceAccount: oshinko
diff --git a/src/main/fabric8/raw/oshinko-rb.yml b/src/main/fabric8/raw/oshinko-rb.yml
@@ -0,0 +1,7 @@
+metadata:
+  name: oshinko
+roleRef:
+  name: admin
+subjects:
+- kind: ServiceAccount
+  name: oshinko
diff --git a/src/main/fabric8/raw/oshinko-rest-svc.yml b/src/main/fabric8/raw/oshinko-rest-svc.yml
@@ -0,0 +1,23 @@
+---
+apiVersion: "v1"
+kind: "Service"
+metadata:
+  annotations:
+  labels:
+    name: "oshinko-rest"
+    provider: "fabric8"
+    project: "oshinko-rest"
+    group: "io.fabric8.ipaas.apps"
+    expose: "false"
+  name: "oshinko-rest"
+spec:
+  ports:
+  - name: o-rest-port
+    port: 8081
+    protocol: TCP
+    targetPort: 8081
+  selector:
+    name: oshinko-rest
+    project: "oshinko-rest"
+    provider: "fabric8"
+    group: "io.fabric8.ipaas.apps"
diff --git a/src/main/fabric8/raw/oshinko-sa.yml b/src/main/fabric8/raw/oshinko-sa.yml
@@ -0,0 +1,2 @@
+metadata:
+  name: "oshinko"
diff --git a/src/main/fabric8/raw/oshinko-web-svc.yml b/src/main/fabric8/raw/oshinko-web-svc.yml
@@ -0,0 +1,24 @@
+---
+apiVersion: "v1"
+kind: "Service"
+metadata:
+  annotations:
+    service.alpha.openshift.io/dependencies: '[{"name":"oshinko-rest","namespace":"","kind":"Service"}]'
+  labels:
+    name: "oshinko-web"
+    provider: "fabric8"
+    project: "oshinko-rest"
+    group: "io.fabric8.ipaas.apps"
+    expose: "true"
+  name: "oshinko-web"
+spec:
+  ports:
+  - name: o-web-port
+    port: 8080
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    name: oshinko-rest
+    project: "oshinko-rest"
+    provider: "fabric8"
+    group: "io.fabric8.ipaas.apps"
diff --git a/src/main/scala/spark/kubernetes/quickstart/Hello.scala b/src/main/scala/spark/kubernetes/quickstart/Hello.scala
@@ -0,0 +1,19 @@
+package spark.kubernetes.quickstart
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+/**
+  *
+  */
+object Hello {
+
+  def main(args: Array[String]): Unit = {
+
+    val conf = new SparkConf().setAppName("Hello")
+    val sc = new SparkContext(conf)
+
+    val result = sc.parallelize(1 to 100).reduce(_ + _)
+    println(result)
+  }
+
+}