Update mesos docker container with PAM for user authentication

architecture/ipython-spark-docker-mesos-standalone-draw.io.xml

@@ -0,0 +1 @@
+<mxfile userAgent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36" type="device"><diagram>7Vpbb+o4EP41SGcfQCE3wmOhzTkvlVbqw3l2iRusOjHrhJburz9je0wuEEpLKd1tAIkwdsZz+WbsmTDw5tnmpySr5a1IKB+4TrIZeNcD13XHYTgKIrhSxBdDHE+icDQNDTGVLEFyRbhj/1IkOkhds4QWjYmlELxkqyZxIfKcLsoG7UHw5hIrklr2FeFuQfgu9TdLyqWhRi6KrOi/KEuXdplxODUjRflieST0gax5OdQkGFPDGbG8tFbeDRhOCgFs1FW2mVOujGcNYlSPO0a3QkqaoyCHb3B9FOOJ8DVKuSN2KsV6hdOoLCn6sWldcm+no3fqQuipKMFPKjJayheYgoyGgeWFYHA95PFcWXo8RdqyZuUQaQQdnm55VzrDBardYQLPP9YE3uxB5CUCcRzB75OsgsbftcvnqB3sUTvksMQsYU8N9cN/1gqQM1CyHBLO0nzgXSklQRUqq3G4SvFb87nfy2UhuJCGgUzvf4Au8AF5ndrVX+pSMXI4y+nQKq/uGY9UzHUtqRy0d1U1MCy06zSXaLXZ5XJLC1HA6C0pjGKaJ5jSsG0uBWRQcIemjWepLRgtywz8cT2GsWJJVoqabVKVJEf3EPNFCZcjKXWymkkEWqCkZ5zPjd2AjzfTb8WllOKR1kam0/k8BghofWv0B/1qIViZ+HnJSnq3IgtFeYb1gabwzCD1XRlXX2csSZQKM/T9tXV8B/gbIDfhdRjk0XQ0rb8mZ8J8gEwumu3cCO/BbOcdm+22xNNssC/j93Gv4/6Okyfahz1GeSkU9T0xb+JsNwQs4jG+XwO8DZTT8L5ve//meAfHy0cY/S3k41fe5zwAtob1fxXw9hbEIOJ/gkevOv5DZFvHvz2ZnoR/HyuUHv99vj8T/N9Y3nnhZ+Z/Hxer4b+d/rqKvY5kBJ5MGChvx3KRq8NxuzbUv5ErGuBM5bODKqJ9fecz88vkiDM1TaCXgj8Lmmag2U1FmtE8uZJSPCvUcVIUbAHEehC14R/H4A0VGMrGMckYV6r/ovyJKkR3OKPGwHHieKbohrXt6CiWRngl8WG8g4JiLXU86R0A2z0lkSndbguv1z62JKn7xdIk5aRkkKHqq+5zFq7wt2A6K1pgRBZMCIx2DWUUwLvqXaI2o4l1su3WtRgZpXcYgVOJUtdOW6kJkHc7BQ5DRC6uE6i94pBg4+jgDXBhZKigvPXCkeg+oj/20egO9WsvOE/G+zvRbZtFTXS/3r7q0V1bx7fNYrsPtpuzr6G7dcMHoNv7ruhuAlnvnxcDsu8GjfZX0PS62277HA1swM/Idx03mkYBHBJcp1mGbBtPFwd6O+9/PNCP6PH/b/O0qQC/bZ5+D15MRuiL5r5ovlTRPLFhcpGi2T6eP+qZaNP+29qso0w+ZyU8sc2ui1TCtgTsk8Zup/mrP1GN48Ze/oU7zToyDz9ODZ3m41QE98e3fmxbq2/9XPB81T48+/aY9OYDVvvc7p+rQHhzn6ct2Yl9HvhZ/cfKTK/+qebd/AE=</diagram></mxfile>

dockerfiles/lab41/spark-mesos-client-ipython/Dockerfile

@@ -34,3 +34,14 @@ ADD config/bootstrap/bootstrap.sh /bootstrap.sh
 
 # add runit services
 ADD config/service /etc/service
+
+# configure PAM for user auth
+RUN apt-get update --assume-yes
+RUN apt-get upgrade --assume-yes
+RUN apt-get --assume-yes build-dep pam
+
+#Rebuild and install libpam with --disable-audit option
+RUN export CONFIGURE_OPTS=--disable-audit && \
+    cd /root && \
+    apt-get -b source pam && \
+    dpkg -i libpam-doc*.deb libpam-modules*.deb libpam-runtime*.deb libpam0g*.deb

runtime/ipython/notebooks/Example_Pi.ipynb

@@ -0,0 +1,116 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# import modules\n",
+    "from pyspark import SparkContext, SparkConf\n",
+    "import os, sys, getopt\n",
+    "import xml.etree.ElementTree as ET\n",
+    "\n",
+    "import pandas as pd\n",
+    "from pandas import Series, DataFrame"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SLF4J: Failed to load class \"org.slf4j.impl.StaticLoggerBinder\".\n",
+      "SLF4J: Defaulting to no-operation (NOP) logger implementation\n",
+      "SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.\n",
+      "Found 3 items\n",
+      "-rw-r--r--   3 root     supergroup  252761350 2015-07-17 20:12 /spark/spark-1.3.0-bin-hadoop2.4.tgz\n",
+      "-rw-r--r--   3 root     supergroup  259384921 2015-07-17 20:11 /spark/spark-1.3.1-bin-hadoop2.6.tgz\n",
+      "-rw-r--r--   3 guest    supergroup  245061043 2015-07-15 22:36 /spark/spark-1.4.1-bin-hadoop2.4.tgz\n"
+     ]
+    }
+   ],
+   "source": [
+    "# verify spark binary in HDFS\n",
+    "!hadoop fs -ls /spark"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "# update SparkContext to use HDFS Spark binary\n",
+    "# TODO: setup this portion to configure on server startup\n",
+    "from pyspark import SparkConf, SparkContext\n",
+    "sc.stop()\n",
+    "conf = (SparkConf()\n",
+    "         .setMaster(\"mesos://mesos-master-fqdn:5050\")\n",
+    "         .setAppName(\"Calculate Pi\")\n",
+    "         .set(\"spark.executor.uri\", \"hdfs:///spark/spark-1.3.0-bin-hadoop2.4.tgz\"))\n",
+    "sc = SparkContext(conf = conf)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Pi is roughly 3.036000\n"
+     ]
+    }
+   ],
+   "source": [
+    "# simple example to test Spark job\n",
+    "from random import *\n",
+    "\n",
+    "def sample(p):\n",
+    "    x, y = random(), random()\n",
+    "    return 1 if x*x + y*y < 1 else 0\n",
+    "\n",
+    "# estimate Pi\n",
+    "NUM_SAMPLES=1000\n",
+    "count = sc.parallelize(xrange(0, NUM_SAMPLES)).map(sample).reduce(lambda a, b: a + b)\n",
+    "print \"Pi is roughly %f\" % (4.0 * count / NUM_SAMPLES)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}