Skip to content
Browse files

first version

  • Loading branch information...
1 parent 02f5eee commit 0adee78c894848993106522f5c2038eb28702acd @zonca committed Nov 13, 2012
Showing with 561 additions and 0 deletions.
  1. +57 −0 PythonHPC.md
  2. +1 −0 buildpres
  3. +115 −0 ipythonparallel.ipynb
  4. +388 −0 pytrilinos.ipynb
View
57 PythonHPC.md
@@ -0,0 +1,57 @@
+% Python for High Performance Computing
+% Andrea Zonca, UCSB
+% 13 Nov 2012
+
+# Topics
+
+* Trivially parallel tasks: IPython parallel
+* Fully parallel software: HDF5 and PyTrilinos
+
+# Trivially parallel tasks with IPython parallel
+
+## Architecture:
+
+* Submit function+arguments to a queue (managed by the `controller`) consumed by workers (engines)
+* function is *serial*, no IPython-specific, easy debugging
+* works the same in serial, locally on multi-core, or on large cluster
+
+# IPython parallel: Local example
+
+open IPython notebook `ipythonparallel.ipynb`
+
+## Tips
+
+Launch `ipcluster` from same folder of the script so the engines have same path
+
+Define the function in its own module, otherwise necessary to rewrite the import statements
+
+# IPython parallel on a HPC cluster
+
+* Launch `ipcontroller` on the login node
+* Submit ipengine job to the queue management system (see documentation on how to build a pbs script)
+* Open IPython on the login node and submit jobs
+to the controller
+
+# Example run for Planck
+
+* 1.5TB of raw data in ~3000 FITS files
+* read files, simple processing and rewrite to disk
+* run IPython parallel with up to 500 engines
+* process the jobs in ~3 hours
+
+# Distributed Linear Algebra
+
+Large complete C++ packages with Python support
+
+* PETSC, petsc4py
+* Trilinos, PyTrilinos
+
+Both use C++ for MPI communication and LAPACK/BLAS for computing
+
+Both subclass numpy arrays
+
+# Trilinos example
+
+`pytrilinos.ipynb`
+
+export pytrilinos.py and run with mpirun -n 3 python pytrilinos.py
View
1 buildpres
@@ -0,0 +1 @@
+pandoc --self-contained --webtex -i -t s5 PythonHPC.md -o PythonHPC.html
View
115 ipythonparallel.ipynb
@@ -0,0 +1,115 @@
+{
+ "metadata": {
+ "name": "ipythonparallel"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+ {
+ "cells": [
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "from IPython.parallel import Client\n",
+ "rc = Client()\n",
+ "# create the balanced view object\n",
+ "lview = rc.load_balanced_view()"
+ ],
+ "language": "python",
+ "metadata": {
+ "slideshow": {
+ "slide_start": false
+ }
+ },
+ "outputs": [],
+ "prompt_number": 1
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "def process_job(filename):\n",
+ " # to avoid importing functions better\n",
+ " # define the function in a module and import it\n",
+ " from time import sleep\n",
+ " sleep(4)\n",
+ " open(filename, 'w').close()"
+ ],
+ "language": "python",
+ "metadata": {
+ "slideshow": {
+ "slide_start": false
+ }
+ },
+ "outputs": [],
+ "prompt_number": 2
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "filenames = [\"out/%d.dat\" % i for i in range(10)]\n",
+ "print filenames"
+ ],
+ "language": "python",
+ "metadata": {
+ "slideshow": {
+ "slide_start": false
+ }
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": [
+ "['out/0.dat', 'out/1.dat', 'out/2.dat', 'out/3.dat', 'out/4.dat', 'out/5.dat', 'out/6.dat', 'out/7.dat', 'out/8.dat', 'out/9.dat']\n"
+ ]
+ }
+ ],
+ "prompt_number": 3
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "lview.map(process_job, filenames)"
+ ],
+ "language": "python",
+ "metadata": {
+ "slideshow": {
+ "slide_start": false
+ }
+ },
+ "outputs": [
+ {
+ "output_type": "pyout",
+ "prompt_number": 4,
+ "text": [
+ "<AsyncMapResult: process_job>"
+ ]
+ }
+ ],
+ "prompt_number": 4
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "for i in range(100, 110):\n",
+ " lview.apply_async(process_job, \"out/%d.dat\" % i)"
+ ],
+ "language": "python",
+ "metadata": {
+ "slideshow": {
+ "slide_start": false
+ }
+ },
+ "outputs": [],
+ "prompt_number": 5
+ }
+ ],
+ "metadata": {}
+ }
+ ]
+}
View
388 pytrilinos.ipynb
@@ -0,0 +1,388 @@
+{
+ "metadata": {
+ "name": "pytrilinos"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+ {
+ "cells": [
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "from time import sleep\n",
+ "import numpy as np\n",
+ "from PyTrilinos import Epetra"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [],
+ "prompt_number": 68
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "# initialize MPI\n",
+ "Comm = Epetra.PyComm()"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [],
+ "prompt_number": 69
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "NumElements = 10\n",
+ "MapUnique = Epetra.Map(NumElements, 0, Comm)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [],
+ "prompt_number": 70
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "print \"Unique map\"\n",
+ "print MapUnique"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": [
+ "Unique map\n",
+ "\n",
+ "Number of Global Elements = 10\n",
+ "Number of Global Points = 10\n",
+ "Maximum of all GIDs = 9\n",
+ "Minimum of all GIDs = 0\n",
+ "Index Base = 0\n",
+ "Constant Element Size = 1\n",
+ "\n",
+ "Number of Local Elements = 10\n",
+ "Number of Local Points = 10\n",
+ "Maximum of my GIDs = 9\n",
+ "Minimum of my GIDs = 0\n",
+ "\n",
+ " MyPID Local Index Global Index \n",
+ " 0 0 0 \n",
+ " 0 1 1 \n",
+ " 0 2 2 \n",
+ " 0 3 3 \n",
+ " 0 4 4 \n",
+ " 0 5 5 \n",
+ " 0 6 6 \n",
+ " 0 7 7 \n",
+ " 0 8 8 \n",
+ " 0 9 9 \n",
+ "\n"
+ ]
+ }
+ ],
+ "prompt_number": 71
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "# Partially overlapping distribution\n",
+ "LocalElements = { 0: [0, 1, 2, 3, 4, 8],\n",
+ " 1: [3, 4, 5, 6],\n",
+ " 2: [6, 7, 8, 9] \n",
+ " }\n",
+ "MapOverl = Epetra.Map(-1, LocalElements[Comm.MyPID()], 0, Comm)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [],
+ "prompt_number": 72
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "print \"Partially overlapping map\"\n",
+ "print MapOverl"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": [
+ "Partially overlapping map\n",
+ "\n",
+ "Number of Global Elements = 6\n",
+ "Number of Global Points = 6\n",
+ "Maximum of all GIDs = 8\n",
+ "Minimum of all GIDs = 0\n",
+ "Index Base = 0\n",
+ "Constant Element Size = 1\n",
+ "\n",
+ "Number of Local Elements = 6\n",
+ "Number of Local Points = 6\n",
+ "Maximum of my GIDs = 8\n",
+ "Minimum of my GIDs = 0\n",
+ "\n",
+ " MyPID Local Index Global Index \n",
+ " 0 0 0 \n",
+ " 0 1 1 \n",
+ " 0 2 2 \n",
+ " 0 3 3 \n",
+ " 0 4 4 \n",
+ " 0 5 8 \n",
+ "\n"
+ ]
+ }
+ ],
+ "prompt_number": 73
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "# Importer from overlapping to unique\n",
+ "Exporter = Epetra.Export(MapOverl, MapUnique)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [],
+ "prompt_number": 74
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "# Create vectors\n",
+ "XOverl = Epetra.Vector(MapOverl)\n",
+ "XUnique = Epetra.Vector(MapUnique) #initialized to 0"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [],
+ "prompt_number": 75
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "XOverl[:] = Comm.MyPID() + 1"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [],
+ "prompt_number": 76
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "Comm.Barrier()\n",
+ "def printVector(label, vector):\n",
+ " Comm.Barrier()\n",
+ " if Comm.MyPID() == 0: print label\n",
+ " sleep(Comm.MyPID())\n",
+ " print (Comm.MyPID(), vector)\n",
+ "printVector(\"Overlapping Vector\", XOverl)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": [
+ "[ 1. 1. 1. 1. 1. 1.]\n",
+ "(0, 'Overlapping Vector')\n"
+ ]
+ }
+ ],
+ "prompt_number": 77
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "XUnique.Export(XOverl, Exporter, Epetra.Average)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [
+ {
+ "output_type": "pyout",
+ "prompt_number": 78,
+ "text": [
+ "0"
+ ]
+ }
+ ],
+ "prompt_number": 78
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "printVector(\"Unique Vector\", XUnique)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": [
+ "[ 1. 1. 1. 1. 1. 0. 0. 0. 1. 0.]\n",
+ "(0, 'Unique Vector')\n"
+ ]
+ }
+ ],
+ "prompt_number": 79
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "M=np.array([[ 2., -1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
+ " [-1., 2., -1., 0., 0., 0., 0., 0., 0., 0.],\n",
+ " [ 0., -1., 2., -1., 0., 0., 0., 0., 0., 0.],\n",
+ " [ 0., 0., -1., 2., -1., 0., 0., 0., 0., 0.],\n",
+ " [ 0., 0., 0., -1., 2., -1., 0., 0., 0., 0.],\n",
+ " [ 0., 0., 0., 0., -1., 2., -1., 0., 0., 0.],\n",
+ " [ 0., 0., 0., 0., 0., -1., 2., -1., 0., 0.],\n",
+ " [ 0., 0., 0., 0., 0., 0., -1., 2., -1., 0.],\n",
+ " [ 0., 0., 0., 0., 0., 0., 0., -1., 2., -1.],\n",
+ " [ 0., 0., 0., 0., 0., 0., 0., 0., -1., 2.]])\n"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [],
+ "prompt_number": 80
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "MyGlobalElements = MapUnique.MyGlobalElements()\n",
+ "Matrix = Epetra.CrsMatrix(Epetra.Copy, MapUnique, 0)\n",
+ "for i in MyGlobalElements:\n",
+ " if i > 0:\n",
+ " Matrix[i, i - 1] = -1\n",
+ " if i < NumElements - 1:\n",
+ " Matrix[i, i + 1] = -1\n",
+ " Matrix[i, i] = 2.\n",
+ "assert Matrix.FillComplete() == 0"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [],
+ "prompt_number": 81
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "for i in MyGlobalElements:\n",
+ " print \"%d: Matrix(%d, %d) = %e\" %(Comm.MyPID(), i, i, Matrix[i, i])"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": [
+ "0: Matrix(0, 0) = 2.000000e+00\n",
+ "0: Matrix(1, 1) = 2.000000e+00\n",
+ "0: Matrix(2, 2) = 2.000000e+00\n",
+ "0: Matrix(3, 3) = 2.000000e+00\n",
+ "0: Matrix(4, 4) = 2.000000e+00\n",
+ "0: Matrix(5, 5) = 2.000000e+00\n",
+ "0: Matrix(6, 6) = 2.000000e+00\n",
+ "0: Matrix(7, 7) = 2.000000e+00\n",
+ "0: Matrix(8, 8) = 2.000000e+00\n",
+ "0: Matrix(9, 9) = 2.000000e+00\n"
+ ]
+ }
+ ],
+ "prompt_number": 82
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "XUnique2 = Epetra.Vector(MapUnique)\n",
+ "Matrix.Multiply(False, XUnique, XUnique2)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [
+ {
+ "output_type": "pyout",
+ "prompt_number": 83,
+ "text": [
+ "0"
+ ]
+ }
+ ],
+ "prompt_number": 83
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "printVector(\"Unique Vector\", XUnique2)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": [
+ "[ 1. 0. 0. 0. 1. -1. 0. -1. 2. -1.]\n",
+ "(0, 'Unique Vector')\n"
+ ]
+ }
+ ],
+ "prompt_number": 84
+ },
+ {
+ "cell_type": "code",
+ "collapsed": false,
+ "input": [
+ "from PyTrilinos import AztecOO\n",
+ "Solution = Epetra.Vector(MapUnique)\n",
+ "LinProb = Epetra.LinearProblem(Matrix, Solution, XUnique2)\n",
+ "IterSolver = AztecOO.AztecOO(LinProb)\n",
+ "IterSolver.Iterate(10, 1e-9)\n",
+ "printVector(\"Correct Solution\", XUnique)\n",
+ "printVector(\"Iterative Solution\", Solution)"
+ ],
+ "language": "python",
+ "metadata": {},
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": [
+ "[ 1. 1. 1. 1. 1. 0. 0. 0. 1. 0.]\n",
+ "(0, 'Solution')\n"
+ ]
+ }
+ ],
+ "prompt_number": 85
+ }
+ ],
+ "metadata": {}
+ }
+ ]
+}

0 comments on commit 0adee78

Please sign in to comment.
Something went wrong with that request. Please try again.