separate into files

standardize output format

output dict from pisum

actual fail and catch

workaround autoscale log bug

ordered plots

results

.travis.yml

@@ -6,7 +6,6 @@ git:
   quiet: true
 
 python:
-- 2.7
 - 3.5
 - 3.6
 - pypy3

Matmul.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from pythonperformance import Path
+from pathlib import Path
 import logging
 import subprocess
 from six import PY2

Pisum.py

@@ -1,92 +1,147 @@
 #!/usr/bin/env python
-from pythonperformance import Path
-import logging
+import warnings
 import subprocess as S
-from six import PY2
 import os
 from argparse import ArgumentParser
-if PY2:
-    FileNotFoundError = OSError
+from typing import Dict
+try:
+    from matplotlib.pyplot import figure, show
+except ImportError:
+    figure = show = None
 
 bdir = 'pisum'
 
 
 def main():
 
     p = ArgumentParser()
-    p.add_argument('-N', type=int, default=1000000)
+    p.add_argument('N', type=int, default=1000000, nargs='?')
     p.add_argument('-Nrun', type=int, default=10)
     p = p.parse_args()
 
-    test_pisum(p.N, p.Nrun)
+    times = benchmark_pisum(p.N, p.Nrun)
+    times = dict(sorted(times.items(), key=lambda x: x[1]))  # Python >= 3.5
 
+    print(times)
 
-def test_pisum(N, Nrun):
+    if times and figure is not None:
+        ax = figure().gca()
+        ax.scatter(times.keys(), times.values())
+        ax.set_title('PiSum, N={}'.format(p.N))
+        ax.set_ylabel('run time [sec.]')
+        ax.set_yscale('log')
+        ax.grid(True)
+        # ax.autoscale(True)  # bug?
+        ax.set_ylim((0.9*min(times.values()), 1.1*max(times.values())))
+        show()
 
+
+def benchmark_pisum(N, Nrun) -> Dict[str, float]:
+    # %% monkey patch windows exe naming
     cexe = './pisumc'
     fexe = './pisumfort'
     if os.name == 'nt':
         cexe = cexe[2:]
         fexe = fexe[2:]
+
+    times = {}
     # %% C
     try:
-        print()
-        S.check_call([cexe, str(N)], cwd='bin/'+bdir)
+        print('--> C')
+        ret = S.check_output([cexe, str(N)], cwd='bin/'+bdir, universal_newlines=True)
+        times['c'] = float(ret.split('\n')[1].split()[0])
     except FileNotFoundError:
-        logging.error('please compile pisum.c as per README')
+        warnings.warn('please compile pisum.c as per README')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
 
     # %% Fortran
     try:
-        print()
-        S.check_call([fexe, str(N), str(Nrun)], cwd='bin/'+bdir)
+        print('--> Fortran')
+        ret = S.check_output([fexe, str(N), str(Nrun)], cwd='bin/'+bdir, universal_newlines=True)
+        times['fortran'] = float(ret.split('\n')[1].split()[0])
     except FileNotFoundError:
-        logging.error('please compile Pisum Fortran code as per README')
-
+        warnings.warn('please compile Pisum Fortran code as per README')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
     # %% Julia
     try:
-        print()
-        S.check_call(['julia', 'pisum.jl', str(N)], cwd=bdir)
+        print('--> Julia')
+        ret = S.check_output(['julia', 'pisum.jl', str(N)], cwd=bdir, universal_newlines=True)
+        times['julia'] = float(ret.split('\n')[1].split()[0])
     except FileNotFoundError:
-        logging.warning('Julia executable not found')
-
+        warnings.warn('Julia executable not found')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
     # %% GDL
     try:
-        print('\n --> GDL')
+        print('--> GDL')
         S.check_call(['gdl', '--version'])
 
-        S.check_call(['gdl', '-q', '-e', 'pisum', '-arg', str(N)], cwd=bdir)
+        ret = S.check_output(['gdl', '-q', '-e', 'pisum', '-arg', str(N)], cwd=bdir, universal_newlines=True)
+        times['gdl'] = float(ret.split('\n')[-2].split()[0])
     except FileNotFoundError:
-        logging.warning('GDL executable not found')
-
+        warnings.warn('GDL executable not found')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
     # %% IDL
     try:
         print('\n --> IDL')
 
-        S.check_call(['idl', '-e', 'pisum', '-arg', str(N)], cwd=bdir)
+        ret = S.check_output(['idl', '-e', 'pisum', '-arg', str(N)], cwd=bdir, universal_newlines=True)
+        times['idl'] = float(ret.split('\n')[-2].split()[0])
     except FileNotFoundError:
-        logging.warning('IDL executable not found')
-
+        warnings.warn('IDL executable not found')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
     # %% Octave
     try:
-        print()
-        S.check_call(['octave-cli', '-q', '--eval', 'pisum({})'.format(N)], cwd=bdir)
+        print('--> Octave')
+        ret = S.check_output(['octave-cli', '-q', '--eval', 'pisum({})'.format(N)], cwd=bdir, universal_newlines=True)
+        times['octave'] = float(ret.split('\n')[1].split()[0])
     except FileNotFoundError:
-        logging.warning('Octave executable not found')
-
+        warnings.warn('Octave executable not found')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
     # %% Matlab
     try:
-        print()
-        S.check_call(['matlab', '-nodesktop', '-nojvm', '-nosplash', '-r',
-                      'pisum({}); exit'.format(N)], cwd=bdir)
+        print('--> Matlab')
+        ret = S.check_output(['matlab', '-nodesktop', '-nojvm', '-nosplash', '-r',
+                              'pisum({}); exit'.format(N)], cwd=bdir, universal_newlines=True)
+        times['matlab'] = float(ret.split('\n')[-2].split()[0])
     except FileNotFoundError:
-        logging.warning('Matlab executable not found')
-
+        warnings.warn('Matlab executable not found')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
     # %% Python
     try:
-        print()
-        S.check_call(['python', 'pisum.py'], cwd=bdir)
+        print('--> Python')
+        ret = S.check_output(['python', 'pisum.py', str(N)], cwd=bdir, universal_newlines=True)
+        times['python'] = float(ret.split('\n')[1].split()[0])
+    except FileNotFoundError:
+        warnings.warn('Python test skipped')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
+    # %% Cython
+    try:
+        print('--> Cython')
+        ret = S.check_output(['python', 'pisum_cython.py', str(N)], cwd=bdir, universal_newlines=True)
+        times['cython'] = float(ret.split('\n')[1].split()[0])
     except FileNotFoundError:
-        logging.warning('Python test skipped')
+        warnings.warn('Cython test skipped')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
+    # %% Numba
+    try:
+        print('--> Numba')
+        ret = S.check_output(['python', 'pisum_numba.py', str(N)], cwd=bdir, universal_newlines=True)
+        times['numba'] = float(ret.split('\n')[1].split()[0])
+    except FileNotFoundError:
+        warnings.warn('Numba test skipped')
+    except S.CalledProcessError:
+        warnings.warn('failed to converge')
+
+    return times
 
 
 # %%

README.md

@@ -13,7 +13,7 @@ Continuous integration conducted on:
 -   MacOS (Travis-CI)
 -   Windows (AppVeyor)
 
-Tested on Python 2.7, 3.5, 3.6 and PyPy3.
+Tested on Python 3.5, 3.6 and PyPy3.
 
 ## Install
 
@@ -27,6 +27,8 @@ Iterative benchmarks, here using the pisum algorithm:
 
     ./RunPisum.py
 
+![Pi Sum benchmark](tests/pisum_1000000.png)
+
 Matrix Multiplication benchmarks:
 
     ./RunMatmul.py

pisum/cpisum.pyx

@@ -1,16 +1,14 @@
-from __future__ import division
-import cython
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-def pisum(int N):
+# import cython
+# for this problem, these options didn't have a noticable speed impact:
+# @cython.boundscheck(False)
+# @cython.wraparound(False)
+def pisum(N: int) -> float:
     """
     Machin formula for Pi http://mathworld.wolfram.com/PiFormulas.html
     """
-    cdef float s
-    cdef int k
+
     s = 0.
     for k in range(1, N+1):
         s += (-1)**(k+1) / (2*k-1)
-    return 4.*s
 
+    return 4.*s

pisum/pisum.c

@@ -15,22 +15,22 @@ int main(int argc, char** argv){
         N = atoi(argv[1]);
     }
 
-    printf("--> C\n");
+    printf("--> C.  N=%d \n", N);
 
 	struct timespec start, end;
-	
+
 	#if (!defined(__APPLE__))
 	  clock_gettime(CLOCK_MONOTONIC, &start);
-	#endif  
-	    
+	#endif
+
     for (int k=1; k<=N; k++) {
        s += pow(-1., k+1.) / (2.*k-1);
     }
-    
+
     #if (!defined(__APPLE__))
       clock_gettime(CLOCK_MONOTONIC, &end);
     #endif
-    
+
     uint64_t diff = BILLION * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec;
 
     s = 4*s;
@@ -41,7 +41,7 @@ int main(int argc, char** argv){
     }
 
     #if (!defined(__APPLE__))
-      printf("pisum: %.3e seconds",(float) diff/BILLION);
+      printf("%.3e seconds \n",(float) diff/BILLION);
     #endif
 
     return EXIT_SUCCESS;

pisum/pisum.f90

@@ -68,11 +68,11 @@ Program run_iter
 
   Implicit None
 
-  integer :: N=100000, Nrun=10, argc, Ni
+  integer :: N=1000000, Nrun=10, argc, Ni
   character(16) :: argv
 
   real(dp) :: t
-  
+
 !  Ni = num_images()
   Ni=1
 
@@ -94,7 +94,7 @@ Program run_iter
 !------pisum----------------
   t = pisum(N, Nrun)
 
-  print  '(A,ES12.4,A)', 'pisum:',t,' sec.'
+  print  '(ES12.4,A)', t,' sec.'
 
 
 End Program

pisum/pisum.jl

@@ -1,5 +1,4 @@
 #!/usr/bin/env julia
-println("--> Julia $VERSION")
 using Test
 
 try
@@ -12,24 +11,23 @@ catch
 end
 
 if length(ARGS) < 1
-  N = 1000
+  N = 1000000
 else
   N = parse(Int, ARGS[1]);
 end
 
-
+println("--> Julia $VERSION    N: $N")
 
 function g(N::Int)
   s = 0.
   for k = 1:N+1
     s = s + (-1)^(k+1) / (2*k-1)
   end
 
-  x=4*s
+  x::Real=4*s
 return x
 end
 
-@test isapprox(g(100), pi, atol=0.01)
-println("pisum:   N: $N")
+@test isapprox(g(N), pi, atol=1e-4)
 o = @benchmark g(N)
-println(o)
+println(minimum(o).time/1e9, " seconds")

pisum/pisum.m

@@ -11,19 +11,24 @@ function pisum(N)
     disp(['--> Matlab ',v.Version])
 end
 
-%% pisum
-fprintf('pisum ')
-  try
+pitry = calcpisum(N);
+
+if abs(pitry-pi)>1e-4
+    warning('Pisum: failed to converge')
+    exit(1)  % necessary for Matlab, sigh
+end
+
+try
     f = @() calcpisum(N);
     t = timeit(f);
-  catch
+catch
     t = inf;
     for i = 1:Nrun
         tic
         calcpisum(N);
         t = min(toc,t);
     end
-  end % try
+end % try
 disp([num2str(t),' sec.'])
 
 end % function pisum
@@ -36,5 +41,3 @@ function pisum(N)
 
 x=4*s;
 end
-
-