Skip to content

Commit

Permalink
julia 0.7 & 1.0 version
Browse files Browse the repository at this point in the history
  • Loading branch information
montyvesselinov committed Sep 17, 2018
1 parent f485d39 commit 47d6008
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 35 deletions.
6 changes: 0 additions & 6 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,6 @@ julia:
- 0.6
notifications:
email: false
addons:
apt:
packages:
- hdf5-tools
before_script:
- if [ $TRAVIS_OS_NAME = osx ]; then brew install gcc; brew link --overwrite gcc; brew install hdf5; fi
# script:
# - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
# - julia -e 'Pkg.clone(pwd()); Pkg.build("RobustPmap"); Pkg.test("RobustPmap"; coverage=true)';
Expand Down
5 changes: 3 additions & 2 deletions REQUIRE
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
julia 0.6
JLD
julia 0.7
JLD2
FileIO
Compat 0.7.15
12 changes: 7 additions & 5 deletions src/RobustPmap.jl
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@ LA-CC-15-080; Copyright Number Assigned: C16008
"""
module RobustPmap

import Distributed
import Compat.String

import JLD
import JLD2
import FileIO

"Check for type exceptions"
function checkexceptions(x::Any, t::Type=Any)
Expand All @@ -49,7 +51,7 @@ end

"Robust pmap call"
function rpmap(f::Function, args...; t::Type=Any)
x = pmap(f, args...; on_error=x->x)
x = Distributed.pmap(f, args...; on_error=x->x)
checkexceptions(x, t)
return convert(Array{t, 1}, x)
end
Expand All @@ -63,13 +65,13 @@ function crpmap(f::Function, checkpointfrequency::Int, filerootname::String, arg
end
for i = 1:ceil(Int, length(args[1]) / checkpointfrequency)
r = (1 + (i - 1) * checkpointfrequency):min(length(args[1]), (i * checkpointfrequency))
filename = string(filerootname, "_", hashargs, "_", i, ".jld")
filename = string(filerootname, "_", hashargs, "_", i, ".jld2")
theseargs = map(x->x[r], args)
if isfile(filename)
partialresult = JLD.load(filename, "partialresult")
partialresult = FileIO.load(filename, "partialresult")
else
partialresult = rpmap(f, map(x->x[r], args)...; t=t)
JLD.save(filename, "partialresult", partialresult)
FileIO.save(filename, "partialresult", partialresult)
end
append!(fullresult, partialresult)
end
Expand Down
46 changes: 24 additions & 22 deletions test/runtests.jl
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
using Distributed

procschage = false
if nworkers() < 2
if Distributed.nworkers() < 2
procschage = true
addprocs(2)
reload("RobustPmap")
import Base.Test
Distributed.addprocs(2)
import RobustPmap
import Test
end

if !isdefined(Symbol("@stderrcapture"))
@everywhere macro stderrcapture(block)
if !isdefined(Base, Symbol("@stderrcapture"))
@Distributed.everywhere macro stderrcapture(block)
quote
if ccall(:jl_generating_output, Cint, ()) == 0
errororiginal = STDERR;
errororiginal = stderr;
(errR, errW) = redirect_stderr();
errorreader = @async readstring(errR);
errorreader = @async read(errR, String);
evalvalue = $(esc(block))
redirect_stderr(errororiginal);
close(errW);
Expand All @@ -23,47 +25,47 @@ if !isdefined(Symbol("@stderrcapture"))
end
end

@stderrcapture @everywhere f1(x) = x > 0 ? 1 : 1.
@stderrcapture @Distributed.everywhere f1(x) = x > 0 ? 1 : 1.
@stderrcapture function testtypecheck()
@Base.Test.test_throws TypeError RobustPmap.rpmap(f1, [-1, 0, 1]; t=Int)
@Test.test_throws TypeError RobustPmap.rpmap(f1, [-1, 0, 1]; t=Int)
end
@stderrcapture function testworks()
@Base.Test.test RobustPmap.rpmap(f1, [-1, 0, 1]) == Any[1., 1., 1]
@Test.test RobustPmap.rpmap(f1, [-1, 0, 1]) == Any[1., 1., 1]
end
@stderrcapture function testparallel()
@Base.Test.test length(unique(RobustPmap.rpmap(i->myid(), 1:2))) != 1
@Test.test length(unique(RobustPmap.rpmap(i->Distributed.myid(), 1:2))) != 1
end
@stderrcapture function testcheckpoint()
result = RobustPmap.crpmap(x->x, 2, joinpath(pwd(), "test"), [-1, 0, 1]; t=Int)
result2 = RobustPmap.crpmap(x->pi, 2, joinpath(pwd(), "test"), [-1, 0, 1]; t=Int)#test it with a different function to make sure it loads from the checkpoints
rm(joinpath(pwd(), string("test", "_", hash(([-1, 0, 1],)), "_1.jld")))
rm(joinpath(pwd(), string("test", "_", hash(([-1, 0, 1],)), "_2.jld")))
@Base.Test.test result == RobustPmap.rpmap(x->x, [-1, 0, 1]; t=Int)
@Base.Test.test result == result2
rm(joinpath(pwd(), string("test", "_", hash(([-1, 0, 1],)), "_1.jld2")))
rm(joinpath(pwd(), string("test", "_", hash(([-1, 0, 1],)), "_2.jld2")))
@Test.test result == RobustPmap.rpmap(x->x, [-1, 0, 1]; t=Int)
@Test.test result == result2
x = rand(100)
y = rand(100)
result = RobustPmap.crpmap((x, y)->x + y, 10, joinpath(pwd(), "test"), x, y; t=Float64)
result2 = RobustPmap.crpmap((x, y)->pi, 10, joinpath(pwd(), "test"), x, y; t=Float64)#test it with a different function to make sure it loads from the checkpoints
for i = 1:10
rm(joinpath(pwd(), string("test", "_", hash((x, y)), "_$i.jld")))
rm(joinpath(pwd(), string("test", "_", hash((x, y)), "_$i.jld2")))
end
@Base.Test.test result == RobustPmap.rpmap((x, y)->x + y, x, y; t=Float64)
@Base.Test.test result == result2
@Test.test result == RobustPmap.rpmap((x, y)->x + y, x, y; t=Float64)
@Test.test result == result2
end
@stderrcapture function onlyonproc1(x)
return x
end

@Base.Test.testset "RobustPmap" begin
@Test.testset "RobustPmap" begin
testtypecheck()
testworks()
testparallel()
testcheckpoint()
@Base.Test.test_throws RemoteException RobustPmap.rpmap(onlyonproc1, 1:10)
@Test.test_throws Distributed.RemoteException RobustPmap.rpmap(onlyonproc1, 1:10)
end

if procschage
rmprocs(workers())
Distributed.rmprocs(Distributed.workers())
end

:passed

0 comments on commit 47d6008

Please sign in to comment.