Merge branch 'master' of https://github.com/JuliaMatrices/MatrixDepot.jl

src/common.jl

@@ -39,26 +39,28 @@ function modgroup(prop::Symbol, mats::Union{Nothing,Vector{<:AbstractString}})
     prop in keys(MATRIXCLASS) && daterr("$prop can not be modified.")
 
     user = abspath(user_dir(), "group.jl")
-    s = read(user, String)          # read complete file into s
-    rg = Regex(repr(prop) * r"\W*=>\W*(\[.*\]\W*,\W*\n)".pattern)
-    ppos = findfirst(rg, s)         # locate the prop in user.jl to remove.
-    if ppos !== nothing
-        start_char = first(ppos) - 1    # the start of the line
-        end_char = last(ppos)           # the end of the line
-    else
-        ppos = findnext(r"\);", s, 1)
-        start_char = ppos !== nothing ? first(ppos) - 1 : length(s)
-        end_char = start_char
-    end
-    if mats !== nothing
-        mats = sort(mats)
-    end
-    open(user, "w") do io
-        write(io, s[1:start_char])
+    if isfile(user)
+        s = read(user, String)          # read complete file into s
+        rg = Regex(repr(prop) * r"\W*=>\W*(\[.*\]\W*,\W*\n)".pattern)
+        ppos = findfirst(rg, s)         # locate the prop in user.jl to remove.
+        if ppos !== nothing
+            start_char = first(ppos) - 1    # the start of the line
+            end_char = last(ppos)           # the end of the line
+        else
+            ppos = findnext(r"\);", s, 1)
+            start_char = ppos !== nothing ? first(ppos) - 1 : length(s)
+            end_char = start_char
+        end
         if mats !== nothing
-            propline(io, prop, mats)
+            mats = sort(mats)
+        end
+        open(user, "w") do io
+            write(io, s[1:start_char])
+            if mats !== nothing
+                propline(io, prop, mats)
+            end
+            write(io, s[end_char+1:end])
         end
-        write(io, s[end_char+1:end])
     end
     if mats !== nothing
         usermatrixclass[prop] = mats

src/download.jl

@@ -111,14 +111,19 @@ function gunzip(fname)
 
     destname = rsplit(fname, ".gz", limit=2)[1]
     BUFFSIZE = 1000000
-    open(destname, "w") do f
-        open(GzipDecompressorStream, fname) do g
-            buffer = read(g, BUFFSIZE)
-            while length(buffer) > 0
-                write(f, buffer)
+    try
+        open(destname, "w") do f
+            open(GzipDecompressorStream, fname) do g
                 buffer = read(g, BUFFSIZE)
+                while length(buffer) > 0
+                    write(f, buffer)
+                    buffer = read(g, BUFFSIZE)
+                end
             end
         end
+    catch
+        @warn "decompression error - file $destname set to empty."
+        open(destname, "w") do f; end
     end
     destname
 end
@@ -142,6 +147,7 @@ function loadmatrix(data::RemoteMatrixData)
     dirfn = localfile(data)
     dir = dirname(localdir(data))
     url = redirect(dataurl(data))
+    tarfile = ""
 
     isdir(dir) || mkpath(dir)
     wdir = pwd()
@@ -155,6 +161,8 @@ function loadmatrix(data::RemoteMatrixData)
             run(`tar -xf $rfile`)
             rm(tarfile; force=true)
         end
+    catch
+
     finally
         cd(wdir)
         rm(dirfn, force=true)

src/logical.jl

@@ -109,13 +109,13 @@ isreal(data::MatrixData) = false
 isinteger(data::MatrixData) = false
 isboolean(data::MatrixData) = false
 
-hasinfo(data::RemoteMatrixData) = data.header.m > 0 && data.header.n > 0
+hasinfo(data::RemoteMatrixData) = data.header.m > 0 && data.header.n > 0 # isassigned(data.properties) && data.properties[] !== nothing
 hasinfo(data::MatrixData) = false
 isremote(data::RemoteMatrixData) = true
 isremote(data::MatrixData) = false
-isloaded(data::RemoteMatrixData) = !isempty(data.metadata)
+isloaded(data::RemoteMatrixData) = hasinfo(data) && !isempty(data.metadata)
 isloaded(data::MatrixData) = false
-isunloaded(data::RemoteMatrixData) = isempty(data.metadata)
+isunloaded(data::RemoteMatrixData) = !isloaded(data)
 isunloaded(data::MatrixData) = false
 isuser(data::GeneratedMatrixData{:U}) = true
 isuser(data::MatrixData) = false

src/matrixmarket.jl

@@ -9,6 +9,9 @@ They may be converted to numerical types by multiplying with a number.
 """
 function mmread(filename::AbstractString)
     open(filename, "r") do file
+        if stat(file).size == 0
+            throw(DataError("matrix file $filename is empty"))
+        end
         mmread(file)
     end
 end
@@ -258,6 +261,9 @@ Read header information from mtx file.
 function mmreadheader(file::AbstractString)
     if isfile(file)
         open(file) do io
+          if stat(io).size == 0
+            return nothing
+          end
           line = lowercase(readline(io))
           while true
             token = split(line)

test/common.jl

@@ -33,14 +33,14 @@ REM = length(mdlist("*/*"))
 @test REM >= 2500
 #@test REM in [2757, 2856]   # depends on whether UFL or TAMU url has been used
 @test length(mdlist(:builtin)) == 59
-@test length(mdlist(:user)) in [0, 1]
+@test mdlist(:user) == String["randorth", "randsym"]
 
 @test mdlist("") == []
 @test mdlist("HB/1138_bus") == ["HB/1138_bus"]
 @test mdlist(sp(1)) == ["HB/1138_bus"]
 @test mdlist(mm(1)) == ["Harwell-Boeing/psadmit/1138_bus"]
-@test sort(mdlist(sp(1:3000))) == mdlist("*/*")
-@test sort(mdlist(mm(1:3000))) == mdlist("*/*/*")
+@test sort(mdlist(sp(:))) == mdlist("*/*")
+@test sort(mdlist(mm(1:1000))) == mdlist("*/*/*")
 @test mdlist(builtin(:)) == mdlist(isbuiltin)
 @test mdlist(user(:)) == mdlist(isuser)
 @test mdlist("*") == mdlist(islocal)

test/data/math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.mtx.gz

@@ -0,0 +1,15 @@
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head>
+<title>300 Multiple Choices</title>
+</head><body>
+<h1>Multiple Choices</h1>
+The document name you requested (<code>/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.mtx.gz</code>) could not be found on this server.
+However, we found documents with names similar to the one you requested.<p>Available documents:
+<ul>
+<li><a href="/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.gif">/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.gif</a> (common basename)
+<li><a href="/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.html">/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.html</a> (common basename)
+<li><a href="/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.pse.gz">/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.pse.gz</a> (common basename)
+<li><a href="/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.stats">/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.stats</a> (common basename)
+<li><a href="/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.txt.gz">/pub/MatrixMarket2/Harwell-Boeing/cegb/cegb2802.txt.gz</a> (common basename)
+</ul>
+</body></html>

test/download.jl

@@ -3,14 +3,14 @@
 @test mdlist(isloaded) == []
 import MatrixDepot: load, loadinfo, loadsvd
 
-# sp
+# Suite Sparse
 @test loadinfo("*/1138_bus") == 1 # load only header
 @test load("**/1138_bus") == 2 # loaded both versions (sp and mm)
 @test load("HB/1138_bus") == 0 # count actually loaded
 @test load("Pajek/Journals") == 1
 @test load("wing") == 0 # do not try to load local matrix
 
-# Matrix Markt
+# Matrix Market
 @test load("Harwell-Boeing/smtape/bp___200") == 1
 
 @test mdlist(isloaded) == ["HB/1138_bus", "Harwell-Boeing/psadmit/1138_bus",
@@ -27,7 +27,7 @@ import MatrixDepot: load, loadinfo, loadsvd
 @test loadinfo(MatrixDepot.mdata("Bates/Chem97Zt")) == 1
 @test length(string(mdinfo("Bates/Chem97Zt"))) == 447
 
-# metatdata access with old interface
+# metadata access with old interface
 @test MatrixDepot.metareader(mdopen("Pajek/Journals"), "Journals.mtx") !== nothing
 @test_throws DataError MatrixDepot.metareader(mdopen("*/1138_bus"), "1138_bus_b")
 @test_throws DataError MatrixDepot.metareader(mdopen("that is nothing"))
@@ -53,6 +53,9 @@ mdesc = mdopen("Bai/dwg961b")
 @test !isboolean(mdesc.data)
 @test metasymbols(mdesc) == [:A]
 @test mdesc.A == matrixdepot("Bai/dwg961b")
+mdesc = mdopen("Harwell-Boeing/cegb/cegb2802")
+@test isloaded(mdesc.data) == false
+@test_throws DataError mdesc.A
 
 # an example with rhs and solution
 mdesc = mdopen("DRIVCAV/cavity14")

test/generators.jl

@@ -53,8 +53,7 @@ end
     @inc("test_smallworld.jl")
 end
 
-@testset "regu and include tests" begin
+@testset "regularization methods" begin
     include("regu.jl")
-    include("include_generator.jl")
 end
 

test/include_generator.jl

@@ -34,6 +34,13 @@ n = rand(1:8)
 @test "randsym" in MatrixDepot.mdlist(:random)
 @test "randsym" in MatrixDepot.mdlist(:symmetric)
 
+@addgroup testgroup = ["rand1"]
+@test mdlist(:testgroup) == ["rand1"]
+@modifygroup testgroup = ["rand2"]
+@test mdlist(:testgroup) == ["rand2"]
+@rmgroup testgroup
+@test_throws ArgumentError mdlist(:testgroup)
+
 begin #Testing backward compatibility deprecation. Delete eventually.
     mydepot_warning = "MY_DEPOT_DIR custom code inclusion is deprecated: load custom generators by calling include_generator and reinitializing matrix depot at runtime. For more information, see: https://matrixdepotjl.readthedocs.io/en/latest/user.html. Duplicate warnings will be suppressed."
 
@@ -55,19 +62,15 @@ begin #Testing backward compatibility deprecation. Delete eventually.
     @test_logs (:warn, mydepot_warning) min_level=Logging.Warn match_mode=:any MatrixDepot.init()
     n = rand(1:8)
 
+    MatrixDepot.modgroup(:testgroup, ["rand1"])
+    @test mdlist(:testgroup) == ["rand1"]
+
     @test matrixdepot("randorth", n) !== nothing
     @test mdinfo("randorth") !== nothing
     @test "randorth" in MatrixDepot.mdlist(:random)
 
-    open(joinpath(MatrixDepot.user_dir(), "generator.jl"), "w") do f
-        write(f, "# include your matrix generators below \n")
-    end
-
-    @test_logs min_level=Logging.Warn MatrixDepot.init()
-
-    rm(joinpath(MatrixDepot.user_dir(), "generator.jl"))
     rm(joinpath(MatrixDepot.user_dir(), "group.jl"))
-
+    rm(joinpath(MatrixDepot.user_dir(), "generator.jl"))
     @test_logs min_level=Logging.Warn MatrixDepot.init()
 end
 

test/runtests.jl

@@ -23,6 +23,7 @@ end
 MatrixDepot.init()
 
 include("generators.jl")
+include("include_generator.jl")
 
 @testset "MatrixDepot simulate remote matrix tests" begin
     tests = [
@@ -32,8 +33,6 @@ include("generators.jl")
             "property",
             ]
 
-    tests = []
-
     @testset "$t" for t in tests
         tp = joinpath(@__DIR__(), "$(t).jl")
         println("running $(tp) ...")