Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] increase test quality #60

Closed
wants to merge 1 commit into from
Closed

[WIP] increase test quality #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Empty file added test/equality_metrics.jl
View file
Open in desktop
Empty file.
Empty file added test/fileio.jl
View file
Open in desktop
Empty file.
0 test/references/dataframe.csv → test/references/dataframe/dataframe.csv
View file
Open in desktop
File renamed without changes.
0 test/references/camera.png → test/references/image/camera.png
View file
Open in desktop
File renamed without changes
0 test/references/camera.sha256 → test/references/image/camera.sha256
View file
Open in desktop
File renamed without changes.
0 test/references/camera.txt → test/references/image/camera.txt
View file
Open in desktop
File renamed without changes.
0 test/references/lena.sha256 → test/references/image/lena.sha256
View file
Open in desktop
File renamed without changes.
0 test/references/lena.txt → test/references/image/lena.txt
View file
Open in desktop
File renamed without changes.
0 test/references/ansii.txt → test/references/string/ansii.txt
View file
Open in desktop
File renamed without changes.
0 test/references/number1.sha256 → test/references/string/number1.sha256
View file
Open in desktop
File renamed without changes.
0 test/references/string1.nottxt → test/references/string/string1.nottxt
View file
Open in desktop
File renamed without changes.
0 test/references/string1.sha256 → test/references/string/string1.sha256
View file
Open in desktop
File renamed without changes.
0 test/references/string1.txt → test/references/string/string1.txt
View file
Open in desktop
File renamed without changes.
0 test/references/string2.sha256 → test/references/string/string2.sha256
View file
Open in desktop
File renamed without changes.
0 test/references/string2.txt → test/references/string/string2.txt
View file
Open in desktop
File renamed without changes.
0 test/references/string3.txt → test/references/string/string3.txt
View file
Open in desktop
File renamed without changes.
0 test/references/string4.txt → test/references/string/string4.txt
View file
Open in desktop
File renamed without changes.
0 test/references/string5.txt → test/references/string/string5.txt
View file
Open in desktop
File renamed without changes.
0 test/references/string6.txt → test/references/string/string6.txt
View file
Open in desktop
File renamed without changes.
Empty file added test/render.jl
View file
Open in desktop
Empty file.
152 changes: 20 additions & 132 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using Test
using ImageInTerminal, TestImages, ImageCore, ImageTransformations
using DataFrames, CSVFiles
using Random

if isinteractive()
Expand All @@ -9,145 +10,32 @@ else
@info ("Ten tests should correctly report failure in the transcript"
* " (but not the test summary).")
end

# check for ambiguities
refambs = detect_ambiguities(ImageInTerminal, Base, Core)

using ReferenceTests
ambs = detect_ambiguities(ReferenceTests, ImageInTerminal, Base, Core)

strip_summary(content::String) = join(split(content, "\n")[2:end], "\n")

@testset "ReferenceTests" begin

@test Set(setdiff(ambs, refambs)) == Set{Tuple{Method,Method}}()

# load/create some example images
lena = testimage("lena_color_256")
camera = testimage("cameraman")
cameras = similar(camera, size(camera)..., 2)
copyto!(view(cameras,:,:,1), camera)
copyto!(view(cameras,:,:,2), camera)
square = Gray{N0f8}[0.1 0.2 0.3; 0.4 0.5 0.6; 0.7 0.6 0.9]
rgb_rect = rand(RGB{N0f8}, 2, 3)

@testset "io2str" begin
@test_throws LoadError eval(@macroexpand @io2str(::IO))
@test_throws ArgumentError @io2str(2)
@test_throws ArgumentError @io2str(string(2))
@test @io2str(print(::IO, "foo")) == "foo"
@test @io2str(println(::IO, "foo")) == "foo\n"
@test @io2str(show(::IO, "foo")) == "\"foo\""
A = ones(30,30)
@test @io2str(show(IOContext(::IO, :limit => true, :displaysize => (5,5)), A)) == "[1.0 1.0 … 1.0 1.0; 1.0 1.0 … 1.0 1.0; … ; 1.0 1.0 … 1.0 1.0; 1.0 1.0 … 1.0 1.0]"
end

@testset "withcolor" begin
@test_throws ArgumentError @withcolor throw(ArgumentError("foo"))
@test @withcolor Base.have_color == true
@test @withcolor @io2str(printstyled(IOContext(::IO, :color => true), "test", color=:green)) == "\e[32mtest\e[39m"
end

@testset "string as txt" begin
foo = "foo"
@test_reference "references/string1.txt" foo * "bar"
@test_reference "references/string1.txt" [foo * "bar"]
A = ones(30,30)
@test_reference "references/string2.txt" @io2str show(IOContext(::IO, :limit=>true, :displaysize=>(5,5)), A)
@test_reference "references/string3.txt" 1337
@test_reference "references/string3.txt" 1338 by=(ref, x)->isapprox(ref, x; atol=10)
@test_reference "references/string4.txt" strip_summary(@io2str show(::IO, MIME"text/plain"(), Int64.(collect(1:5))))

# ignore CRLF/LF differences
@test_reference "references/string5.txt" """
This is a\r
multiline string that does not end with a new line."""
@test_reference "references/string5.txt" """
This is a
multiline string that does not end with a new line."""
const refroot = joinpath(@__DIR__(), "references")

@test_reference "references/string6.txt" """
This on the other hand is a
multiline string that does indeed end with a new line.
"""

@test_throws ErrorException @test_reference "references/string1.txt" "intentionally wrong to check that this message prints"
@test_throws ErrorException @test_reference "references/string5.txt" """
This is an incorrect
multiline string that does not end with a new line."""
end
test_files = [
"equality_metrics.jl",
"fileio.jl",
"utils.jl",
"render.jl",
"test_reference.jl",
]

@testset "string as unknown file type" begin
@test_reference "references/string1.nottxt" "This is not a .txt file, but it should be treated as such.\n"
end

@testset "images as txt using ImageInTerminal" begin
#@test_throws MethodError @test_reference "references/fail.txt" rand(2,2)

@test_reference "references/camera.txt" camera size=(5,10)
@test_reference "references/lena.txt" lena
end

@testset "plain ansi string" begin
@test_reference(
"references/ansii.txt",
@io2str(printstyled(IOContext(::IO, :color=>true), "this should be blue", color=:blue)),
render = ReferenceTests.BeforeAfterFull()
)
@test_throws ErrorException @test_reference(
"references/ansii.txt",
@io2str(printstyled(IOContext(::IO, :color=>true), "this should be red", color=:red)),
render = ReferenceTests.BeforeAfterFull()
)
end

@testset "string as SHA" begin
@test_reference "references/number1.sha256" 1337
foo = "foo"
@test_reference "references/string1.sha256" foo * "bar"
A = ones(30,30)
@test_reference "references/string2.sha256" @io2str show(IOContext(::IO, :limit=>true, :displaysize=>(5,5)), A)
end

@testset "images as SHA" begin
@test_reference "references/camera.sha256" camera
@test_reference "references/lena.sha256" convert(Matrix{RGB{Float64}}, lena)
end

@testset "images as PNG" begin
@test_reference "references/camera.png" imresize(camera, (64,64))
@test_reference "references/camera.png" imresize(camera, (64,64)) by=psnr_equality(25)
@test_throws ErrorException @test_reference "references/camera.png" imresize(lena, (64,64))
@test_throws Exception @test_reference "references/camera.png" camera # unequal size
end

using DataFrames, CSVFiles
@testset "DataFrame as CSV" begin
@test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"])
@test_throws ErrorException @test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["c","b","c"])

end

@testset "Create new $ext" for (ext, val) in (
(".csv", DataFrame(v1=[1,2,3], v2=["c","b","c"])),
(".png", imresize(camera, (64,64))),
(".txt", "Lorem ipsum dolor sit amet, labore et dolore magna aliqua."),
)
newfilename = "references/newfilename.$ext"
@assert !isfile(newfilename)
@test_reference newfilename val # this should create it
@test isfile(newfilename) # Was created
@test_reference newfilename val # Matches expected content
rm(newfilename, force=true)
end

@testset "Create new image as txt" begin
# This is a sperate testset as need to use the `size` argument to ``@test_reference`
newfilename = "references/new_camera.txt"
@assert !isfile(newfilename)
@test_reference newfilename camera size=(5,10) # this should create it
@test isfile(newfilename) # Was created
@test_reference newfilename camera size=(5,10) # Matches expected content
rm(newfilename, force=true)
end
include("testutils.jl")

@testset "ReferenceTests" begin
@test Set(setdiff(ambs, refambs)) == Set{Tuple{Method,Method}}()

for file in test_files
filename = first(splitext(file))
@testset "File: $filename" begin
include(file)
end
end
end # top level testset
41 changes: 41 additions & 0 deletions test/test_reference.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
type_files = [
"string.jl",
"image.jl",
"number_array.jl",
"dataframe.jl"
]

for file in type_files
type = first(splitext(file))
@testset "Type: $type" begin
include(joinpath("types", file))
end
end

# TODO: split this testset into previous files
@testset "Reference regeneration" begin
camera = testimage("cameraman")

@testset "Create new $ext" for (ext, val) in (
(".csv", DataFrame(v1=[1,2,3], v2=["c","b","c"])),
(".png", imresize(camera, (64,64))),
(".txt", "Lorem ipsum dolor sit amet, labore et dolore magna aliqua."),
)
newfilename = joinpath(refroot, "newfilename.$ext")
@assert !isfile(newfilename)
@test_reference newfilename val # this should create it
@test isfile(newfilename) # Was created
@test_reference newfilename val # Matches expected content
rm(newfilename, force=true)
end

@testset "Create new image as txt" begin
# This is a sperate testset as need to use the `size` argument to ``@test_reference`
newfilename = joinpath(refroot, "new_camera.txt")
@assert !isfile(newfilename)
@test_reference newfilename camera size=(5,10) # this should create it
@test isfile(newfilename) # Was created
@test_reference newfilename camera size=(5,10) # Matches expected content
rm(newfilename, force=true)
end
end
1 change: 1 addition & 0 deletions test/testutils.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
strip_summary(content::String) = join(split(content, "\n")[2:end], "\n")
7 changes: 7 additions & 0 deletions test/types/dataframe.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
refdir = joinpath(refroot, "dataframe")

@testset "DataFrame as CSV" begin
@test_reference joinpath(refdir, "dataframe.csv") DataFrame(v1=[1,2,3], v2=["a","b","c"])
@test_throws ErrorException @test_reference joinpath(refdir, "dataframe.csv") DataFrame(v1=[1,2,3], v2=["c","b","c"])

end
29 changes: 29 additions & 0 deletions test/types/image.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# load/create some example images
refdir = joinpath(refroot, "image")

lena = testimage("lena_color_256")
camera = testimage("cameraman")
cameras = similar(camera, size(camera)..., 2)
copyto!(view(cameras,:,:,1), camera)
copyto!(view(cameras,:,:,2), camera)
square = Gray{N0f8}[0.1 0.2 0.3; 0.4 0.5 0.6; 0.7 0.6 0.9]
rgb_rect = rand(RGB{N0f8}, 2, 3)

@testset "images as txt using ImageInTerminal" begin
#@test_throws MethodError @test_reference "references/fail.txt" rand(2,2)

@test_reference joinpath(refdir, "camera.txt") camera size=(5,10)
@test_reference joinpath(refdir, "lena.txt") lena
end

@testset "images as SHA" begin
@test_reference joinpath(refdir, "camera.sha256") camera
@test_reference joinpath(refdir, "lena.sha256") convert(Matrix{RGB{Float64}}, lena)
end

@testset "images as PNG" begin
@test_reference joinpath(refdir, "camera.png") imresize(camera, (64,64))
@test_reference joinpath(refdir, "camera.png") imresize(camera, (64,64)) by=psnr_equality(25)
@test_throws ErrorException @test_reference joinpath(refdir, "camera.png") imresize(lena, (64,64))
@test_throws Exception @test_reference joinpath(refdir, "camera.png") camera # unequal size
end
1 change: 1 addition & 0 deletions test/types/number_array.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
refdir = joinpath(refroot, "number_array")
55 changes: 55 additions & 0 deletions test/types/string.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
refdir = joinpath(refroot, "string")

@testset "string as unknown file type" begin
@test_reference joinpath(refdir, "string1.nottxt") "This is not a .txt file, but it should be treated as such.\n"
end

@testset "string as txt" begin
foo = "foo"
@test_reference joinpath(refdir, "string1.txt") foo * "bar"
@test_reference joinpath(refdir, "string1.txt") [foo * "bar"]
A = ones(30,30)
@test_reference joinpath(refdir, "string2.txt") @io2str show(IOContext(::IO, :limit=>true, :displaysize=>(5,5)), A)
@test_reference joinpath(refdir, "string3.txt") 1337
@test_reference joinpath(refdir, "string3.txt") 1338 by=(ref, x)->isapprox(ref, x; atol=10)
@test_reference joinpath(refdir, "string4.txt") strip_summary(@io2str show(::IO, MIME"text/plain"(), Int64.(collect(1:5))))

# ignore CRLF/LF differences
@test_reference joinpath(refdir, "string5.txt") """
This is a\r
multiline string that does not end with a new line."""
@test_reference joinpath(refdir, "string5.txt") """
This is a
multiline string that does not end with a new line."""

@test_reference joinpath(refdir, "string6.txt") """
This on the other hand is a
multiline string that does indeed end with a new line.
"""

@test_throws ErrorException @test_reference joinpath(refdir, "string1.txt") "intentionally wrong to check that this message prints"
@test_throws ErrorException @test_reference joinpath(refdir, "string5.txt") """
This is an incorrect
multiline string that does not end with a new line."""
end

@testset "string as SHA" begin
@test_reference joinpath(refdir, "number1.sha256") 1337
foo = "foo"
@test_reference joinpath(refdir, "string1.sha256") foo * "bar"
A = ones(30,30)
@test_reference joinpath(refdir, "string2.sha256") @io2str show(IOContext(::IO, :limit=>true, :displaysize=>(5,5)), A)
end

@testset "plain ansi string" begin
@test_reference(
joinpath(refdir, "ansii.txt"),
@io2str(printstyled(IOContext(::IO, :color=>true), "this should be blue", color=:blue)),
render = ReferenceTests.BeforeAfterFull()
)
@test_throws ErrorException @test_reference(
joinpath(refdir, "ansii.txt"),
@io2str(printstyled(IOContext(::IO, :color=>true), "this should be red", color=:red)),
render = ReferenceTests.BeforeAfterFull()
)
end
16 changes: 16 additions & 0 deletions test/utils.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
@testset "io2str" begin
@test_throws LoadError eval(@macroexpand @io2str(::IO))
@test_throws ArgumentError @io2str(2)
@test_throws ArgumentError @io2str(string(2))
@test @io2str(print(::IO, "foo")) == "foo"
@test @io2str(println(::IO, "foo")) == "foo\n"
@test @io2str(show(::IO, "foo")) == "\"foo\""
A = ones(30,30)
@test @io2str(show(IOContext(::IO, :limit => true, :displaysize => (5,5)), A)) == "[1.0 1.0 … 1.0 1.0; 1.0 1.0 … 1.0 1.0; … ; 1.0 1.0 … 1.0 1.0; 1.0 1.0 … 1.0 1.0]"
end

@testset "withcolor" begin
@test_throws ArgumentError @withcolor throw(ArgumentError("foo"))
@test @withcolor Base.have_color == true
@test @withcolor @io2str(printstyled(IOContext(::IO, :color => true), "test", color=:green)) == "\e[32mtest\e[39m"
end