Skip to content

Commit

Permalink
Merge pull request #98 from velexi-research/dev/0.6.2
Browse files Browse the repository at this point in the history 
Improve robustness of unit tests. Improve readability of `runtests.jl` output.
  • Loading branch information
@ktchu
ktchu committed Jun 27, 2024
2 parents d572c65 + e281a94 commit bd12e3b
Show file tree
Hide file tree
Showing 9 changed files with 98 additions and 10 deletions.
9 changes: 9 additions & 0 deletions NEWS.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
TestTools Release Notes
=======================

--------------------------------------------------------------------------------------------
v0.6.2 (2024-06-26)
-------------------
**Developer Updates:**
- Updated unit tests.
- Improved robustness of unit tests.
- Added missing calls to `cd(cwd)` to restore working directory during tests.
- Improved readability of `runtests.jl` output.

--------------------------------------------------------------------------------------------
v0.6.1 (2024-06-24)
-------------------
Expand Down
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "TestTools"
uuid = "a8cb73c1-6ac8-4a70-8c57-09fe1bc7c59b"
authors = ["Kevin Chu <kevin@velexi.com> and contributors"]
version = "0.6.1"
version = "0.6.2"

[deps]
ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "TestTools"
version = "0.6.1"
version = "0.6.2"
description = ""
license = "Apache-2.0"
readme = "README.md"
Expand Down
9 changes: 9 additions & 0 deletions test/jlcodestyle/cli_tests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,8 @@ end
"""
@test output == ""

cd(cwd) # Restore current directory

# Case: verbose = true
cd(test_file_dir)

Expand All @@ -249,6 +251,8 @@ end
"""
@test output == expected_output

cd(cwd) # Restore current directory

# Case: `paths` contains only source files without style errors, overwrite = true
cd(test_file_dir)

Expand All @@ -275,8 +279,11 @@ end
"""
@test output == expected_output

cd(cwd) # Restore current directory

# Case: `paths` contains only source files with style errors, overwrite = true
cd(test_file_dir)

bluestyle_pass_file = joinpath(test_file_dir, "blue-style.jl")

tmp_dir = mktempdir()
Expand Down Expand Up @@ -309,6 +316,8 @@ end
Formatting $(realpath(yasstyle_fail_file))
"""
@test output == expected_output

cd(cwd) # Restore current directory
end

@testset EnhancedTestSet "jlcodestyle.cli.run(): invalid arguments" begin
Expand Down
12 changes: 11 additions & 1 deletion test/jlcoverage/cli_tests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ end
cp(joinpath(@__DIR__, "data", "TestPackage"), test_pkg_dir)

# Generate coverage data for TestPackage
cmd_options = `--startup-file=no --project=@. -O0`
cmd_options = `--startup-file=no --project=. -O0`
cmd = Cmd(
`julia $(cmd_options) -e 'import Pkg; Pkg.test(coverage=true)'`; dir=test_pkg_dir
)
Expand All @@ -119,6 +119,7 @@ end

# Case: `paths` contains a single directory, verbose=false
cd(test_pkg_dir)

output = @capture_out begin
cli.run([test_pkg_dir])
end
Expand All @@ -135,11 +136,13 @@ $(joinpath("test", "runtests.jl")) 0
TOTAL 6 3 50.0%
"""
@test output == expected_output

cd(cwd) # Restore current directory

# Case: `paths` contains a single directory, verbose=true
# TODO: add test to check that log messages are generated
cd(test_pkg_dir)

output = @capture_out begin
cli.run([test_pkg_dir]; verbose=true)
end
Expand All @@ -156,10 +159,12 @@ $(joinpath("test", "runtests.jl")) 0
TOTAL 6 3 50.0%
"""
@test output == expected_output

cd(cwd) # Restore current directory

# Case: `paths` contains a file
cd(test_pkg_dir)

src_file = joinpath("src", "methods.jl")
output = @capture_out begin
cli.run([src_file])
Expand All @@ -174,10 +179,12 @@ $(joinpath("src", "methods.jl")) 3
TOTAL 3 1 66.7%
"""
@test output == expected_output

cd(cwd) # Restore current directory

# Case: `paths` is empty and current directory is a Julia package
cd(test_pkg_dir)

output = @capture_out begin
cli.run([])
end
Expand All @@ -193,10 +200,12 @@ $(joinpath("src", "more_methods.jl")) 2
TOTAL 6 3 50.0%
"""
@test output == expected_output

cd(cwd) # Restore current directory

# Case: `paths` is empty and current directory is not a Julia package
cd(joinpath(test_pkg_dir, "src"))

output = @capture_out begin
cli.run([])
end
Expand All @@ -211,6 +220,7 @@ more_methods.jl 2 2 0.0%
TOTAL 6 3 50.0%
"""
@test output == expected_output

cd(cwd) # Restore current directory
end

Expand Down
9 changes: 8 additions & 1 deletion test/jlcoverage/utils_tests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ end
cp(joinpath(@__DIR__, "data", "TestPackage"), test_pkg_dir)

# Generate coverage data for TestPackage
cmd_options = `--startup-file=no --project=@. -O0`
cmd_options = `--startup-file=no --project=. -O0`
cmd = Cmd(
`julia $(cmd_options) -e 'import Pkg; Pkg.test(coverage=true)'`; dir=test_pkg_dir
)
Expand All @@ -74,6 +74,7 @@ end

# Case: default keyword arguments
cd(test_pkg_dir)

output = @capture_out begin
display_coverage(coverage)
end
Expand All @@ -89,10 +90,12 @@ $(joinpath("src", "more_methods.jl")) 2
TOTAL 6 3 50.0%
"""
@test output == expected_output

cd(cwd) # Restore current directory

# Case: startpath = test/jlcoverage/data/TestPackage/src/
cd(test_pkg_dir)

startpath = "src"
output = @capture_out begin
display_coverage(coverage; startpath=startpath)
Expand All @@ -109,10 +112,12 @@ more_methods.jl 2 2 0.0%
TOTAL 6 3 50.0%
"""
@test output == expected_output

cd(cwd) # Restore current directory

# Case: startpath = pwd()/test/jlcoverage/data/TestPackage/src/
cd(test_pkg_dir)

startpath = joinpath(pwd(), "src")
output = @capture_out begin
display_coverage(coverage; startpath=startpath)
Expand All @@ -129,10 +134,12 @@ more_methods.jl 2 2 0.0%
TOTAL 6 3 50.0%
"""
@test output == expected_output

cd(cwd) # Restore current directory

# Case: startpath = ""
cd(test_pkg_dir)

startpath = ""
output = @capture_out begin
display_coverage(coverage; startpath=startpath)
Expand Down
19 changes: 14 additions & 5 deletions test/jltest/cli_tests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,9 @@ end
@testset EnhancedTestSet "jltest.cli.run(): basic tests" begin
# --- Preparations

# Get current directory
cwd = pwd()

# Construct path to test directory
test_dir = joinpath(@__DIR__, "data-basic-tests")
test_dir_relpath = relpath(test_dir)
Expand Down Expand Up @@ -246,6 +249,7 @@ end

# Case: `tests` is empty
cd(test_dir)

tests = []
local tests_passed = true
local error = nothing
Expand Down Expand Up @@ -291,6 +295,11 @@ end

expected_output_more_tests = "$(joinpath("subdir", "more_tests")): .."
@test occursin(expected_output_more_tests, output)

# --- Clean up

# Restore current directory
cd(cwd)
end

@testset EnhancedTestSet "jltest.cli.run(): keyword argument tests" begin
Expand All @@ -302,15 +311,15 @@ end

# Precompute commonly used values
some_tests_file = joinpath(test_dir, "some_tests.jl")
expected_output_some_tests = "some_tests: .."
expected_output_some_tests = "$(joinpath(test_dir_relpath, "some_tests")): .."

some_tests_no_testset_file = joinpath(test_dir, "some_tests_no_testset.jl")
expected_output_some_tests_no_testset = "some_tests_no_testset: .."
expected_output_some_tests_no_testset = "$(joinpath(test_dir_relpath, "some_tests_no_testset")): .."

failing_tests_file = joinpath(test_dir, "failing_tests.jl")
expected_output_failing_tests = Regex(
make_windows_safe_regex(strip("""
failing_tests: .
$(joinpath(test_dir_relpath, "failing_tests")): .
=====================================================
failing tests: Test Failed at $(failing_tests_file):[0-9]+
Expression: 2 == 1
Expand All @@ -320,7 +329,7 @@ end

expected_output_failing_tests_fail_fast_prefix = Regex(
make_windows_safe_regex(strip("""
failing_tests: .
$(joinpath(test_dir_relpath, "failing_tests")): .
=====================================================
Test Failed at $(failing_tests_file):[0-9]+
Expression: 2 == 1
Expand All @@ -338,7 +347,7 @@ end
failing_tests_no_testset_file = joinpath(test_dir, "failing_tests_no_testset.jl")
expected_output_failing_tests_no_testset = Regex(
make_windows_safe_regex(strip("""
failing_tests_no_testset: .
$(joinpath(test_dir_relpath, "failing_tests_no_testset")): .
=====================================================
All tests: Test Failed at $(failing_tests_no_testset_file):[0-9]+
Expression: 2 == 1
Expand Down
2 changes: 1 addition & 1 deletion test/jltest/utils_tests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -620,7 +620,7 @@ end
cp(joinpath(@__DIR__, "data-missing-package-dependency", "TestPackage"), test_pkg_dir)

# Create system command to run test
cmd_options = `--startup-file=no --project=@. -O0`
cmd_options = `--startup-file=no --project=. -O0`
cmd = Cmd(
`julia $(cmd_options) -e 'import Pkg; Pkg.test(coverage=true)'`; dir=test_pkg_dir
)
Expand Down
44 changes: 44 additions & 0 deletions test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,20 @@ cwd = pwd()
# --- Normal unit tests

# installer tests
println()
println("=============================== pkg tests start ===============================")

tests = [joinpath(@__DIR__, "pkg_tests.jl")]
cd(cwd)
jltest.run_tests(tests; desc="installer")

println()
println("================================ pkg tests end ================================")
println()

# `jltest` tests
println("============================== jltest tests start =============================")

tests = [
joinpath(@__DIR__, "jltest", "isolated_test_module_tests.jl"),
joinpath(@__DIR__, "jltest", "EnhancedTestSet_utils_tests.jl"),
Expand All @@ -71,21 +80,40 @@ tests = [joinpath(@__DIR__, "jltest", "verbose_mode_tests.jl")]
cd(cwd)
jltest.run_tests(tests; desc="jltest: verbose mode tests", test_set_type=nothing)

println()
println()
println("=============================== jltest tests end ==============================")
println()

# `jlcodestyle` tests
println("=========================== jlcodestyle tests start ===========================")

tests = [joinpath(@__DIR__, "jlcodestyle", "cli_tests.jl")]
cd(cwd)
jltest.run_tests(tests; desc="jlcodestyle")

println()
println("============================ jlcodestyle tests end ============================")
println()

# `jlcoverage` tests
println("============================ jlcoverage tests start ===========================")

tests = [
joinpath(@__DIR__, "jlcoverage", "cli_tests.jl"),
joinpath(@__DIR__, "jlcoverage", "utils_tests.jl"),
]
cd(cwd)
jltest.run_tests(tests; desc="jlcoverage")

println()
println("============================= jlcoverage tests end =============================")
println()

# --- jltest unit tests that have expected failures and errors

println("========================= EnhancedTestSet tests start =========================")

# EnhancedTestSet with failing tests
println()
test_file = joinpath(@__DIR__, "jltest", "EnhancedTestSet_failing_tests.jl")
Expand Down Expand Up @@ -247,8 +275,14 @@ print("jltest/EnhancedTestSet_nested_test_set_tests: ")
end
end

println()
println("========================== EnhancedTestSet tests end ==========================")
println()

# utils.jl
println("=========================== jltest.utils tests start ==========================")
println()

test_file = joinpath(@__DIR__, "jltest", "utils_tests.jl")

local log_message
Expand Down Expand Up @@ -387,8 +421,14 @@ print("jltest/utils_tests: ")
end
end

println()
println("============================ jltest.utils tests end ===========================")
println()

# cli.jl
println("============================ jltest.cli tests start ===========================")
println()

test_file = joinpath(@__DIR__, "jltest", "cli_tests.jl")

local log_message
Expand Down Expand Up @@ -478,3 +518,7 @@ print("jltest/cli_tests: ")
@test occursin(expected_output, output)
end
end

println()
println("============================= jltest.cli tests end ============================")
println()

2 comments on commit bd12e3b

@ktchu
Copy link
Member Author

@ktchu ktchu commented on bd12e3b Jun 27, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/109887

Tip: Release Notes

Did you know you can add release notes too? Just add markdown formatted text underneath the comment after the text
"Release notes:" and it will be added to the registry PR, and if TagBot is installed it will also be added to the
release that TagBot creates. i.e.

@JuliaRegistrator register

Release notes:

## Breaking changes

- blah

To add them here just re-invoke and the PR will be updated.

Tagging

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.6.2 -m "<description of version>" bd12e3b150761a339cdfd2e9a9a21b05d006f042
git push origin v0.6.2

Please sign in to comment.