Skip to content

Commit

Permalink
improving reporting for failed image comparisons; reenabling map test…
Browse files Browse the repository at this point in the history 
…s for everything but macos; pinning selenium version
  • Loading branch information
@mjgleason
mjgleason committed Jul 25, 2023
1 parent 9173392 commit b00cbf6
Show file tree
Hide file tree
Showing 6 changed files with 117 additions and 41 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/codecov.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
python -m pip install .[dev]
- name: Generate Coverage Report
run: |
python -m pytest --disable-warnings --cov=./ --cov-report=xml:coverage.xml -m "not maptest"
python -m pytest --disable-warnings --cov=./ --cov-report=xml:coverage.xml
- name: Upload Coverage to Codecov
uses: codecov/codecov-action@v1
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pull_request_tests_unix.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,4 @@ jobs:
python -m pip install .[dev]
- name: Run Pytest
run: |
python -m pytest -v --disable-warnings -m "not maptest"
python -m pytest -v --disable-warnings
2 changes: 1 addition & 1 deletion .github/workflows/pull_request_tests_windows.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,4 @@ jobs:
- name: Run Pytest
shell: powershell
run: |
python -m pytest -v --disable-warnings -m "not maptest"
python -m pytest -v --disable-warnings
4 changes: 3 additions & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
"A data portal for reviewing Renewable Energy Potential Model "
"(reV) outputs"
)
DEV_REQUIREMENTS = ["pylint", "dash[testing]==2.3.0", "selenium", "imagehash"]
DEV_REQUIREMENTS = [
"pylint", "dash[testing]==2.3.0", "selenium==4.8.0", "imagehash"
]
TEST_REQUIREMENTS = ["pytest", "pytest-cov"]
GUNICORN_REQUIREMENTS = ["gunicorn"]

Expand Down
100 changes: 70 additions & 30 deletions tests/test_cli/test_cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,9 +270,13 @@ def test_make_maps(
for out_png_name in out_png_names:
expected_png = data_dir_test.joinpath("plots", out_png_name)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -310,9 +314,13 @@ def test_make_maps_wind_keep_zero(
"plots", out_png_name.replace(".png", "_kz.png")
)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -349,9 +357,13 @@ def test_make_maps_wind_drop_zero(
"plots", out_png_name.replace(".png", "_dz.png")
)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -388,9 +400,13 @@ def test_make_maps_wind_drop_legend(
"plots", out_png_name.replace(".png", "_drop_legend.png")
)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -427,9 +443,13 @@ def test_make_maps_boundaries(
"plots", out_png_name.replace(".png", "_boundaries.png")
)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -499,9 +519,13 @@ def test_map_column_happy(
"plots", out_png_name.replace(".png", "_happy.png")
)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -537,9 +561,13 @@ def test_map_column_formatting(
"plots", out_png_name.replace(".png", "_formatting.png")
)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -631,9 +659,13 @@ def test_map_column_boundaries(
"plots", out_png_name.replace(".png", "_boundaries.png")
)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -666,9 +698,13 @@ def test_map_column_boundaries_kwargs(
"plots", out_png_name.replace(".png", "_boundaries_kwargs.png")
)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -701,9 +737,13 @@ def test_map_column_drop_legend(
"plots", out_png_name.replace(".png", "_drop_legend.png")
)
out_png = output_path.joinpath(out_png_name)
assert compare_images_approx(expected_png, out_png), \
"Output image does not match expected image " \
f"{expected_png}"
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down
48 changes: 41 additions & 7 deletions tests/test_utils/test_plots.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,11 @@ def compare_images_approx(

max_diff_bits = int(np.ceil(hash_size * max_diff_pct))

return (expected_hash - out_hash) <= max_diff_bits
diff = expected_hash - out_hash
matches = diff <= max_diff_bits
pct_diff = float(diff) / hash_size

return matches, pct_diff


def test_YBFixedBounds_happy():
Expand Down Expand Up @@ -123,8 +127,13 @@ def test_map_geodataframe_column_happy(

expected_png = data_dir_test.joinpath("plots", out_png_name)

assert compare_images_approx(expected_png, out_png), \
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -172,8 +181,13 @@ def test_map_geodataframe_column_styling(

expected_png = data_dir_test.joinpath("plots", out_png_name)

assert compare_images_approx(expected_png, out_png), \
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -215,8 +229,13 @@ def test_map_geodataframe_column_repeat(

expected_png = data_dir_test.joinpath("plots", out_png_name)

assert compare_images_approx(expected_png, out_png), \
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -248,8 +267,13 @@ def test_map_geodataframe_column_no_legend(

expected_png = data_dir_test.joinpath("plots", out_png_name)

assert compare_images_approx(expected_png, out_png), \
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -283,8 +307,13 @@ def test_map_geodataframe_column_boundaries_kwargs(

expected_png = data_dir_test.joinpath("plots", out_png_name)

assert compare_images_approx(expected_png, out_png), \
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


@pytest.mark.maptest
Expand Down Expand Up @@ -345,8 +374,13 @@ def test_map_geodataframe_polygons(

expected_png = data_dir_test.joinpath("plots", out_png_name)

assert compare_images_approx(expected_png, out_png), \
images_match, pct_diff = compare_images_approx(
expected_png, out_png
)
assert images_match, (
f"Output image does not match expected image {expected_png}"
f"Difference is {pct_diff * 100}%"
)


if __name__ == '__main__':
Expand Down

0 comments on commit b00cbf6

Please sign in to comment.