Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added equality helper and assert message for allClose #18

Merged
merged 1 commit into from
Aug 18, 2016
Merged

Added equality helper and assert message for allClose #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
63 changes: 47 additions & 16 deletions python/lsst/utils/tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -537,8 +537,10 @@ def plotImageDiff(lhs, rhs, bad=None, diff=None, plotFileName=None):


@inTestCase
def assertClose(testCase, lhs, rhs, rtol=sys.float_info.epsilon, atol=sys.float_info.epsilon, relTo=None,
printFailures=True, plotOnFailure=False, plotFileName=None, invert=False):
def assertFloatsAlmostEqual(testCase, lhs, rhs, rtol=sys.float_info.epsilon,
atol=sys.float_info.epsilon, relTo=None,
printFailures=True, plotOnFailure=False,
plotFileName=None, invert=False, msg=None):
"""!Highly-configurable floating point comparisons for scalars and arrays.

The test assertion will fail if all elements lhs and rhs are not equal to within the tolerances
Expand All @@ -553,8 +555,8 @@ def assertClose(testCase, lhs, rhs, rtol=sys.float_info.epsilon, atol=sys.float_
is expected.

@param[in] testCase unittest.TestCase instance the test is part of
@param[in] lhs LHS value(s) to compare; may be a scalar or a numpy array of any dimension
@param[in] rhs RHS value(s) to compare; may be a scalar or a numpy array of any dimension
@param[in] lhs LHS value(s) to compare; may be a scalar or array-like of any dimension
@param[in] rhs RHS value(s) to compare; may be a scalar or array-like of any dimension
@param[in] rtol Relative tolerance for comparison; defaults to double-precision epsilon.
@param[in] atol Absolute tolerance for comparison; defaults to double-precision epsilon.
@param[in] relTo Value to which comparison with rtol is relative.
Expand All @@ -564,8 +566,9 @@ def assertClose(testCase, lhs, rhs, rtol=sys.float_info.epsilon, atol=sys.float_
@param[in] plotFileName Filename to save the plot to. If None, the plot will be displayed in a
a window.
@param[in] invert If True, invert the comparison and fail only if any elements *are* equal.
Used to implement assertNotClose, which should generally be used instead
Used to implement assertFloatsNotEqual, which should generally be used instead
for clarity.
@param[in] msg String to append to the error message when assert fails.
"""
if not numpy.isfinite(lhs).all():
testCase.fail("Non-finite values in lhs")
Expand Down Expand Up @@ -594,21 +597,21 @@ def assertClose(testCase, lhs, rhs, rtol=sys.float_info.epsilon, atol=sys.float_
else:
cmpStr = "!="
failStr = "differ"
msg = []
errMsg = []
if failed:
if numpy.isscalar(bad):
msg = ["%s %s %s; diff=%s/%s=%s with rtol=%s, atol=%s"
% (lhs, cmpStr, rhs, absDiff, relTo, absDiff/relTo, rtol, atol)]
errMsg = ["%s %s %s; diff=%s/%s=%s with rtol=%s, atol=%s"
% (lhs, cmpStr, rhs, absDiff, relTo, absDiff/relTo, rtol, atol)]
else:
msg = ["%d/%d elements %s with rtol=%s, atol=%s"
% (bad.sum(), bad.size, failStr, rtol, atol)]
errMsg = ["%d/%d elements %s with rtol=%s, atol=%s"
% (bad.sum(), bad.size, failStr, rtol, atol)]
if plotOnFailure:
if len(lhs.shape) != 2 or len(rhs.shape) != 2:
raise ValueError("plotOnFailure is only valid for 2-d arrays")
try:
plotImageDiff(lhs, rhs, bad, diff=diff, plotFileName=plotFileName)
except ImportError:
msg.append("Failure plot requested but matplotlib could not be imported.")
errMsg.append("Failure plot requested but matplotlib could not be imported.")
if printFailures:
# Make sure everything is an array if any of them are, so we can treat
# them the same (diff and absDiff are arrays if either rhs or lhs is),
Expand All @@ -620,14 +623,42 @@ def assertClose(testCase, lhs, rhs, rtol=sys.float_info.epsilon, atol=sys.float_
if numpy.isscalar(rhs):
rhs = numpy.ones(bad.shape, dtype=float) * rhs
for a, b, diff, rel in zip(lhs[bad], rhs[bad], absDiff[bad], relTo[bad]):
msg.append("%s %s %s (diff=%s/%s=%s)" % (a, cmpStr, b, diff, rel, diff/rel))
testCase.assertFalse(failed, msg="\n".join(msg))
errMsg.append("%s %s %s (diff=%s/%s=%s)" % (a, cmpStr, b, diff, rel, diff/rel))

if msg is not None:
errMsg.append(msg)
testCase.assertFalse(failed, msg="\n".join(errMsg))


@inTestCase
def assertNotClose(testCase, lhs, rhs, **kwds):
"""!Fail a test if the given floating point values are completely equal to within the given tolerances.
def assertFloatsNotEqual(testCase, lhs, rhs, **kwds):
"""
Fail a test if the given floating point values are equal to within the given tolerances.

See assertClose for more information.
"""
return assertClose(testCase, lhs, rhs, invert=True, **kwds)
return assertFloatsAlmostEqual(testCase, lhs, rhs, invert=True, **kwds)


@inTestCase
def assertFloatsEqual(testCase, lhs, rhs, **kwargs):
"""
Assert that lhs == rhs (both numeric types, whether scalar or array).

See assertClose (called with rtol=atol=0) for more information.
"""
return assertFloatsAlmostEqual(testCase, lhs, rhs, rtol=0, atol=0, **kwargs)


@inTestCase
def assertClose(*args, **kwargs):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sweet

warnings.warn("assertClose is deprecated; please use TestCase.assertFloatsAlmostEqual",
DeprecationWarning)
return assertFloatsAlmostEqual(*args, **kwargs)


@inTestCase
def assertNotClose(*args, **kwargs):
warnings.warn("assertNotClose is deprecated; please use TestCase.assertFloatsNotEqual",
DeprecationWarning)
return assertFloatsNotEqual(*args, **kwargs)
202 changes: 142 additions & 60 deletions tests/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,79 +21,161 @@
# the GNU General Public License along with this program. If not,
# see <http://www.lsstcorp.org/LegalNotices/>.
#

"""
Tests for utilities

Run with:
python utils.py
or
python
>>> import utils; utils.run()
"""

import unittest
import numpy
import numpy as np

import lsst.utils.tests


# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# set to True to test plotting and printing by-eye when tests fail
display = False


class UtilsTestCase(lsst.utils.tests.TestCase):
"""A test case for Utils"""

def testCompareArrays(self):
self.assertClose(0.0, 0.0)
self.assertClose(0.0, 1E-8, atol=1E-7)
self.assertClose(0.0, 1E-8, atol=1E-7, rtol=None)
self.assertClose(0.0, 1E-8, atol=None, rtol=1E-5, relTo=1E-2)
self.assertNotClose(0.0, 1E-8, atol=1E-9)
self.assertNotClose(0.0, 1E-8, atol=1E-9, rtol=None)
self.assertNotClose(0.0, 1E-8, atol=None, rtol=1E-7, relTo=1E-2)
self.assertClose(100, 100 + 1E-8, rtol=1E-7)
self.assertClose(100, 100 + 1E-8, rtol=1E-7, atol=None)
self.assertClose(100, 100 + 1E-8, rtol=1E-7, relTo=100.0)
self.assertNotClose(100, 100 + 1E-8, rtol=1E-12)
self.assertNotClose(100, 100 + 1E-8, rtol=1E-12, atol=None)
self.assertNotClose(100, 100 + 1E-8, rtol=1E-12, relTo=100.0)
a = numpy.zeros((5, 5), dtype=float)
b = numpy.zeros((5, 5), dtype=float)
self.assertClose(a, b)
self.assertClose(a, b, rtol=None)
self.assertClose(a, b, atol=None)
self.assertRaises(ValueError, self.assertClose, a, b, atol=None, rtol=None)
b[:, :] = 1E-8
self.assertClose(a, b, atol=1E-7)
self.assertClose(a, 1E-8, atol=1E-7)
self.assertClose(a, b, atol=1E-7, rtol=None)
self.assertNotClose(a, b, atol=1E-9)
self.assertNotClose(a, b, atol=1E-9, rtol=None)
a[:, :] = 100.0
b[:, :] = 100.0 + 1E-8
self.assertClose(a, b, rtol=1E-7)
self.assertClose(a, 100.0 + 1E-8, rtol=1E-7)
self.assertClose(a, b, rtol=1E-7, atol=None)
self.assertClose(a, b, rtol=1E-7, relTo=100.0)
self.assertNotClose(a, b, rtol=1E-12)
self.assertNotClose(a, b, rtol=1E-12, atol=None)
self.assertNotClose(a, b, rtol=1E-12, relTo=100.0)
a[:, :] = numpy.arange(-12, 13).reshape(5, 5)
b[:, :] = a
b[2, :] += numpy.linspace(-1E-4, 1E-4, 5)
self.assertClose(a, b, rtol=1E-3, atol=1E-4)
if False:
# set to True to test plotting and printing by-eye when tests fail
def setUp(self):
self.large = 100.
self.epsilon = 1e-8
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not use sys.float_info.epsilon here? No big deal, just curious.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This epsilon is used in some tests, while others just take the default value, to test both the defaults and passing a kwarg.

self.zeros = np.zeros((5, 5), dtype=float)
self.zeros2 = np.zeros((5, 5), dtype=float)
self.epsilons = np.zeros((5, 5), dtype=float) + self.epsilon
self.larges = np.zeros((5, 5), dtype=float) + self.large
self.larges2 = np.zeros((5, 5), dtype=float) + self.large
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why pre-compute the copies? I suspect it would be slightly clearer to set larges2=self.larges2.copy() when needed. But no big deal.

Copy link
Contributor

@parejkoj parejkoj Aug 18, 2016 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't want to test x==x, but rather x==y where y was created in the same way as x. Maybe the distinction isn't important. But this is the place to do it, so I can always just use the thing later on.

Also, the places I use .copy() here are so that I can slightly tweak the values for later testing.

self.largesOneOff = self.larges.copy()
self.largesOneOff[0] += self.epsilon
self.largesEpsilon = np.zeros((5, 5), dtype=float) + self.large + self.epsilon
self.ranges = np.arange(-12., 13.).reshape(5, 5)
self.rangesEpsilon = self.ranges.copy()
self.rangesEpsilon += np.linspace(-1E-4, 1E-4, 5.)

def test_assertFloatsAlmostEqual(self):
# zero scalar tests
self.assertFloatsAlmostEqual(0.0, 0.0)
self.assertFloatsAlmostEqual(0.0, 1E-8, atol=1E-7)
self.assertFloatsAlmostEqual(0.0, 1E-8, atol=1E-7, rtol=None)
self.assertFloatsAlmostEqual(0.0, 1E-8, atol=None, rtol=1E-5, relTo=1E-2)

# zero array vs. scalar tests
self.assertFloatsAlmostEqual(self.zeros, 0.)
self.assertFloatsAlmostEqual(self.zeros, self.epsilon, atol=1E-7)
self.assertFloatsAlmostEqual(self.zeros, self.epsilon, atol=1E-7, rtol=None)
self.assertFloatsAlmostEqual(self.zeros, self.epsilon, atol=None, rtol=1E-5, relTo=1E-2)

# zero array vs. array tests
self.assertFloatsAlmostEqual(self.zeros, self.zeros2)
self.assertFloatsAlmostEqual(self.zeros, self.zeros2, rtol=None)
self.assertFloatsAlmostEqual(self.zeros, self.zeros2, atol=None)
self.assertFloatsAlmostEqual(self.zeros, self.epsilons, atol=1E-7)
self.assertFloatsAlmostEqual(self.zeros, self.epsilons, atol=1E-7, rtol=None)
self.assertFloatsAlmostEqual(self.zeros, self.epsilons, atol=None, rtol=1E-5, relTo=1E-2)

# invalid value tests
with self.assertRaises(ValueError):
self.assertFloatsAlmostEqual(self.zeros, self.zeros2, atol=None, rtol=None)

# non-zero scalar tests
self.assertFloatsAlmostEqual(self.large, 100.)
self.assertFloatsAlmostEqual(self.large, self.large + self.epsilon, rtol=1E-7)
self.assertFloatsAlmostEqual(self.large, self.large + self.epsilon, rtol=1E-7, atol=None)
self.assertFloatsAlmostEqual(self.large, self.large + self.epsilon, rtol=1E-7, relTo=100.0)

# Non-zero array vs. scalar tests
self.assertFloatsAlmostEqual(self.larges, self.large)
self.assertFloatsAlmostEqual(self.larges, self.large + self.epsilon, rtol=1E-7)
self.assertFloatsAlmostEqual(self.larges, self.large + self.epsilon, rtol=1E-7, atol=None)
self.assertFloatsAlmostEqual(self.larges, self.large + self.epsilon, rtol=1E-7, relTo=100.0)

# Non-zero array vs. array tests
self.assertFloatsAlmostEqual(self.larges, self.larges2)
self.assertFloatsAlmostEqual(self.larges, self.largesEpsilon, rtol=1E-7)
self.assertFloatsAlmostEqual(self.larges, self.largesEpsilon, rtol=1E-7, atol=None)
self.assertFloatsAlmostEqual(self.larges, self.largesEpsilon, rtol=1E-7, relTo=100.0)
self.assertFloatsAlmostEqual(self.larges, self.largesOneOff, atol=1e-7)
self.assertFloatsAlmostEqual(self.ranges, self.rangesEpsilon, rtol=1E-3, atol=1E-4)

if display:
# should see failures on the center row of the 5x5 image, but not the very center point
self.assertClose(a, b, rtol=1E-6, plotOnFailure=True)
nonzeroCenter = self.zeros.copy()
nonzeroCenter[2, :] = 1e-5
with self.assertRaises(AssertionError):
self.assertFloatsAlmostEqual(self.zeros, nonzeroCenter, rtol=1E-6, plotOnFailure=True)

with self.assertRaises(AssertionError) as cm:
self.assertFloatsAlmostEqual(10, 0, msg="This is an error message.")
self.assertIn("This is an error message.", str(cm.exception))
self.assertIn("10 != 0; diff=10/10=1.0 with rtol=", str(cm.exception))

def test_assertFloatsNotEqual(self):
# zero scalar tests
self.assertFloatsNotEqual(0.0, 1.0)
self.assertFloatsNotEqual(0.0, 1E-8, atol=1E-9)
self.assertFloatsNotEqual(0.0, 1E-8, atol=1E-9, rtol=None)
self.assertFloatsNotEqual(0.0, 1E-8, atol=None, rtol=1E-7, relTo=1E-2)

# zero array vs. scalar tests
self.assertFloatsNotEqual(self.zeros, 1.)
self.assertFloatsNotEqual(self.zeros, self.epsilon, atol=1E-9)
self.assertFloatsNotEqual(self.zeros, self.epsilon, atol=1E-9, rtol=None)
self.assertFloatsNotEqual(self.zeros, self.epsilon, atol=None, rtol=1E-7, relTo=1E-2)

# zero array vs. array tests
self.assertFloatsNotEqual(self.zeros, self.larges)
self.assertFloatsNotEqual(self.zeros, self.epsilon, atol=1E-9, rtol=None)
self.assertFloatsNotEqual(self.zeros, self.epsilon, atol=None, rtol=1e-5, relTo=1e-5)
self.assertFloatsNotEqual(self.zeros, self.epsilons, atol=1E-9)
self.assertFloatsNotEqual(self.zeros, self.epsilons, atol=1E-9, rtol=None)
self.assertFloatsNotEqual(self.zeros, self.epsilons, atol=None, rtol=1E-7, relTo=1E-2)

# invalid value tests
with self.assertRaises(ValueError):
self.assertFloatsNotEqual(self.zeros, self.zeros2, atol=None, rtol=None)

# non-zero scalar tests
self.assertFloatsNotEqual(self.large, 1.)
self.assertFloatsNotEqual(self.large, self.large + self.epsilon, atol=1E-9)
self.assertFloatsNotEqual(self.large, self.large + self.epsilon, rtol=1E-11, atol=None)
self.assertFloatsNotEqual(self.large, self.large + self.epsilon, rtol=1E-12, relTo=1.)

# Non-zero array vs. scalar tests
self.assertFloatsNotEqual(self.larges, self.large + self.epsilon, atol=1e-9)
self.assertFloatsNotEqual(self.larges, self.large + self.epsilon, atol=1e-9, rtol=None)
self.assertFloatsNotEqual(self.larges, self.large + self.epsilon, rtol=1E-11, atol=None)
self.assertFloatsNotEqual(self.larges, self.large + self.epsilon, rtol=1E-12, relTo=1.)

# Non-zero array vs. array tests
self.assertFloatsNotEqual(self.larges, self.zeros)
self.assertFloatsNotEqual(self.larges, self.largesEpsilon, rtol=1E-12)
self.assertFloatsNotEqual(self.larges, self.largesEpsilon, rtol=1E-12, atol=None)
self.assertFloatsNotEqual(self.larges, self.largesEpsilon, rtol=1E-11, relTo=100.0)
self.assertFloatsNotEqual(self.larges, self.largesOneOff, atol=1e-9)
self.assertFloatsNotEqual(self.larges, self.largesOneOff, atol=1e-9, rtol=None)
self.assertFloatsNotEqual(self.ranges, self.rangesEpsilon)

with self.assertRaises(AssertionError) as cm:
self.assertFloatsNotEqual(10, 10, msg="This is an error message.")
self.assertIn("This is an error message.", str(cm.exception))
self.assertIn("10 == 10; diff=0/10=0.0 with rtol=", str(cm.exception))

def test_assertFloatsEqual(self):
self.assertFloatsEqual(0, 0)
self.assertFloatsEqual(0., 0.)
self.assertFloatsEqual(1, 1)
self.assertFloatsEqual(1., 1.)
self.assertFloatsEqual(self.zeros, self.zeros2)
self.assertFloatsEqual(self.zeros, 0)
self.assertFloatsEqual(self.zeros, 0.)
self.assertFloatsEqual(self.larges, self.large)
with self.assertRaises(AssertionError):
self.assertFloatsEqual(self.larges, 0.)
with self.assertRaises(AssertionError):
self.assertFloatsEqual(self.larges, self.largesEpsilon)
with self.assertRaises(AssertionError):
self.assertFloatsEqual(self.larges, self.zeros)
with self.assertRaises(AssertionError):
self.assertFloatsEqual(self.larges, self.largesEpsilon)


class TestMemory(lsst.utils.tests.MemoryTestCase):
pass

# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-


def setup_module(module):
lsst.utils.tests.init()
Expand Down