branch: generate-test-harness

[samreid]

To support phetsims/aqua#24, automatic generation of QUnit test harnesses.

[samreid]

I added a new grunt task which can be used like so:

cd faradays-law
grunt generate-test-harness

Add js/faradays-law-test.js:

// Copyright 2014-2017, University of Colorado Boulder

/**
 * Test entry point for the 'Faradays Law'
 *
 * @author Sam Reid (PhET Interactive Simulations)
 */
define( function( require ) {
  'use strict';

  // modules
  var FaradaysLawConstants = require( 'FARADAYS_LAW/faradays-law/FaradaysLawConstants' );
  var FaradaysLawModel = require( 'FARADAYS_LAW/faradays-law/model/FaradaysLawModel' );
  var Tandem = require( 'TANDEM/Tandem' );

  QUnit.test( 'Test Faradays Law Model', function( assert ) {
    var bounds = FaradaysLawConstants.LAYOUT_BOUNDS;
    var tandem = Tandem.rootTandem.createTandem( 'model' );
    var model = new FaradaysLawModel( bounds, tandem );
    assert.ok( model.magnet, 'model should have a magnet' );

    // Add any code here that uses any of the modules declared in this sim or its dependencies
    // ...
  } );
} );

Run faradays-law-test.html

Tests can be defined and loaded through the same module system that we use to load files. Tests will not be included in the built version.

It is naive to believe this strategy will work for all of our repos (particularly non-sims), but I tried it on axon and it seemed to work properly.

The next step will be to discuss this with @jonathanolson to see if he has recommendations for improvements, and to discuss how it could be integrated with bayes automatic testing. At a minimum we will need to include code like QUnit.log and QUnit.done, but I'd like to better understand how I can test that before working on it.

[samreid]

I think it would not be too much work to create a compiled test HTML, we would need to point the requirejs step at the test config file instead of the "main" config file and update the sim template. Let's get feedback from @jonathanolson before going much further though.

[jonathanolson]

I haven't reviewed the implementation, but this looks great for sim tests. Hopefully the newer QUnit version supports properly identifying when tests are finished running (otherwise automated testing will need to wait for 40s).

It is naive to believe this strategy will work for all of our repos (particularly non-sims), but I tried it on axon and it seemed to work properly.

I'm curious if we can get it to work for all repos, so it is consistent?

[samreid]

I got this working for dot. The main recipe is:

1. cd dot
2. grunt generate-test-harness
3. create js/dot-test.js
4. Port tests like so:

a. convert e.g., dot.Bounds2 to Bounds2 and add a module load statement for Bounds2.
b. replace test => QUnit.test
c. add assert argument to QUnit tests

Here's a snapshot of dot-test.js:

// Copyright 2017, University of Colorado Boulder

/**
 * Test entry point.
 *
 * @author Sam Reid (PhET Interactive Simulations)
 */
define( function( require ) {
  'use strict';

  // modules
  require( 'DOT/Bounds2Test' );
} );

Here's a snapshot of Bounds2Test, ported from dot/tests/qunit/js/bounds2.js

// Copyright 2014-2017, University of Colorado Boulder

/**
 * @author Sam Reid (PhET Interactive Simulations)
 */
define( function( require ) {
  'use strict';

  // modules
  var Bounds2 = require( 'DOT/Bounds2' );
  var Matrix3 = require( 'DOT/Matrix3' );
  var Rectangle = require( 'DOT/Rectangle' );
  var Vector2 = require( 'DOT/Vector2' );

  var epsilon = 0.00000001;

  function approximateBoundsEquals( assert, a, b, msg ) {
    assert.ok( Math.abs( a.minX - b.minX ) < epsilon, msg + ' minX: expected: ' + b.minX + ', result: ' + a.minX );
    assert.ok( Math.abs( a.minY - b.minY ) < epsilon, msg + ' minY: expected: ' + b.minY + ', result: ' + a.minY );
    assert.ok( Math.abs( a.maxX - b.maxX ) < epsilon, msg + ' maxX: expected: ' + b.maxX + ', result: ' + a.maxX );
    assert.ok( Math.abs( a.maxY - b.maxY ) < epsilon, msg + ' maxY: expected: ' + b.maxY + ', result: ' + a.maxY );
  }

  QUnit.test( 'Rectangle', function( assert ) {
    assert.ok( new Bounds2( -2, -4, 2, 4 ).equals( new Rectangle( -2, -4, 4, 8 ) ), 'Bounds2-Rectangle equivalence' );
  } );

  QUnit.test( 'Basic', function( assert ) {
    var bounds = new Bounds2( 1, 2, 3, 4 );
    assert.ok( bounds.minX === 1, 'minX' );
    assert.ok( bounds.minY === 2, 'minY' );
    assert.ok( bounds.maxX === 3, 'maxX' );
    assert.ok( bounds.maxY === 4, 'maxY' );
    assert.ok( bounds.width === 2, 'width' );
    assert.ok( bounds.height === 2, 'height' );
    assert.ok( bounds.x === 1, 'x' );
    assert.ok( bounds.y === 2, 'y' );
    assert.ok( bounds.centerX === 2, 'centerX' );
    assert.ok( bounds.centerY === 3, 'centerY' );
  } );

  QUnit.test( 'Coordinates', function( assert ) {
    var bounds = new Bounds2( 1, 2, 3, 4 );
    assert.ok( !bounds.isEmpty(), 'isEmpty' );

    assert.ok( !bounds.containsCoordinates( 0, 0 ), 'coordinates #1' );
    assert.ok( !bounds.containsCoordinates( 2, 0 ), 'coordinates #2' );
    assert.ok( bounds.containsCoordinates( 2, 2 ), 'coordinates #3 (on boundary)' );
    assert.ok( !bounds.containsCoordinates( 4, 2 ), 'coordinates #4' );

    assert.ok( !Bounds2.NOTHING.containsBounds( bounds ), 'nothing.contains' );
    assert.ok( Bounds2.EVERYTHING.containsBounds( bounds ), 'everything.contains' );

    assert.ok( bounds.equals( bounds ), 'reflexive' );
    assert.ok( !bounds.equals( Bounds2.NOTHING ), 'reflexive' );
    assert.ok( !Bounds2.NOTHING.equals( bounds ), 'reflexive' );

    assert.ok( bounds.intersectsBounds( new Bounds2( 2, 3, 4, 5 ) ), 'intersect #1' );
    assert.ok( bounds.intersectsBounds( new Bounds2( 3, 4, 5, 6 ) ), 'intersect #2 (boundary point)' );
    assert.ok( !bounds.intersectsBounds( new Bounds2( 4, 5, 6, 7 ) ), 'intersect #3' );

    assert.ok( Bounds2.NOTHING.isEmpty(), 'Bounds2.NOTHING.isEmpty()' );
    assert.ok( !Bounds2.EVERYTHING.isEmpty(), '!Bounds2.EVERYTHING.isEmpty()' );
  } );

  function A() { return new Bounds2( 0, 0, 2, 3 ); }

  function B() { return new Bounds2( 1, 1, 5, 4 ); }

  function C() { return new Bounds2( 1.5, 1.2, 5.7, 4.8 ); }

  QUnit.test( 'Mutable / immutable versions', function( assert ) {
    approximateBoundsEquals( assert, A().union( B() ), A().includeBounds( B() ), 'union / includeBounds' );
    approximateBoundsEquals( assert, A().intersection( B() ), A().constrainBounds( B() ), 'intersection / constrainBounds' );
    approximateBoundsEquals( assert, A().withCoordinates( 10, 12 ), A().addCoordinates( 10, 12 ), 'withCoordinates / addCoordinates' );
    approximateBoundsEquals( assert, A().withPoint( new Vector2( 10, 12 ) ), A().addPoint( new Vector2( 10, 12 ) ), 'withPoint / addPoint' );

    approximateBoundsEquals( assert, A().withMinX( 1.5 ), A().setMinX( 1.5 ), 'withMinX / setMinX' );
    approximateBoundsEquals( assert, A().withMinY( 1.5 ), A().setMinY( 1.5 ), 'withMinY / setMinY' );
    approximateBoundsEquals( assert, A().withMaxX( 1.5 ), A().setMaxX( 1.5 ), 'withMaxX / setMaxX' );
    approximateBoundsEquals( assert, A().withMaxY( 1.5 ), A().setMaxY( 1.5 ), 'withMaxY / setMaxY' );

    approximateBoundsEquals( assert, C().roundedOut(), C().roundOut(), 'roundedOut / roundOut' );
    approximateBoundsEquals( assert, C().roundedIn(), C().roundIn(), 'roundedIn / roundIn' );

    var matrix = Matrix3.rotation2( Math.PI / 4 ).timesMatrix( Matrix3.translation( 11, -13 ) ).timesMatrix( Matrix3.scale( 2, 3.5 ) );
    approximateBoundsEquals( assert, A().transformed( matrix ), A().transform( matrix ), 'transformed / transform' );

    approximateBoundsEquals( assert, A().dilated( 1.5 ), A().dilate( 1.5 ), 'dilated / dilate' );
    approximateBoundsEquals( assert, A().eroded( 1.5 ), A().erode( 1.5 ), 'eroded / erode' );
    approximateBoundsEquals( assert, A().shiftedX( 1.5 ), A().shiftX( 1.5 ), 'shiftedX / shiftX' );
    approximateBoundsEquals( assert, A().shiftedY( 1.5 ), A().shiftY( 1.5 ), 'shiftedY / shiftY' );
    approximateBoundsEquals( assert, A().shifted( 1.5, 2 ), A().shift( 1.5, 2 ), 'shifted / shift' );
  } );

  QUnit.test( 'Bounds transforms', function( assert ) {
    approximateBoundsEquals( assert, A().transformed( Matrix3.translation( 10, 20 ) ), new Bounds2( 10, 20, 12, 23 ), 'translation' );
    approximateBoundsEquals( assert, A().transformed( Matrix3.rotation2( Math.PI / 2 ) ), new Bounds2( -3, 0, 0, 2 ), 'rotation' );
    approximateBoundsEquals( assert, A().transformed( Matrix3.scale( 3, 2 ) ), new Bounds2( 0, 0, 6, 6 ), 'scale' );
  } );

  QUnit.test( 'Equality', function( assert ) {
    assert.ok( new Bounds2( 0, 1, 2, 3 ).equals( new Bounds2( 0, 1, 2, 3 ) ), 'Without epsilon: regular - reflexive' );
    assert.ok( new Bounds2( 0, 1, 2, 3 ).equalsEpsilon( new Bounds2( 0, 1, 2, 3 ), epsilon ), 'With epsilon: regular - reflexive' );
    assert.ok( !new Bounds2( 0, 1, 2, 3 ).equals( new Bounds2( 0, 1, 2, 5 ) ), 'Without epsilon: regular - different' );
    assert.ok( !new Bounds2( 0, 1, 2, 3 ).equalsEpsilon( new Bounds2( 0, 1, 2, 5 ), epsilon ), 'With epsilon: regular - different' );
    assert.ok( Bounds2.NOTHING.equals( Bounds2.NOTHING ), 'Without epsilon: Nothing - reflexive' );
    assert.ok( Bounds2.NOTHING.equalsEpsilon( Bounds2.NOTHING, epsilon ), 'With epsilon: Nothing - reflexive' );
    assert.ok( Bounds2.NOTHING.equals( Bounds2.NOTHING.copy() ), 'Without epsilon: Nothing - copy - reflexive' );
    assert.ok( Bounds2.NOTHING.equalsEpsilon( Bounds2.NOTHING.copy(), epsilon ), 'With epsilon: Nothing - copy - reflexive' );
    assert.ok( Bounds2.EVERYTHING.equals( Bounds2.EVERYTHING ), 'Without epsilon: Everything - reflexive' );
    assert.ok( Bounds2.EVERYTHING.equalsEpsilon( Bounds2.EVERYTHING, epsilon ), 'With epsilon: Everything - reflexive' );
    assert.ok( Bounds2.EVERYTHING.equals( Bounds2.EVERYTHING.copy() ), 'Without epsilon: Everything - copy - reflexive' );
    assert.ok( Bounds2.EVERYTHING.equalsEpsilon( Bounds2.EVERYTHING.copy(), epsilon ), 'With epsilon: Everything - copy - reflexive' );
    assert.ok( !Bounds2.NOTHING.equals( Bounds2.EVERYTHING ), 'Without epsilon: Nothing !== Everything' );
    assert.ok( !Bounds2.NOTHING.equalsEpsilon( Bounds2.EVERYTHING, epsilon ), 'With epsilon: Nothing !== Everything' );
    assert.ok( new Bounds2( 0, 0, 5, Number.POSITIVE_INFINITY ).equals( new Bounds2( 0, 0, 5, Number.POSITIVE_INFINITY ) ), 'Without epsilon: Mixed finite-ness - reflexive' );
    assert.ok( new Bounds2( 0, 0, 5, Number.POSITIVE_INFINITY ).equalsEpsilon( new Bounds2( 0, 0, 5, Number.POSITIVE_INFINITY ), epsilon ), 'With epsilon: Mixed finite-ness - reflexive' );
    assert.ok( !new Bounds2( 0, 0, 5, Number.POSITIVE_INFINITY ).equals( new Bounds2( 0, 0, 5, Number.NEGATIVE_INFINITY ) ), 'Without epsilon: Mixed finite-ness - swapped infinity' );
    assert.ok( !new Bounds2( 0, 0, 5, Number.POSITIVE_INFINITY ).equalsEpsilon( new Bounds2( 0, 0, 5, Number.NEGATIVE_INFINITY ), epsilon ), 'With epsilon: Mixed finite-ness - swapped infinity' );
    assert.ok( !new Bounds2( 0, 0, 5, Number.POSITIVE_INFINITY ).equals( new Bounds2( 0, 0, 6, Number.POSITIVE_INFINITY ) ), 'Without epsilon: Mixed finite-ness - different finite number' );
    assert.ok( !new Bounds2( 0, 0, 5, Number.POSITIVE_INFINITY ).equalsEpsilon( new Bounds2( 0, 0, 6, Number.POSITIVE_INFINITY ), epsilon ), 'With epsilon: Mixed finite-ness - different finite number' );
  } );
} );

Here are the results of running the test:

I elected to put the tests adjacent to the types they are testing, for instance js/Bounds2.js has js/Bounds2Test.js. This is to (1) keep code co-located to what it tests and (2) to give tests equal footing instead of putting them somewhere else (I know this is unconventional).

I'm inclined to move this branch to master and start porting unit tests. My outstanding questions:

1. Are the scenery tests doing something outside of the box that will be difficult for this framework?
2. How do we get Bayes to run tests from here for all repos that have them?
3. We need to add a build step that will output a compiled test HTML so we can test things after compilation.

@jonathanolson let's touch base again before I merge chipper and start porting tests.

[samreid]

@jonathanolson said this looks promising and we should try it out. I can take a few steps in aqua, but I'll need to coordinate with @jonathanolson to move it to the server.

[samreid]

I wrote to @ariel-phet to apprise him of the proposed plan:

I spoke with Jonathan today and he agreed the proposal for Unit Testing in #624 sounds promising. The main advantages are (a) we will be able to write tests using the same environment as for our simulation code (instead of having to use alternate patterns and module loading strategies) and (b) significantly reduced overhead to create unit tests. I'm going to spend some time porting dot/axon unit tests over to this paradigm then coordinate with Jon to get it running on Bayes. After that, we'll be able to evaluate whether it is straightforward enough for an intern or QA team member to port the rest of the unit tests or complex enough (or quick enough) for me to finish kite/scenery/cck/other tests.

[samreid]

Notes from discussion with @jonathanolson.

Using specific test HTML like http://localhost/scenery/tests/qunit/unit-tests.html
You can run it in the aqua harness like so: http://localhost/aqua/html/qunit-test.html?url=http://localhost/scenery/tests/qunit/unit-tests.html

May need encodeURIComponent( 'http://localhost/scenery/tests/qunit/unit-tests.html' ) on some platforms.

[samreid]

I've merged the chipper branch to master, and we can continue with review and improvement in master.