NoMethodError is randomly raised

[arika]

The following test code may randomly raised NoMethodError in setup block:

setup do
  @a = [1, 2, 3]
end

sub_test_case 't' do
  setup do
    @b = @a[1] # this line randomly raised "NoMethodError: undefined method `[]' for nil:NilClass"
  end
  test 't' do
    assert true
  end
end

I think:

• setup block is GC'ed after define_mehod
• the setup block's object id is reused for the another setup block
• reused object id is added multiple times in fixture.after_callbacks(:setup) list
• then too early inner setup block call is occured

test code:

# frozen_string_literal: true

require 'test/unit'
require 'test/unit/version'

p Test::Unit::VERSION

class T < Test::Unit::TestCase
  n = 10
  m = 100
  t = [[0] * n, [1] * n, [2] * n]

  msg = lambda do |t|
    names = t.class.fixture.after_callbacks(:setup)
      .each_with_object(Hash.new(0)) { |name, h| h[name] += 1 }
      .reject { |_name, c| c == 1 }
      .keys
    locations = []
    t.class.ancestors.each do |cls|
      names.each do |name|
        m = cls.instance_method(name)
        locations << "#{name} #{m.owner.name} #{m.source_location.join(':')}"
      rescue NameError
      end
    end
    locations.sort.join("\n")
  end

  exception_handler do |t, e|
    puts msg.call(t) if e.is_a?(NoMethodError)
    false
  end

  setup { @t = [] }

  setup { @t << [] }
  n.times { setup { @t.last << 0 } }

  m.times do |i|
    sub_test_case "t1_#{i}" do
      setup { @t << [] }
      n.times { setup { @t.last << 1 } }

      sub_test_case 't2' do
        setup { @t << [] }
        n.times { setup { @t.last << 2 } }

        test 't' do
          assert_equal t, @t , -> { msg.call(self) }
        end
      end
    end
  end
end

result:

$ for i in $(seq 10); do ruby test_unit_test.rb --stop-on-failure || break; done
"3.4.0"
Loaded suite test_unit_test
Started
....................................................................................................
Finished in 0.021225 seconds.
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
100 tests, 100 assertions, 0 failures, 0 errors, 0 pendings, 0 omissions, 0 notifications
100% passed
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
4711.43 tests/s, 4711.43 assertions/s
"3.4.0"
Loaded suite test_unit_test
Started
....................................................................................................
Finished in 0.020865 seconds.
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
100 tests, 100 assertions, 0 failures, 0 errors, 0 pendings, 0 omissions, 0 notifications
100% passed
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
4792.72 tests/s, 4792.72 assertions/s
"3.4.0"
Loaded suite test_unit_test
Started
......................F
=====================================================================================================================================================================================
Failure: test: t(T::t1_29::t2)
test_unit_test.rb:57:in `block (4 levels) in <class:T>'
     54:         n.times { setup { @t.last << 2 } }
     55: 
     56:         test 't' do
  => 57:           assert_equal t, @t , -> { msg.call(self) }
     58:         end
     59:       end
     60:     end
setup_70182433290420 T test_unit_test.rb:45
setup_70182433290420 T::t1_29 test_unit_test.rb:50
setup_70182433290420 T::t1_29 test_unit_test.rb:50
<[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
 [2, 2, 2, 2, 2, 2, 2, 2, 2, 2]]> expected but was
<[[0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
 [2, 2, 2, 2, 2, 2, 2, 2, 2, 2]]>

diff:
? [[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
   [2, 2, 2, 2, 2, 2, 2, 2, 2, 2]]
=====================================================================================================================================================================================

Finished in 0.011826 seconds.
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
23 tests, 23 assertions, 1 failures, 0 errors, 0 pendings, 0 omissions, 0 notifications
95.6522% passed
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
1944.87 tests/s, 1944.87 assertions/s

[kou]

Interesting.
Could you show the version of Ruby that can reproduce this case?

[arika]

2.6.x only can reproduce this problem. 😓

• 2.6.6 - can reproduce
• 2.6.7 - can reproduce
• 2.7.3 - can not reproduce
• 3.0.1 - can not reproduce

[kou]

Thanks.
I can confirm this with Ruby 2.6.

I've fixed and released a new version.