Fix CE tests and leakage issue (#259)

src/scenic/core/external_params.py

@@ -206,10 +206,12 @@ def __init__(self, params, globalParams):
         if usingProbs and samplerType == "ce":
             if samplerParams is None:
                 samplerParams = DotMap()
+            else:
+                samplerParams = samplerParams.copy()  # avoid mutating original
             if "cont" in samplerParams or "disc" in samplerParams:
                 raise RuntimeError(
                     "CE distributions specified in both VerifaiParameters"
-                    "and verifaiSamplerParams"
+                    " and verifaiSamplerParams"
                 )
             cont_buckets = []
             cont_dists = []

tests/syntax/test_verifai_samplers.py

@@ -1,5 +1,6 @@
 import random
 
+import numpy as np
 import pytest
 
 from tests.utils import compileScenic, sampleEgo, sampleParamP
@@ -25,12 +26,45 @@ def sampleEgoWithFeedback(scenario, f, numSamples, maxIterations=1):
     return egos
 
 
-def checkCEConvergence(scenario, rangeCheck=(lambda x: x == -1 or x == 1)):
+@pytest.fixture
+def checkCEConvergence(pytestconfig):
+    fast = pytestconfig.getoption("--fast")
+
+    def helper(*args, **kwargs):
+        if "fast" not in kwargs:
+            kwargs["fast"] = fast
+        return _ceHelper(*args, **kwargs)
+
+    return helper
+
+
+def _ceHelper(scenario, rangeCheck=None, warmup=0, fast=False):
+    if not rangeCheck:
+        rangeCheck = lambda x: x == -1 or x == 1
     f = lambda ego: -1 if ego.position.x > 0 else 1
-    xs = [ego.position.x for ego in sampleEgoWithFeedback(scenario, f, 1200)]
+    if fast:
+        iterations = 120
+    else:
+        iterations = 1250 + warmup
+    xs = [ego.position.x for ego in sampleEgoWithFeedback(scenario, f, iterations)]
     assert all(rangeCheck(x) for x in xs)
-    assert 22 <= sum(x < 0 for x in xs[:30])
-    assert 143 <= sum(x > 0 for x in xs[200:])
+    assert any(x <= 0 for x in xs[:120])
+    if fast:
+        return
+    # The following parameters are for alpha=0.99, 1250 iterations, 2 buckets
+    # and all the counterexamples in the x > 0 bucket. With a uniform prior, no
+    # warmup is needed; for 9:1 odds against the right bucket, use warmup=1300.
+    xs = xs[warmup:]
+    assert 58 <= sum(x > 0 for x in xs[:250])
+    assert 590 <= sum(x > 0 for x in xs[250:])
+    assert 195 <= sum(x > 0 for x in xs[1000:])
+
+
+def checkCEStationary(scenario):
+    f = lambda ego: 1
+    xs = [ego.position.x for ego in sampleEgoWithFeedback(scenario, f, 250)]
+    # These parameters assume 2 buckets with 9:1 odds against the positive bucket
+    assert 175 <= sum(x <= 0 for x in xs) < 250
 
 
 ## Particular samplers
@@ -46,61 +80,89 @@ def test_halton():
     assert 29 <= sum(x < 10 for x in xs) <= 31
 
 
-def test_cross_entropy():
+def test_cross_entropy(checkCEConvergence):
     scenario = compileScenic(
-        'param verifaiSamplerType = "ce"\n'
-        "from dotmap import DotMap\n"
-        "ce_params = DotMap()\n"
-        "ce_params.alpha = 0.9\n"
-        "ce_params.cont.buckets = 2\n"
-        "param verifaiSamplerParams = ce_params\n"
-        "ego = new Object at VerifaiRange(5, 15) @ 0"
+        """
+        param verifaiSamplerType = "ce"
+        from dotmap import DotMap
+        ce_params = DotMap()
+        ce_params.alpha = 0.99
+        ce_params.cont.buckets = 2
+        param verifaiSamplerParams = ce_params
+        ego = new Object at VerifaiRange(-5, 5) @ 0
+        """
     )
-    f = lambda ego: -1 if ego.position.x < 10 else 1
-    xs = [ego.position.x for ego in sampleEgoWithFeedback(scenario, f, 120)]
-    assert all(5 <= x <= 15 for x in xs)
-    assert any(x > 10 for x in xs)
-    assert 66 <= sum(x < 10 for x in xs[50:])
+    checkCEConvergence(scenario, rangeCheck=(lambda x: -5 <= x <= 5), fast=False)
 
 
-def test_cross_entropy_inline():
+def test_cross_entropy_weights(checkCEConvergence):
     scenario = compileScenic(
-        'param verifaiSamplerType = "ce"\n'
-        "from dotmap import DotMap\n"
-        "param verifaiSamplerParams = DotMap(alpha=0.99)\n"
-        "ego = new Object at VerifaiRange(-1, 1, weights=[100, 1]) @ 0"
+        """
+        param verifaiSamplerType = "ce"
+        from dotmap import DotMap
+        param verifaiSamplerParams = DotMap(alpha=0.99)
+        ego = new Object at VerifaiRange(-1, 1, weights=[9, 1]) @ 0
+        """
     )
-    checkCEConvergence(scenario, rangeCheck=(lambda x: -1 <= x <= 1))
+
+    # Save scenario state which should not be mutated by sampling
+    params = scenario.params["verifaiSamplerParams"].copy()
+    contCESampler = scenario.externalSampler.sampler.domainSampler.cont_sampler
+    prior = contCESampler.dist.copy()
+
+    # Generate samples and check convergence to the correct bucket
+    checkCEConvergence(scenario, rangeCheck=(lambda x: -1 <= x <= 1), warmup=1300)
+
+    # Check the scenario state is unchanged
+    assert scenario.params["verifaiSamplerParams"] == params
+    scenario.resetExternalSampler()
+    contCESampler = scenario.externalSampler.sampler.domainSampler.cont_sampler
+    assert np.array_equal(contCESampler.dist, prior)
+
+    # Generate new samples without feedback and check the prior distribution is used
+    checkCEStationary(scenario)
 
 
-def test_cross_entropy_options():
+def test_cross_entropy_options(checkCEConvergence):
     scenario = compileScenic(
-        'param verifaiSamplerType = "ce"\n'
-        "from dotmap import DotMap\n"
-        "param verifaiSamplerParams = DotMap(alpha=0.99)\n"
-        "ego = new Object at VerifaiOptions({-1: 100, 1: 1}) @ 0"
+        """
+        param verifaiSamplerType = "ce"
+        from dotmap import DotMap
+        param verifaiSamplerParams = DotMap(alpha=0.99)
+        ego = new Object at VerifaiOptions({-1: 9, 1: 1}) @ 0
+        """
     )
-    checkCEConvergence(scenario)
+    checkCEConvergence(scenario, warmup=1300)
+    scenario.resetExternalSampler()
+    checkCEStationary(scenario)
 
 
-def test_cross_entropy_prior():
+def test_cross_entropy_prior(checkCEConvergence):
     scenario = compileScenic(
-        'param verifaiSamplerType = "ce"\n'
-        "from dotmap import DotMap\n"
-        "param verifaiSamplerParams = DotMap(alpha=0.99)\n"
-        "ego = new Object at VerifaiParameter.withPrior(Options({-1: 100, 1: 1})) @ 0"
+        """
+        param verifaiSamplerType = "ce"
+        from dotmap import DotMap
+        param verifaiSamplerParams = DotMap(alpha=0.99)
+        ego = new Object at VerifaiParameter.withPrior(Options({-1: 9, 1: 1})) @ 0
+        """
     )
-    checkCEConvergence(scenario)
+    checkCEConvergence(scenario, warmup=1300)
+    scenario.resetExternalSampler()
+    checkCEStationary(scenario)
 
 
-def test_cross_entropy_prior_normal():
+def test_cross_entropy_prior_normal(checkCEConvergence):
     scenario = compileScenic(
-        'param verifaiSamplerType = "ce"\n'
-        "from dotmap import DotMap\n"
-        "param verifaiSamplerParams = DotMap(alpha=0.99)\n"
-        "ego = new Object at VerifaiParameter.withPrior(Normal(-1, 0.7)) @ 0"
+        """
+        param verifaiSamplerType = "ce"
+        from dotmap import DotMap
+        param verifaiSamplerParams = DotMap(alpha=0.99)
+        ego = new Object at VerifaiParameter.withPrior(Normal(-1, 0.8)) @ 0
+        """
     )
-    checkCEConvergence(scenario, rangeCheck=(lambda x: True))
+    checkCEConvergence(scenario, rangeCheck=(lambda x: True), warmup=1300)
+    scenario.resetExternalSampler()
+    checkCEStationary(scenario)
 
 
 ## Reproducibility and noninterference