Inconsistent test: test_folder_operations (hypothesis, flaky) #811
Labels
Comments
|
One more occurrence: https://ci.appveyor.com/project/touilleMan/parsec-cloud/builds/29016436 |
This was referenced Dec 4, 2019
tests/core/mountpoint/test_folder_operations.py::test_folder_operations FAILED [ 50%]
================================== FAILURES ===================================
___________________________ test_folder_operations ____________________________
self = <hypothesis.core.StateForActualGivenExecution object at 0x0000027B7265B390>
data = ConjectureData(INTERESTING, 59 bytes, frozen)
def _execute_once_for_engine(self, data):
"""Wrapper around ``execute_once`` that intercepts test failure
exceptions and single-test control exceptions, and turns them into
appropriate method calls to `data` instead.
This allows the engine to assume that any exception other than
``StopTest`` must be a fatal error, and should stop the entire engine.
"""
try:
> result = self.execute_once(data)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\core.py:647:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <hypothesis.core.StateForActualGivenExecution object at 0x0000027B7265B390>
data = ConjectureData(INTERESTING, 59 bytes, frozen), print_example = False
is_final = False, expected_failure = None
def execute_once(
self, data, print_example=False, is_final=False, expected_failure=None
):
"""Run the test function once, using ``data`` as input.
If the test raises an exception, it will propagate through to the
caller of this method. Depending on its type, this could represent
an ordinary test failure, or a fatal error, or a control exception.
If this method returns normally, the test might have passed, or
it might have placed ``data`` in an unsuccessful state and then
swallowed the corresponding control exception.
"""
data.is_find = self.is_find
text_repr = [None]
if self.settings.deadline is None:
test = self.test
else:
@proxies(self.test)
def test(*args, **kwargs):
self.__test_runtime = None
initial_draws = len(data.draw_times)
start = benchmark_time()
result = self.test(*args, **kwargs)
finish = benchmark_time()
internal_draw_time = sum(data.draw_times[initial_draws:])
runtime = datetime.timedelta(
seconds=finish - start - internal_draw_time
)
self.__test_runtime = runtime
current_deadline = self.settings.deadline
if not is_final:
current_deadline = (current_deadline // 4) * 5
if runtime >= current_deadline:
raise DeadlineExceeded(runtime, self.settings.deadline)
return result
def run(data):
# Set up dynamic context needed by a single test run.
with local_settings(self.settings):
with deterministic_PRNG():
with BuildContext(data, is_final=is_final):
# Generate all arguments to the test function.
args, kwargs = data.draw(self.search_strategy)
if expected_failure is not None:
text_repr[0] = arg_string(test, args, kwargs)
if print_example or current_verbosity() >= Verbosity.verbose:
output = StringIO()
printer = RepresentationPrinter(output)
if print_example:
printer.text("Falsifying example:")
else:
printer.text("Trying example:")
if self.print_given_args:
printer.text(" ")
printer.text(test.__name__)
with printer.group(indent=4, open="(", close=""):
printer.break_()
for v in args:
printer.pretty(v)
# We add a comma unconditionally because
# generated arguments will always be
# kwargs, so there will always be more
# to come.
printer.text(",")
printer.breakable()
# We need to make sure to print these in the argument order for
# Python 2 and older versionf of Python 3.5. In modern versions
# this isn't an issue because kwargs is ordered.
arg_order = {
v: i
for i, v in enumerate(
getfullargspec(self.test).args
)
}
for i, (k, v) in enumerate(
sorted(
kwargs.items(),
key=lambda t: (
arg_order.get(t[0], float("inf")),
t[0],
),
)
):
printer.text(k)
printer.text("=")
printer.pretty(v)
printer.text(",")
if i + 1 < len(kwargs):
printer.breakable()
printer.break_()
printer.text(")")
printer.flush()
report(output.getvalue())
return test(*args, **kwargs)
# Run the test function once, via the executor hook.
# In most cases this will delegate straight to `run(data)`.
> result = self.test_runner(data, run)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\core.py:602:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
data = ConjectureData(INTERESTING, 59 bytes, frozen)
function = <function StateForActualGivenExecution.execute_once.<locals>.run at 0x0000027B724AF400>
def default_new_style_executor(data, function):
> return function(data)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\executors.py:52:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
data = ConjectureData(INTERESTING, 59 bytes, frozen)
def run(data):
# Set up dynamic context needed by a single test run.
with local_settings(self.settings):
with deterministic_PRNG():
with BuildContext(data, is_final=is_final):
# Generate all arguments to the test function.
args, kwargs = data.draw(self.search_strategy)
if expected_failure is not None:
text_repr[0] = arg_string(test, args, kwargs)
if print_example or current_verbosity() >= Verbosity.verbose:
output = StringIO()
printer = RepresentationPrinter(output)
if print_example:
printer.text("Falsifying example:")
else:
printer.text("Trying example:")
if self.print_given_args:
printer.text(" ")
printer.text(test.__name__)
with printer.group(indent=4, open="(", close=""):
printer.break_()
for v in args:
printer.pretty(v)
# We add a comma unconditionally because
# generated arguments will always be
# kwargs, so there will always be more
# to come.
printer.text(",")
printer.breakable()
# We need to make sure to print these in the argument order for
# Python 2 and older versionf of Python 3.5. In modern versions
# this isn't an issue because kwargs is ordered.
arg_order = {
v: i
for i, v in enumerate(
getfullargspec(self.test).args
)
}
for i, (k, v) in enumerate(
sorted(
kwargs.items(),
key=lambda t: (
arg_order.get(t[0], float("inf")),
t[0],
),
)
):
printer.text(k)
printer.text("=")
printer.pretty(v)
printer.text(",")
if i + 1 < len(kwargs):
printer.breakable()
printer.break_()
printer.text(")")
printer.flush()
report(output.getvalue())
> return test(*args, **kwargs)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\core.py:598:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
factory = <class 'tests.core.mountpoint.test_folder_operations.test_folder_operations.<locals>.FolderOperationsStateMachine'>
data = data(...)
@settings
@given(st.data())
def run_state_machine(factory, data):
machine = factory()
if not isinstance(machine, _GenericStateMachine):
raise InvalidArgument(
"Expected RuleBasedStateMachine but state_machine_factory() "
"returned %r (type=%s)" % (machine, type(machine).__name__)
)
data.conjecture_data.hypothesis_runner = machine
print_steps = (
current_build_context().is_final or current_verbosity() >= Verbosity.debug
)
try:
if print_steps:
machine.print_start()
machine.check_invariants()
max_steps = settings.stateful_step_count
steps_run = 0
cd = data.conjecture_data
while True:
# We basically always want to run the maximum number of steps,
# but need to leave a small probability of terminating early
# in order to allow for reducing the number of steps once we
# find a failing test case, so we stop with probability of
# 2 ** -16 during normal operation but force a stop when we've
# generated enough steps.
cd.start_example(STATE_MACHINE_RUN_LABEL)
if steps_run == 0:
cd.draw_bits(16, forced=1)
elif steps_run >= max_steps:
cd.draw_bits(16, forced=0)
break
else:
# All we really care about is whether this value is zero
# or non-zero, so if it's > 1 we discard it and insert a
# replacement value after
cd.start_example(SHOULD_CONTINUE_LABEL)
should_continue_value = cd.draw_bits(16)
if should_continue_value > 1:
cd.stop_example(discard=True)
cd.draw_bits(16, forced=int(bool(should_continue_value)))
else:
cd.stop_example()
if should_continue_value == 0:
break
steps_run += 1
value = data.conjecture_data.draw(machine.steps())
# Assign 'result' here in case 'execute_step' fails below
result = multiple()
try:
> result = machine.execute_step(value)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\stateful.py:134:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = FolderOperationsStateMachine({'file': [VarReference(name='v2')],
'folder': [VarReference(name='v1'), VarReference(name='v3')]})
step = (Rule(targets=(), function=test_folder_operations.<locals>.FolderOperationsStateMachine.iterdir, arguments={'path': Bundle(name='folder')}, precondition=None, bundles=(Bundle(name='folder'),)), {'path': VarReference(name='v3')})
def execute_step(self, step):
rule, data = step
data = dict(data)
for k, v in list(data.items()):
if isinstance(v, VarReference):
data[k] = self.names_to_values[v.name]
> result = rule.function(self, **data)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\stateful.py:849:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = FolderOperationsStateMachine({'file': [VarReference(name='v2')],
'folder': [VarReference(name='v1'), VarReference(name='v3')]})
path = PathElement(absolute_path='/w/jzs', parsec_root=WindowsPath('C:/Users/VssAdministrator/AppData/Local/Temp/pytest-of-Vs...VssAdministrator/AppData/Local/Temp/pytest-of-VssAdministrator/pytest-1/test_folder_operations0/oracle-test-240/root'))
@rule(path=Folders)
def iterdir(self, path):
expected_exc = None
try:
expected_children = {x.name for x in path.to_oracle().iterdir()}
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
> children = {x.name for x in path.to_parsec().iterdir()}
tests\core\mountpoint\test_folder_operations.py:219:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tests.core.mountpoint.test_folder_operations.expect_raises object at 0x0000027B7390DDA0>
exc_type = None, exc_value = None, traceback = None
def __exit__(self, exc_type, exc_value, traceback):
__tracebackhide__ = True
if not self.expected_exc:
return False
if not exc_type:
> raise AssertionError(f"DID NOT RAISED {self.expected_exc!r}")
E AssertionError: DID NOT RAISED FileNotFoundError(2, 'The system cannot find the path specified')
tests\core\mountpoint\test_folder_operations.py:39: AssertionError
During handling of the above exception, another exception occurred:
tmpdir = local('C:\\Users\\VssAdministrator\\AppData\\Local\\Temp\\pytest-of-VssAdministrator\\pytest-1\\test_folder_operations0')
caplog = <_pytest.logging.LogCaptureFixture object at 0x0000027B722D7438>
hypothesis_settings = settings(database=DirectoryBasedExampleDatabase('d:\\a\\_temp\\empty\\.hypothesis\\examples'), deadline=None, derandom...nt_blob=False, report_multiple_bugs=True, stateful_step_count=50, suppress_health_check=(), verbosity=Verbosity.normal)
mountpoint_service_factory = <function mountpoint_service_factory.<locals>._mountpoint_service_factory at 0x0000027B7251A2F0>
@pytest.mark.slow
@pytest.mark.mountpoint
def test_folder_operations(tmpdir, caplog, hypothesis_settings, mountpoint_service_factory):
tentative = 0
class FolderOperationsStateMachine(RuleBasedStateMachine):
Files = Bundle("file")
Folders = Bundle("folder")
# Moving mountpoint
NonRootFolder = Folders.filter(lambda x: not x.is_workspace())
@initialize(target=Folders)
def init(self):
nonlocal tentative
tentative += 1
caplog.clear()
async def _bootstrap(user_fs, mountpoint_manager):
wid = await user_fs.workspace_create("w")
await mountpoint_manager.mount_workspace(wid)
self.mountpoint_service = mountpoint_service_factory(_bootstrap)
self.folder_oracle = Path(tmpdir / f"oracle-test-{tentative}")
self.folder_oracle.mkdir()
oracle_root = self.folder_oracle / "root"
oracle_root.mkdir()
self.folder_oracle.chmod(0o500) # Root oracle can no longer be removed this way
(oracle_root / "w").mkdir()
oracle_root.chmod(0o500) # Also protect workspace from deletion
return PathElement(f"/w", self.mountpoint_service.base_mountpoint, oracle_root)
def teardown(self):
if hasattr(self, "mountpoint_service"):
self.mountpoint_service.stop()
@rule(target=Files, parent=Folders, name=st_entry_name)
def touch(self, parent, name):
path = parent / name
expected_exc = None
try:
path.to_oracle().touch(exist_ok=False)
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
path.to_parsec().touch(exist_ok=False)
return path
@rule(target=Folders, parent=Folders, name=st_entry_name)
def mkdir(self, parent, name):
path = parent / name
expected_exc = None
try:
path.to_oracle().mkdir(exist_ok=False)
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
path.to_parsec().mkdir(exist_ok=False)
return path
@rule(path=Files)
def unlink(self, path):
expected_exc = None
try:
path.to_oracle().unlink()
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
path.to_parsec().unlink()
@rule(path=Files, length=st.integers(min_value=0, max_value=16))
def resize(self, path, length):
expected_exc = None
try:
os.truncate(path.to_oracle(), length)
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
os.truncate(path.to_parsec(), length)
@rule(path=NonRootFolder)
def rmdir(self, path):
expected_exc = None
try:
path.to_oracle().rmdir()
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
path.to_parsec().rmdir()
def _move(self, src, dst_parent, dst_name):
dst = dst_parent / dst_name
expected_exc = None
try:
oracle_rename(src.to_oracle(), dst.to_oracle())
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
src.to_parsec().rename(str(dst.to_parsec()))
return dst
@rule(target=Files, src=Files, dst_parent=Folders, dst_name=st_entry_name)
def move_file(self, src, dst_parent, dst_name):
return self._move(src, dst_parent, dst_name)
@rule(target=Folders, src=NonRootFolder, dst_parent=Folders, dst_name=st_entry_name)
def move_folder(self, src, dst_parent, dst_name):
return self._move(src, dst_parent, dst_name)
@rule(path=Folders)
def iterdir(self, path):
expected_exc = None
try:
expected_children = {x.name for x in path.to_oracle().iterdir()}
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
children = {x.name for x in path.to_parsec().iterdir()}
if not expected_exc:
assert children == expected_children
> run_state_machine_as_test(FolderOperationsStateMachine, settings=hypothesis_settings)
tests\core\mountpoint\test_folder_operations.py:224:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis_trio\stateful.py:318: in run_state_machine_as_test
return original(state_machine_factory, settings=settings)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\stateful.py:161: in run_state_machine_as_test
run_state_machine(state_machine_factory)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\stateful.py:80: in run_state_machine
@given(st.data())
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\pareto.py:317: in run
lambda data: data.status >= Status.VALID
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinker.py:419: in shrink
self.greedy_shrink()
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinker.py:502: in greedy_shrink
block_program("--X"),
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinker.py:551: in fixate_shrink_passes
if sp.step():
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinker.py:1475: in step
lambda chooser: self.run_with_chooser(self.shrinker, chooser),
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\choicetree.py:117: in step
f(chooser)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinker.py:1475: in <lambda>
lambda chooser: self.run_with_chooser(self.shrinker, chooser),
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinker.py:1102: in minimize_duplicated_blocks
full=False,
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\common.py:127: in shrink
shrinker.run()
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\common.py:144: in run
self.run_step()
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\lexical.py:65: in run_step
self.minimize_as_integer()
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\lexical.py:53: in minimize_as_integer
full=full,
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\common.py:127: in shrink
shrinker.run()
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\common.py:136: in run
if self.short_circuit():
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\integer.py:33: in short_circuit
if self.consider(i):
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\common.py:176: in consider
return self.incorporate(value)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\common.py:163: in incorporate
if self.__predicate(value):
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\lexical.py:51: in <lambda>
lambda c: c == self.current_int or self.incorporate_int(c),
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\lexical.py:42: in incorporate_int
return self.incorporate(int_to_bytes(i, self.size))
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinking\common.py:163: in incorporate
if self.__predicate(value):
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinker.py:1100: in <lambda>
lambda b: self.try_shrinking_blocks(targets, b),
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinker.py:820: in try_shrinking_blocks
initial_data = self.cached_test_function(initial_attempt)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\shrinker.py:388: in cached_test_function
result = self.engine.cached_test_function(buffer)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\core.py:699: in _execute_once_for_engine
data.mark_interesting((type(e), filename, lineno))
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\data.py:1050: in mark_interesting
self.conclude_test(Status.INTERESTING, interesting_origin)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\data.py:1046: in conclude_test
self.freeze()
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\data.py:974: in freeze
self.observer.conclude_test(self.status, self.interesting_origin)
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\datatree.py:408: in conclude_test
inconsistent_generation()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def inconsistent_generation():
raise Flaky(
> "Inconsistent data generation! Data generation behaved differently "
"between different runs. Is your data generation depending on external "
"state?"
)
E hypothesis.errors.Flaky: Inconsistent data generation! Data generation behaved differently between different runs. Is your data generation depending on external state?
c:\hostedtoolcache\windows\python\3.6.8\x64\lib\site-packages\hypothesis\internal\conjecture\datatree.py:36: Flaky
---------------------------- Captured stdout call -----------------------------
You can add @seed(269616729533588432152393235229938201085) to this test or run pytest with --hypothesis-seed=269616729533588432152393235229938201085 to reproduce this failure.
------ generated xml file: d:\a\_temp\empty\test-results-mountpoint.xml -------
========================== slowest 10 test durations ==========================
50.76s call tests/core/mountpoint/test_folder_operations.py::test_folder_operations
44.80s call tests/core/mountpoint/test_file_operations.py::test_file_operations
36.75s call tests/core/mountpoint/test_base.py::test_mountpoint_iterdir_with_many_files[/foo-1000]
31.54s call tests/core/mountpoint/test_base.py::test_mountpoint_iterdir_with_many_files[/-1000]
7.78s call tests/core/mountpoint/test_base.py::test_mountpoint_access_unicode
1.73s setup tests/core/mountpoint/test_base.py::test_mountpoint_iterdir_with_many_files[/-100]
1.55s call tests/core/mountpoint/test_base.py::test_work_within_logged_core
1.19s call tests/core/mountpoint/test_base.py::test_mountpoint_iterdir_with_many_files[/foo-100]
1.11s call tests/core/mountpoint/test_base.py::test_mountpoint_iterdir_with_many_files[/-100]
0.77s call tests/core/mountpoint/test_base.py::test_mountpoint_revoke_access[write]
====== 1 failed, 24 passed, 2 skipped, 1038 deselected in 195.35 seconds ======
##[error]Bash exited with code '1'.
##[section]Finishing: Tests mountpoint |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Seen on appveyor (
mountpoint\test_folder_operations.py::test_folder_operations):The text was updated successfully, but these errors were encountered: