diff --git a/datafiles/settings.py b/datafiles/settings.py index d8c82a25..15bdd5bc 100644 --- a/datafiles/settings.py +++ b/datafiles/settings.py @@ -9,3 +9,5 @@ MINIMIZE_LIST_DIFFS = True YAML_LIBRARY = 'ruamel.yaml' + +WRITE_DELAY = 0.0 # seconds diff --git a/datafiles/utils.py b/datafiles/utils.py index 94e55c9b..e65d88fe 100644 --- a/datafiles/utils.py +++ b/datafiles/utils.py @@ -1,6 +1,7 @@ """Internal helper functions.""" import dataclasses +import time from contextlib import suppress from functools import lru_cache from pathlib import Path @@ -10,6 +11,7 @@ import log +from . import settings from .types import Missing @@ -114,6 +116,7 @@ def write(filename_or_path: Union[str, Path], text: str, *, display=False) -> No path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text) + time.sleep(settings.WRITE_DELAY) # ensure the file modification time changes def read(filename: str, *, display=False) -> str: diff --git a/docs/settings.md b/docs/settings.md index a7632ac8..41c5b8da 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -80,3 +80,15 @@ This setting controls the underlying YAML library used to read and write files. - `'ruamel.yaml'` (default) - `'PyYAML'` + +# `WRITE_DELAY` + +One some file systems, the modification time of a file ([`st_mtime`](https://docs.python.org/3/library/os.html#os.stat_result.st_mtime)) is unchanged if a file is read immediately after writing. This may cause intermittent issues if your use case involves rapidly changing files. + +To compensate for this, a short delay can be inserted after `datafiles` writes to the file system: + +```python +import datafiles + +datafiles.settings.WRITE_DELAY = 0.01 # seconds +``` diff --git a/poetry.lock b/poetry.lock index a64bf911..f5dc8113 100644 --- a/poetry.lock +++ b/poetry.lock @@ -42,20 +42,6 @@ dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pytest", "sphinx", "wheel", "p docs = ["sphinx"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pytest"] -[[package]] -name = "astroid" -version = "2.4.2" -description = "An abstract syntax tree for Python with inference support." -category = "dev" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -lazy-object-proxy = ">=1.4.0,<1.5.0" -six = ">=1.12,<2.0" -typed-ast = {version = ">=1.4.0,<1.5", markers = "implementation_name == \"cpython\" and python_version < \"3.8\""} -wrapt = ">=1.11,<2.0" - [[package]] name = "async-generator" version = "1.10" @@ -484,14 +470,6 @@ python-versions = "*" [package.dependencies] pygments = ">=2.4.1,<3" -[[package]] -name = "lazy-object-proxy" -version = "1.4.3" -description = "A fast and thorough lazy object proxy." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - [[package]] name = "livereload" version = "2.6.3" @@ -526,14 +504,6 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" -[[package]] -name = "mccabe" -version = "0.6.1" -description = "McCabe checker, plugin for flake8" -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "minilog" version = "2.0" @@ -1369,14 +1339,6 @@ python-versions = "*" [package.dependencies] notebook = ">=4.4.1" -[[package]] -name = "wrapt" -version = "1.12.1" -description = "Module for decorators, wrappers and monkey patching." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "zipp" version = "3.2.0" @@ -1392,7 +1354,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "f2b13e51d7570666c16cdbdcc2651cd71b394763a68fab08893b57bd65704263" +content-hash = "a1b70808b034a941d06511295dda8ed51c545b49651c4fc46b78baeb4ad4359c" [metadata.files] ansiwrap = [ @@ -1427,10 +1389,6 @@ argon2-cffi = [ {file = "argon2_cffi-20.1.0-cp39-cp39-win32.whl", hash = "sha256:e2db6e85c057c16d0bd3b4d2b04f270a7467c147381e8fd73cbbe5bc719832be"}, {file = "argon2_cffi-20.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:8a84934bd818e14a17943de8099d41160da4a336bcc699bb4c394bbb9b94bd32"}, ] -astroid = [ - {file = "astroid-2.4.2-py3-none-any.whl", hash = "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386"}, - {file = "astroid-2.4.2.tar.gz", hash = "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703"}, -] async-generator = [ {file = "async_generator-1.10-py3-none-any.whl", hash = "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b"}, {file = "async_generator-1.10.tar.gz", hash = "sha256:6ebb3d106c12920aaae42ccb6f787ef5eefdcdd166ea3d628fa8476abe712144"}, @@ -1632,29 +1590,6 @@ jupyterlab-pygments = [ {file = "jupyterlab_pygments-0.1.2-py2.py3-none-any.whl", hash = "sha256:abfb880fd1561987efaefcb2d2ac75145d2a5d0139b1876d5be806e32f630008"}, {file = "jupyterlab_pygments-0.1.2.tar.gz", hash = "sha256:cfcda0873626150932f438eccf0f8bf22bfa92345b814890ab360d666b254146"}, ] -lazy-object-proxy = [ - {file = "lazy-object-proxy-1.4.3.tar.gz", hash = "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0"}, - {file = "lazy_object_proxy-1.4.3-cp27-cp27m-macosx_10_13_x86_64.whl", hash = "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442"}, - {file = "lazy_object_proxy-1.4.3-cp27-cp27m-win32.whl", hash = "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4"}, - {file = "lazy_object_proxy-1.4.3-cp27-cp27m-win_amd64.whl", hash = "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a"}, - {file = "lazy_object_proxy-1.4.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d"}, - {file = "lazy_object_proxy-1.4.3-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a"}, - {file = "lazy_object_proxy-1.4.3-cp34-cp34m-win32.whl", hash = "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e"}, - {file = "lazy_object_proxy-1.4.3-cp34-cp34m-win_amd64.whl", hash = "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357"}, - {file = "lazy_object_proxy-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50"}, - {file = "lazy_object_proxy-1.4.3-cp35-cp35m-win32.whl", hash = "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db"}, - {file = "lazy_object_proxy-1.4.3-cp35-cp35m-win_amd64.whl", hash = "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449"}, - {file = "lazy_object_proxy-1.4.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156"}, - {file = "lazy_object_proxy-1.4.3-cp36-cp36m-win32.whl", hash = "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531"}, - {file = "lazy_object_proxy-1.4.3-cp36-cp36m-win_amd64.whl", hash = "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb"}, - {file = "lazy_object_proxy-1.4.3-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08"}, - {file = "lazy_object_proxy-1.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383"}, - {file = "lazy_object_proxy-1.4.3-cp37-cp37m-win32.whl", hash = "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142"}, - {file = "lazy_object_proxy-1.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea"}, - {file = "lazy_object_proxy-1.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62"}, - {file = "lazy_object_proxy-1.4.3-cp38-cp38-win32.whl", hash = "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd"}, - {file = "lazy_object_proxy-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239"}, -] livereload = [ {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, ] @@ -1697,10 +1632,6 @@ markupsafe = [ {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"}, {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] -mccabe = [ - {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, - {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, -] minilog = [ {file = "minilog-2.0-py3-none-any.whl", hash = "sha256:891ad346bdd63aee4c210faa688497f7ba412cf52a54fb6cba6bf511a34a5138"}, {file = "minilog-2.0.tar.gz", hash = "sha256:58499302bca86cf507eb3c3dfa3853de9389bfde275ba5155bdc3b4551175918"}, @@ -2139,9 +2070,6 @@ widgetsnbextension = [ {file = "widgetsnbextension-3.5.1-py2.py3-none-any.whl", hash = "sha256:bd314f8ceb488571a5ffea6cc5b9fc6cba0adaf88a9d2386b93a489751938bcd"}, {file = "widgetsnbextension-3.5.1.tar.gz", hash = "sha256:079f87d87270bce047512400efd70238820751a11d2d8cb137a5a5bdbaf255c7"}, ] -wrapt = [ - {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, -] zipp = [ {file = "zipp-3.2.0-py3-none-any.whl", hash = "sha256:43f4fa8d8bb313e65d8323a3952ef8756bf40f9a5c3ea7334be23ee4ec8278b6"}, {file = "zipp-3.2.0.tar.gz", hash = "sha256:b52f22895f4cfce194bc8172f3819ee8de7540aa6d873535a8668b730b8b411f"}, diff --git a/pyproject.toml b/pyproject.toml index 5f8ddc2d..510620d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "datafiles" -version = "0.12b3" +version = "0.12b4" description = "File-based ORM for dataclasses." license = "MIT" @@ -66,7 +66,7 @@ isort = "=5.5.1" # Linters mypy = "~0.790" -pylint = { git = "https://github.com/PyCQA/pylint", branch = "master" } +pylint = "~2.6.0" pydocstyle = "*" # Testing diff --git a/pytest.ini b/pytest.ini index bfdf8357..89bb2591 100644 --- a/pytest.ini +++ b/pytest.ini @@ -15,9 +15,6 @@ cache_dir = .cache log_level = DEBUG log_format = %(relpath)s:%(lineno)-4d %(levelname)5s: %(message)s -markers = - flaky - [pytest-watch] ignore =.cache,htmlcov,tml diff --git a/tests/conftest.py b/tests/conftest.py index 9a0cf6ad..20d6f680 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import os from pathlib import Path from shutil import rmtree @@ -9,9 +8,7 @@ settings.HIDE_TRACEBACK_IN_HOOKS = False - - -xfail_on_ci = pytest.mark.xfail(bool(os.getenv('CI')), reason="Flaky on CI") +settings.WRITE_DELAY = 0.1 def pytest_configure(config): @@ -20,13 +17,6 @@ def pytest_configure(config): log.init() # TODO: determine why the 'relpath' filter wasn't added automatically -def pytest_collection_modifyitems(items): - for item in items: - for marker in item.iter_markers(): - if marker.name == 'flaky': - item.add_marker(xfail_on_ci) - - @pytest.fixture(autouse=True) def create_tmp(): path = Path('tmp') diff --git a/tests/test_automatic_attributes.py b/tests/test_automatic_attributes.py index a042f052..2cda5f24 100644 --- a/tests/test_automatic_attributes.py +++ b/tests/test_automatic_attributes.py @@ -1,8 +1,6 @@ # pylint: disable=unused-variable -import pytest - from datafiles import datafile from datafiles.utils import logbreak @@ -13,7 +11,6 @@ class Sample: def describe_auto_attr(): - @pytest.mark.flaky def with_builtin(expect): sample = Sample('abc') @@ -28,7 +25,6 @@ def with_builtin(expect): logbreak("Getting attribute") expect(sample.count) == 4 - @pytest.mark.flaky def with_empty_list(expect): sample = Sample('abc') @@ -44,7 +40,6 @@ def with_empty_list(expect): logbreak("Getting attribute") expect(sample.empty_items) == [4.2, "abc"] - @pytest.mark.flaky def with_homogeneous_list(expect): sample = Sample('abc') @@ -59,7 +54,6 @@ def with_homogeneous_list(expect): logbreak("Getting attribute") expect(sample.same_items) == [1, 2, 3] - @pytest.mark.flaky def with_heterogeneous_list(expect): sample = Sample('abc') @@ -74,7 +68,6 @@ def with_heterogeneous_list(expect): logbreak("Getting attribute") expect(sample.mixed_items) == [1, "abc", 3.2] - @pytest.mark.flaky def with_dict(expect): sample = Sample('abc') diff --git a/tests/test_custom_converters.py b/tests/test_custom_converters.py index cdb67b9e..a78266ce 100644 --- a/tests/test_custom_converters.py +++ b/tests/test_custom_converters.py @@ -2,8 +2,6 @@ from datetime import datetime -import pytest - from datafiles import Missing, converters, datafile @@ -31,7 +29,6 @@ def to_python_value(cls, deserialized_data, **_kwargs): return datetime.fromisoformat(deserialized_data) -@pytest.mark.flaky def test_extension(expect): @datafile("../tmp/sample.yml") class Timestamp: @@ -47,7 +44,6 @@ class Timestamp: expect(ts.dt.day) == 11 -@pytest.mark.flaky def test_extension_with_default(expect): @datafile("../tmp/sample.yml") class Timestamp: @@ -63,7 +59,6 @@ class Timestamp: expect(ts.dt.day) == 11 -@pytest.mark.flaky def test_registration(expect): converters.register(datetime, DateTimeConverter) @@ -82,7 +77,6 @@ class Timestamp: expect(ts.dt.day) == 22 -@pytest.mark.flaky def test_registration_with_default(expect): converters.register(datetime, DateTimeConverter) diff --git a/tests/test_custom_converters_future.py b/tests/test_custom_converters_future.py index df9895e6..6c58b915 100644 --- a/tests/test_custom_converters_future.py +++ b/tests/test_custom_converters_future.py @@ -11,7 +11,6 @@ from .test_custom_converters import MyDateTime -@pytest.mark.flaky def test_extension(expect): @datafile("../tmp/sample.yml") class MyObject: diff --git a/tests/test_extended_converters.py b/tests/test_extended_converters.py index c61d552f..7ffaf1dd 100644 --- a/tests/test_extended_converters.py +++ b/tests/test_extended_converters.py @@ -38,7 +38,6 @@ def with_float_to_integer(sample, expect): """ ) - @pytest.mark.flaky def with_integer_to_float(sample, expect): write( 'tmp/sample.yml', @@ -73,7 +72,6 @@ def with_single_line(sample, expect): """ ) - @pytest.mark.flaky def with_multiple_lines(sample, expect): sample.text = '\n'.join(f'Line {i+1}' for i in range(3)) diff --git a/tests/test_loading.py b/tests/test_loading.py index 9ab582a2..0be1daf5 100644 --- a/tests/test_loading.py +++ b/tests/test_loading.py @@ -245,7 +245,6 @@ def with_extra_attributes(sample, expect): expect(sample.nested.score) == 3.4 expect(hasattr(sample.nested, 'extra')).is_(False) - @pytest.mark.flaky def with_multiple_levels(expect): @dataclass class Bottom: diff --git a/tests/test_patched_methods.py b/tests/test_patched_methods.py index b34c2d4b..3812578b 100644 --- a/tests/test_patched_methods.py +++ b/tests/test_patched_methods.py @@ -5,8 +5,6 @@ from dataclasses import dataclass, field from typing import Dict, List -import pytest - from datafiles import datafile from datafiles.utils import dedent, logbreak, read, write @@ -32,7 +30,6 @@ class SampleWithNesting: def describe_automatic_load(): - @pytest.mark.flaky def with_getattribute(expect): sample = Sample() @@ -48,7 +45,6 @@ def with_getattribute(expect): def describe_automatic_save(): - @pytest.mark.flaky def with_setattr(expect): sample = Sample() @@ -186,7 +182,6 @@ def with_update(expect): def describe_automatic_load_before_save(): - @pytest.mark.flaky def with_setattr(expect): sample = Sample()