diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md deleted file mode 100644 index 24e368fc..00000000 --- a/.github/CONTRIBUTING.md +++ /dev/null @@ -1,47 +0,0 @@ -# Contributing to Causal Testing Framework - -### Questions -Ask any questions about the Causal Testing Framework or surrounding concepts on the -[discussions board](https://github.com/CITCOM-project/CausalTestingFramework/discussions). Before opening a new -discussion, please see whether a relevant one already exists - someone may have answered your question already. - -### Reporting Bugs and Making Suggestions -Upon identifying any bugs or features that could be improved, please open an -[issue](https://github.com/CITCOM-project/CausalTestingFramework/issues) and label with bug or suggestion. Every issue -should clearly explain the bug or feature to be improved and, where necessary, instructions to replicate. - -### Making a Pull Request -In order to directly contribute to the Causal Testing Framework, the following steps must be taken: -1. Create a new [branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository). - - This branch should have a name that describes the feature which is changed or added. - - Work directly onto this branch, making sure that you follow our style guidelines outlined below. -2. Open a [pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) - from your branch to the main branch. - - Explain the changes made or feature added. - - [Request a review](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review). - - The pull request will have to pass our [continuous integration (CI) checks](#continuous-integration-ci) and receive an - approving review, which will be determined by our [review guidelines](). - -### Continuous Integration (CI) -Upon pushing or pulling, the following GitHub actions will be triggered: - - Build: install Python dependencies - - Linting: check style guidelines have been met. - - Testing: run unit and regression tests. - -### Coding Style -In the Causal Testing Framework, we aim to provide highly reusable and easily maintainable packages. To this end, -we ask contributors to stick to the following guidelines: -1. Make small and frequent changes rather than verbose and infrequent changes. -2. Favour readable and informative variable, method, and class names over concise names. -3. Use logging instead of print. -4. For every method and class, include detailed docstrings following the - [reStructuredText/Sphinx](https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html) guidelines. -5. Add credit and license information where existing code has been used (in the method or class docstring). -6. If a method implements a specific algorithm/technique from a paper, add a citation to the docstring. -7. Use [variable](https://www.python.org/dev/peps/pep-0008/#variable-annotations) and [function](https://www.python.org/dev/peps/pep-0008/#function-annotations) - annotations. -8. All methods should be thoroughly tested with PyTest (see [Testing]() below). -9. Optionally run `autopep8 causal_testing` to prevent linter errors. -10. Optionally run `isort causal_testing` to ensure imports are done in the right order (this will also help prevent linter errors). -11. Format code using `black causal_testing`. -12. Linting code using `pylint causal_testing` is advised. diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index dc1403c1..31bd8503 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - name: Set up Python diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..c048887f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,40 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-added-large-files + - id: check-merge-conflict + - id: debug-statements + - id: mixed-line-ending + files: \.py$ + + # Black + - repo: https://github.com/psf/black + rev: 25.9.0 + hooks: + - id: black + args: ['--line-length=120', '--target-version=py310'] + files: ^causal_testing/ + + # isort + - repo: https://github.com/pycqa/isort + rev: 7.0.0 + hooks: + - id: isort + args: ['--profile', 'black', '--line-length', '120'] + files: ^causal_testing/ + + # Pylint + - repo: local + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: ['--max-line-length=120', '--max-positional-arguments=12'] + files: ^causal_testing/ diff --git a/.pylintrc b/.pylintrc index 4ab81c8c..ddb6341a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -82,14 +82,15 @@ persistent=yes # Minimum Python version to use for version dependent checks. Will default to # the version used to run pylint. -py-version=3.9 +py-version=3.10 # Discover python modules and packages in the file system subtree. recursive=no # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. -suggestion-mode=yes +# REMOVED: suggestion-mode is deprecated and not recognized in newer versions +# suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. @@ -154,6 +155,9 @@ disable=raw-checker-failed, use-symbolic-message-instead, logging-fstring-interpolation, import-error, + too-many-positional-arguments, + unbalanced-tuple-unpacking, + possibly-used-before-assignment, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -244,7 +248,14 @@ good-names=i, df, Run, z3, - _ + _, + x, + y, + z, + e, + ax, + fig, + id # Good variable names regexes, separated by a comma. If names match any regex, # they will always be accepted @@ -338,10 +349,10 @@ exclude-too-few-public-methods= ignored-parents= # Maximum number of arguments for function / method. -max-args=5 +max-args=12 # Maximum number of attributes for a class (see R0902). -max-attributes=7 +max-attributes=10 # Maximum number of boolean expressions in an if statement (see R0916). max-bool-expr=5 @@ -350,22 +361,22 @@ max-bool-expr=5 max-branches=12 # Maximum number of locals for function / method body. -max-locals=15 +max-locals=20 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of public methods for a class (see R0904). -max-public-methods=20 +max-public-methods=25 # Maximum number of return / yield for function / method body. max-returns=6 # Maximum number of statements in function / method body. -max-statements=50 +max-statements=60 # Minimum number of public methods for a class (see R0903). -min-public-methods=2 +min-public-methods=1 [EXCEPTIONS] @@ -394,7 +405,7 @@ indent-string=' ' max-line-length=120 # Maximum number of lines in a module. -max-module-lines=1000 +max-module-lines=1500 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. @@ -538,7 +549,11 @@ contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=deap.creator.*, + deap.tools.*, + networkx.graphml, + networkx.*, + deap.base.Toolbox.*, # Tells whether to warn about missing members when the owner of the attribute # is inferred to be None. @@ -561,7 +576,7 @@ ignored-checks-for-mixins=no-member, # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace,deap.creator,deap.base.Toolbox # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. @@ -613,3 +628,6 @@ init-import=no # List of qualified module names which can have objects that can redefine # builtins. redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + +[causal_testing.specification.causal_dag] +disable=no-member diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 2f7254ef..53cec224 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.13" # Build documentation in the docs/ directory with Sphinx sphinx: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..ceaf96a0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,92 @@ +# How to Contribute + +### Questions +Please ask any questions about the Causal Testing Framework or surrounding concepts on the +[discussions board](https://github.com/CITCOM-project/CausalTestingFramework/discussions). Before opening a new +discussion, please see whether a relevant one already exists - someone may have answered your question already. + +### Reporting Bugs and Making Suggestions +Upon identifying any bugs or features that could be improved, please open an +[issue](https://github.com/CITCOM-project/CausalTestingFramework/issues) and label with bug or feature suggestion. Every issue +should clearly explain the bug or feature to be improved and, where necessary, instructions to replicate. We also +provide templates for common scenarios when creating an issue. + +### Contributing to the Codebase +To contribute to our work, please ensure the following: + +1. [Fork the repository](https://help.github.com/articles/fork-a-repo/) into your own GitHub account, and [clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) it to your local machine. +2. [Create a new branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) in your forked repository. Give this branch an appropriate name, and create commits that describe the changes. +3. [Push your changes](https://docs.github.com/en/get-started/using-git/pushing-commits-to-a-remote-repository) to your new branch in your remote fork, compare with `CausalTestingFramework/main`, and ensure any conflicts are resolved. +4. Create a draft [pull request](https://docs.github.com/en/get-started/quickstart/hello-world#opening-a-pull-request) from your branch, and ensure you have [linked](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls) it to any relevant issues in your description. + +### Continuous Integration (CI) and Code Quality +Our CI/CD tests include: + - Build: Install the necessary Python and runtime dependencies. + - Linting: [pylint](https://pypi.org/project/pylint/) is employed for our code linter and analyser. + - Testing: We use the [unittest]() module to develop our tests and the [pytest](https://pytest.org/en/latest/) framework as our test discovery. + - Formatting: We use [black](https://pypi.org/project/black/) for our code formatting. + +To find the other (optional) developer dependencies, please check `pyproject.toml`. + +### Pre-commit Hooks +We use [pre-commit](https://pre-commit.com/) to automatically run code quality checks before each commit. This ensures consistent code style and catches issues early. + +Automated checks include: + +- Trailing whitespace removal. +- End-of-file fixing. +- YAML and TOML validation. +- Black formatting. +- isort import sorting. +- Pylint code analysis. + +To use pre-commit: +```bash +# Install pre-commit hooks (one-time setup of .pre-commit-config.yaml) +pre-commit install + +# Manually run hooks on all files (optional) +pre-commit run --all-files +``` + +### Coding Style +In the Causal Testing Framework, we aim to provide highly reusable and easily maintainable packages. To this end, +we ask contributors to stick to the following guidelines: +1. Make small and frequent changes rather than verbose and infrequent changes. +2. Favour readable and informative variable, method, and class names over concise names. +3. Use logging instead of print. +4. For every method and class, include detailed docstrings following the + [reStructuredText/Sphinx](https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html) guidelines. +5. Add credit and license information where existing code has been used (in the method or class docstring). +6. If a method implements a specific algorithm/technique from a paper, add a citation to the docstring. +7. Use [variable](https://www.python.org/dev/peps/pep-0008/#variable-annotations) and [function](https://www.python.org/dev/peps/pep-0008/#function-annotations) + annotations. +8. All methods should be thoroughly tested with PyTest (see [Testing]() below). +9. Code formatting and linting is handled automatically by pre-commit hooks (see above). + +### Manual Code Quality Checks (Optional) +While pre-commit handles most formatting automatically, you can run these commands manually if needed: + +```bash +# Format code +black causal_testing + +# Sort imports +isort causal_testing + +# Run linter +pylint causal_testing + +# Run tests +pytest +``` + +### Compatibility Testing Across Python Versions with tox + +For compatibility testing Python versions, we use [tox](https://pypi.org/project/tox/) to automate +testing across all supported Python versions (3.10, 3.11, 3.12, and 3.13): + +- Install tox: `pip install tox`. +- Test all versions: `tox` (runs tests on all Python versions + linting in the root folder). +- Test specific version: `tox -e py313` (or py310, py311, py312). +- Quick iteration: Use `pytest` directly for fast testing during development. diff --git a/README.md b/README.md index 5d31817c..8c4c0634 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,12 @@ [![DOI](https://joss.theoj.org/papers/10.21105/joss.07739/status.svg)](https://doi.org/10.21105/joss.07739) [![DOI](https://img.shields.io/badge/doi-10.26180/5c6e1160b8d8a-blue.svg?style=flat&labelColor=whitesmoke&logo=data%3Aimage%2Fpng%3Bbase64%2CiVBORw0KGgoAAAANSUhEUgAAAB8AAAAfCAYAAAAfrhY5AAAJsklEQVR42qWXd1DTaRrHf%2BiB2Hdt5zhrAUKz4IKEYu9IGiGFFJJQ0gkJCAKiWFDWBRdFhCQUF3UVdeVcRQEBxUI3yY9iEnQHb3bdW1fPubnyz%2F11M7lvEHfOQee2ZOYzPyDv%2B3yf9%2Fk95YX4fx%2BltfUt08GcFEuPR4U9hDDZ%2FVngIlhb%2FSiI6InkTgLzgDcgfvtnovhH4BzoVlrbwr55QnhCtBW4QHXnFrZbPBaQoBh4%2FSYH2EnpBEtqcDMVzB93wA%2F8AFwa23XFGcc8CkT3mxz%2BfXWtq9T9IQlLIXYEuHojudb%2BCM7Hgdq8ydi%2FAHiBXyY%2BLjwFlAEnS6Jnar%2FvnQVhvdzasad0eKvWZKe8hvDB2ofLZ%2FZEcWsh%2BhyIuyO5Bxs2iZIE4nRv7NWAb0EO8AC%2FWPxjYAWuOEX2MSXZVgPxzmRL3xKz3ScGpx6p6QnOx4mDIFqO0w6Q4fEhO5IzwxlSwyD2FYHzwAW%2BAZ4fEsf74gCumykwNHskLM7taQxLYjjIyy8MUtraGhTWdkfhkFJqtvuVl%2F9l2ZquDfEyrH8B0W06nnpH3JtIyRGpH1iJ6SfxDIHjRXHJmdQjLpfHeN54gnfFx4W9QRnovx%2FN20aXZeTD2J84hn3%2BqoF2Tqr14VqTPUCIcP%2B5%2Fly4qC%2BUL3sYxSvNj1NwsVYPsWdMUfomsdkYm3Tj0nbV0N1wRKwFe1MgKACDIBdMAhPE%2FwicwNWxll8Ag40w%2BFfhibJkGHmutjYeQ8gVlaN%2BjO51nDysa9TwNUFMqaGbKdRJZFfOJSp6mkRKsv0rRIpEVWjAvyFkxNOEpwvcAVPfEe%2Bl8ojeNTx3nXLBcWRrYGxSRjDEk0VlpxYrbe1ZmaQ5xuT0u3r%2B2qe5j0J5uytiZPGsRL2Jm32AldpxPUNJ3jmmsN4x62z1cXrbedXBQf2yvIFCeZrtyicZZG2U2nrrBJzYorI2EXLrvTfCSB43s41PKEvbZDEfQby6L4JTj%2FfIwam%2B4%2BwucBu%2BDgNK05Nle1rSt9HvR%2FKPC4U6LTfvUIaip1mjIa8fPzykii23h2eanT57zQ7fsyYH5QjywwlooAUcAdOh5QumgTHx6aAO7%2FL52eaQNEShrxfhL6albEDmfhGflrsT4tps8gTHNOJbeDeBlt0WJWDHSgxs6cW6lQqyg1FpD5ZVDfhn1HYFF1y4Eiaqa18pQf3zzYMBhcanlBjYfgWNayAf%2FASOgklu8bmgD7hADrk4cRlOL7NSOewEcbqSmaivT33QuFdHXj5sdvjlN5yMDrAECmdgDWG2L8P%2BAKLs9ZLZ7dJda%2BB4Xl84t7QvnKfvpXJv9obz2KgK8dXyqISyV0sXGZ0U47hOA%2FAiigbEMECJxC9aoKp86re5O5prxOlHkcksutSQJzxZRlPZmrOKhsQBF5zEZKybUC0vVjG8PqOnhOq46qyDTDnj5gZBriWCk4DvXrudQnXQmnXblebhAC2cCB6zIbM4PYgGl0elPSgIf3iFEA21aLdHYLHUQuVkpgi02SxFdrG862Y8ymYGMvXDzUmiX8DS5vKZyZlGmsSgQqfLub5RyLNS4zfDiZc9Edzh%2FtCE%2BX8j9k%2FqWB071rcZyMImne1SLkL4GRw4UPHMV3jjwEYpPG5uW5fAEot0aTSJnsGAwHJi2nvF1Y5OIqWziVCQd5NT7t6Q8guOSpgS%2Fa1dSRn8JGGaCD3BPXDyQRG4Bqhu8XrgAp0yy8DMSvvyVXDgJcJTcr1wQ2BvFKf65jqhvmxXUuDpGBlRvV36XvGjQzLi8KAKT2lYOnmxQPGorURSV0NhyTIuIyqOmKTMhQ%2BieEsgOgpc4KBbfDM4B3SIgFljvfHF6cef7qpyLBXAiQcXvg5l3Iunp%2FWv4dH6qFziO%2BL9PbrimQ9RY6MQphEfGUpOmma7KkGzuS8sPUFnCtIYcKCaI9EXo4HlQLgGrBjbiK5EqMj2AKWt9QWcIFMtnVvQVDQV9lXJJqdPVtUQpbh6gCI2Ov1nvZts7yYdsnvRgxiWFOtNJcOMVLn1vgptVi6qrNiFOfEjHCDB3J%2BHDLqUB77YgQGwX%2Fb1eYna3hGKdlqJKIyiE4nSbV8VFgxmxR4b5mVkkeUhMgs5YTi4ja2XZ009xJRHdkfwMi%2BfocaancuO7h%2FMlcLOa0V%2FSw6Dq47CumRQAKhgbOP8t%2BMTjuxjJGhXCY6XpmDDFqWlVYbQ1aDJ5Cptdw4oLbf3Ck%2BdWkVP0LpH7s9XLPXI%2FQX8ws%2Bj2In63IcRvOOo%2BTTjiN%2BlssfRsanW%2B3REVKoavBOAPTXABW4AL7e4NygHdpAKBscmlDh9Jysp4wxbnUNna3L3xBvyE1jyrGIkUHaqQMuxhHElV6oj1picvgL1QEuS5PyZTEaivqh5vUCKJqOuIgPFGESns8kyFk7%2FDxyima3cYxi%2FYOQCj%2F%2B9Ms2Ll%2Bhn4FmKnl7JkGXQGDKDAz9rUGL1TIlBpuJr9Be2JjK6qPzyDg495UxXYF7JY1qKimw9jWjF0iV6DRIqE%2B%2FeWG0J2ofmZTk0mLYVd4GLiFCOoKR0Cg727tWq981InYynvCuKW43aXgEjofVbxIqrm0VL76zlH3gQzWP3R3Bv9oXxclrlO7VVtgBRpSP4hMFWJ8BrUSBCJXC07l40X4jWuvtc42ofNCxtlX2JH6bdeojXgTh5TxOBKEyY5wvBE%2BACh8BtOPNPkApjoxi5h%2B%2FFMQQNpWvZaMH7MKFu5Ax8HoCQdmGkJrtnOiLHwD3uS5y8%2F2xTSDrE%2F4PT1yqtt6vGe8ldMBVMEPd6KwqiYECHDlfbvzphcWP%2BJiZuL5swoWQYlS%2Br7Yu5mNUiGD2retxBi9fl6RDGn4Ti9B1oyYy%2BMP5G87D%2FCpRlvdnuy0PY6RC8BzTA40NXqckQ9TaOUDywkYsudxJzPgyDoAWn%2BB6nEFbaVxxC6UXjJiuDkW9TWq7uRBOJocky9iMfUhGpv%2FdQuVVIuGjYqACbXf8aa%2BPeYNIHZsM7l4s5gAQuUAzRUoT51hnH3EWofXf2vkD5HJJ33vwE%2FaEWp36GHr6GpMaH4AAPuqM5eabH%2FhfG9zcCz4nN6cPinuAw6IHwtvyB%2FdO1toZciBaPh25U0ducR2PI3Zl7mokyLWKkSnEDOg1x5fCsJE9EKhH7HwFNhWMGMS7%2BqxyYsbHHRUDUH4I%2FAheQY7wujJNnFUH4KdCju83riuQeHU9WEqNzjsJFuF%2FdTDAZ%2FK7%2F1WaAU%2BAWymT59pVMT4g2AxcwNa0XEBDdBDpAPvgDIH73R25teeuAF5ime2Ul0OUIiG4GpSAEJeYW9wDTf43wfwHgHLKJoPznkwAAAABJRU5ErkJggg%3D%3D)](http://doi.org/10.15131/shef.data.24427516.v2) -Causal testing is a causal inference-driven framework for functional black-box testing. This framework utilises -graphical causal inference (CI) techniques for the specification and functional testing of software from a black-box -perspective. In this framework, we use causal directed acyclic graphs (DAGs) to express the anticipated cause-effect -relationships amongst the inputs and outputs of the system-under-test and the supporting mathematical framework to -design statistical procedures capable of making causal inferences. Each causal test case focuses on the causal effect of -an intervention made to the system-under test. That is, a prescribed change to the input configuration of the -system-under-test that is expected to cause a change to some output(s). +The Causal Testing Framework is composed of a :term:`causal inference`-driven architecture designed for functional black-box testing. +It leverages graphical causal inference (CI) techniques to specify and evaluate software behaviour from a black-box perspective. +Within this framework, causal directed acyclic graphs (DAGs) are used to represent the expected cause–effect relationships between +the inputs and outputs of the system under test, supported by mathematical foundations for designing statistical procedures that +enable causal inference. Each causal test case targets the causal effect of a specific intervention on the system under test--that is, +a deliberate modification to the input configuration expected to produce a corresponding change in one or more outputs. ![Causal Testing Workflow](images/schematic-dark.png#gh-dark-mode-only) ![Causal Testing Workflow](images/schematic.png#gh-light-mode-only) @@ -26,9 +25,7 @@ system-under-test that is expected to cause a change to some output(s). ## Installation ### Requirements -- Python 3.10, 3.11 and 3.12 - -- Microsoft Visual C++ 14.0+ (Windows only). +- Python 3.10, 3.11, 3.12 and 3.13 To install the latest stable release of the Causal Testing Framework: @@ -52,10 +49,10 @@ git checkout tags/ -b pip install . # For core API only pip install -e . # For editable install, useful for development work ``` -For more information on how to use the Causal Testing Framework, please refer to our [documentation](https://causal-testing-framework.readthedocs.io/en/latest/?badge=latest). +For more information on how to use the Causal Testing Framework, please refer to our [documentation](https://causal-testing-framework.readthedocs.io/en/latest/?badge=latest). >[!NOTE] ->We recommend you use a 64 bit OS (standard in most modern machines) as we have had reports of the installation crashing on some 32 bit Debian installations. +>We recommend you use a 64-bit OS (standard in most modern machines) as we have had reports of the installation crashing on legacy 32-bit Debian systems. ## Usage >[!NOTE] @@ -86,7 +83,6 @@ The paper citation should be the Causal Testing Framework [paper](https://dl.acm and the software citation should contain the specific Figshare [DOI](https://orda.shef.ac.uk/articles/software/CITCOM_Software_Release/24427516) of the version used in your work. -
BibTeX Citations @@ -127,20 +123,6 @@ and the software citation should contain the specific Figshare [DOI](https://ord
-## How to Contribute - -To contribute to our work, please ensure the following: - -1. [Fork the repository](https://help.github.com/articles/fork-a-repo/) into your own GitHub account, and [clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) it to your local machine. -2. [Create a new branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) in your forked repository. Give this branch an appropriate name, and create commits that describe the changes. -3. [Push your changes](https://docs.github.com/en/get-started/using-git/pushing-commits-to-a-remote-repository) to your new branch in your remote fork, compare with `CausalTestingFramework/main`, and ensure any conflicts are resolved. -4. Create a draft [pull request](https://docs.github.com/en/get-started/quickstart/hello-world#opening-a-pull-request) from your branch, and ensure you have [linked](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls) it to any relevant issues in your description. - -We use the [unittest]() module to develop our tests and the [pytest](https://pytest.org/en/latest/) framework as our test discovery, [pylint](https://pypi.org/project/pylint/) for our code analyser, and [black](https://pypi.org/project/black/) for our code formatting. -To find the other (optional) developer dependencies, please check `pyproject.toml`. - - - ## Acknowledgements The Causal Testing Framework is supported by the UK's Engineering and Physical Sciences Research Council (EPSRC), diff --git a/causal_testing/__main__.py b/causal_testing/__main__.py index bfe3fd1f..77bb11a2 100644 --- a/causal_testing/__main__.py +++ b/causal_testing/__main__.py @@ -1,12 +1,14 @@ """This module contains the main entrypoint functionality to the Causal Testing Framework.""" -import logging -import tempfile import json +import logging import os +import tempfile +from pathlib import Path from causal_testing.testing.metamorphic_relation import generate_causal_tests -from .main import setup_logging, parse_args, CausalTestingPaths, CausalTestingFramework, Command + +from .main import CausalTestingFramework, CausalTestingPaths, Command, parse_args, setup_logging def main() -> None: @@ -68,7 +70,9 @@ def main() -> None: with open(file_path, "r", encoding="utf-8") as f: all_results.extend(json.load(f)) - # Save the final stitched results to your desired location + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w", encoding="utf-8") as f: json.dump(all_results, f, indent=4) else: diff --git a/causal_testing/estimation/abstract_regression_estimator.py b/causal_testing/estimation/abstract_regression_estimator.py index 53597a0f..e0bdb8b9 100644 --- a/causal_testing/estimation/abstract_regression_estimator.py +++ b/causal_testing/estimation/abstract_regression_estimator.py @@ -1,15 +1,15 @@ """This module contains the RegressionEstimator, which is an abstract class for concrete regression estimators.""" import logging -from typing import Any from abc import abstractmethod +from typing import Any import pandas as pd -from statsmodels.regression.linear_model import RegressionResultsWrapper from patsy import dmatrix # pylint: disable = no-name-in-module +from statsmodels.regression.linear_model import RegressionResultsWrapper -from causal_testing.specification.variable import Variable from causal_testing.estimation.abstract_estimator import Estimator +from causal_testing.specification.variable import Variable from causal_testing.testing.base_test_case import BaseTestCase logger = logging.getLogger(__name__) diff --git a/causal_testing/estimation/cubic_spline_estimator.py b/causal_testing/estimation/cubic_spline_estimator.py index 2fec6496..1b7d8607 100644 --- a/causal_testing/estimation/cubic_spline_estimator.py +++ b/causal_testing/estimation/cubic_spline_estimator.py @@ -6,9 +6,9 @@ import pandas as pd -from causal_testing.specification.variable import Variable -from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.estimation.effect_estimate import EffectEstimate +from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator +from causal_testing.specification.variable import Variable from causal_testing.testing.base_test_case import BaseTestCase logger = logging.getLogger(__name__) diff --git a/causal_testing/estimation/effect_estimate.py b/causal_testing/estimation/effect_estimate.py index ad89abd3..865ecd61 100644 --- a/causal_testing/estimation/effect_estimate.py +++ b/causal_testing/estimation/effect_estimate.py @@ -3,6 +3,7 @@ """ from dataclasses import dataclass + import pandas as pd diff --git a/causal_testing/estimation/experimental_estimator.py b/causal_testing/estimation/experimental_estimator.py index e94959d0..f8ed77a8 100644 --- a/causal_testing/estimation/experimental_estimator.py +++ b/causal_testing/estimation/experimental_estimator.py @@ -1,7 +1,8 @@ """This module contains the ExperimentalEstimator class for directly interacting with the system under test.""" -from typing import Any from abc import abstractmethod +from typing import Any + import pandas as pd from causal_testing.estimation.abstract_estimator import Estimator diff --git a/causal_testing/estimation/genetic_programming_regression_fitter.py b/causal_testing/estimation/genetic_programming_regression_fitter.py index b8a52e8d..782dbc68 100644 --- a/causal_testing/estimation/genetic_programming_regression_fitter.py +++ b/causal_testing/estimation/genetic_programming_regression_fitter.py @@ -4,20 +4,18 @@ """ import copy +import random from inspect import isclass from operator import add, mul -import random -import patsy import numpy as np import pandas as pd -import statsmodels.formula.api as smf +import patsy import statsmodels +import statsmodels.formula.api as smf import sympy - -from deap import base, creator, tools, gp - -from numpy import power, log +from deap import base, creator, gp, tools +from numpy import log, power def reciprocal(x: float) -> float: diff --git a/causal_testing/estimation/instrumental_variable_estimator.py b/causal_testing/estimation/instrumental_variable_estimator.py index 2b5eaf2d..c7775363 100644 --- a/causal_testing/estimation/instrumental_variable_estimator.py +++ b/causal_testing/estimation/instrumental_variable_estimator.py @@ -3,6 +3,7 @@ import logging from math import ceil + import pandas as pd import statsmodels.api as sm diff --git a/causal_testing/estimation/ipcw_estimator.py b/causal_testing/estimation/ipcw_estimator.py index ab038df1..1e4040a0 100644 --- a/causal_testing/estimation/ipcw_estimator.py +++ b/causal_testing/estimation/ipcw_estimator.py @@ -4,7 +4,6 @@ from typing import Any from uuid import uuid4 - import numpy as np import pandas as pd import statsmodels.formula.api as smf @@ -12,8 +11,8 @@ from causal_testing.estimation.abstract_estimator import Estimator from causal_testing.estimation.effect_estimate import EffectEstimate -from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.specification.variable import Variable +from causal_testing.testing.base_test_case import BaseTestCase logger = logging.getLogger(__name__) diff --git a/causal_testing/estimation/linear_regression_estimator.py b/causal_testing/estimation/linear_regression_estimator.py index fcc1d8ae..7869c4de 100644 --- a/causal_testing/estimation/linear_regression_estimator.py +++ b/causal_testing/estimation/linear_regression_estimator.py @@ -5,12 +5,12 @@ import pandas as pd import statsmodels.formula.api as smf -from patsy import dmatrix, ModelDesc # pylint: disable = no-name-in-module +from patsy import ModelDesc, dmatrix # pylint: disable = no-name-in-module -from causal_testing.specification.variable import Variable -from causal_testing.estimation.genetic_programming_regression_fitter import GP from causal_testing.estimation.abstract_regression_estimator import RegressionEstimator from causal_testing.estimation.effect_estimate import EffectEstimate +from causal_testing.estimation.genetic_programming_regression_fitter import GP +from causal_testing.specification.variable import Variable from causal_testing.testing.base_test_case import BaseTestCase logger = logging.getLogger(__name__) diff --git a/causal_testing/estimation/logistic_regression_estimator.py b/causal_testing/estimation/logistic_regression_estimator.py index f4f14725..bd95b35f 100644 --- a/causal_testing/estimation/logistic_regression_estimator.py +++ b/causal_testing/estimation/logistic_regression_estimator.py @@ -6,8 +6,8 @@ import pandas as pd import statsmodels.formula.api as smf -from causal_testing.estimation.effect_estimate import EffectEstimate from causal_testing.estimation.abstract_regression_estimator import RegressionEstimator +from causal_testing.estimation.effect_estimate import EffectEstimate logger = logging.getLogger(__name__) diff --git a/causal_testing/main.py b/causal_testing/main.py index 3cbbf493..be2b0dc6 100644 --- a/causal_testing/main.py +++ b/causal_testing/main.py @@ -3,25 +3,25 @@ import argparse import json import logging -from enum import Enum from dataclasses import dataclass +from enum import Enum from pathlib import Path -from typing import Dict, Any, Optional, List, Union, Sequence +from typing import Any, Dict, List, Optional, Sequence, Union -from tqdm import tqdm -import pandas as pd import numpy as np +import pandas as pd +from tqdm import tqdm +from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator +from causal_testing.estimation.logistic_regression_estimator import LogisticRegressionEstimator from causal_testing.specification.causal_dag import CausalDAG +from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.specification.scenario import Scenario from causal_testing.specification.variable import Input, Output -from causal_testing.specification.causal_specification import CausalSpecification -from causal_testing.testing.causal_test_case import CausalTestCase from causal_testing.testing.base_test_case import BaseTestCase -from causal_testing.testing.causal_effect import NoEffect, SomeEffect, Positive, Negative +from causal_testing.testing.causal_effect import Negative, NoEffect, Positive, SomeEffect +from causal_testing.testing.causal_test_case import CausalTestCase from causal_testing.testing.causal_test_result import CausalTestResult -from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator -from causal_testing.estimation.logistic_regression_estimator import LogisticRegressionEstimator logger = logging.getLogger(__name__) @@ -418,6 +418,9 @@ def save_results(self, results: List[CausalTestResult], output_path: str = None) output_path = self.paths.output_path logger.info(f"Saving results to {output_path}") + # Create parent directory if it doesn't exist + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + # Load original test configs to preserve test metadata with open(self.paths.test_config_path, "r", encoding="utf-8") as f: test_configs = json.load(f) diff --git a/causal_testing/specification/causal_dag.py b/causal_testing/specification/causal_dag.py index 42327051..e2692f9b 100644 --- a/causal_testing/specification/causal_dag.py +++ b/causal_testing/specification/causal_dag.py @@ -4,7 +4,7 @@ import logging from itertools import combinations -from typing import Union, Set, Generator +from typing import Generator, Set, Union import networkx as nx @@ -13,7 +13,6 @@ from .scenario import Scenario from .variable import Output - Node = Union[str, int] # Node type hint: A node is a string or an int logger = logging.getLogger(__name__) diff --git a/causal_testing/specification/variable.py b/causal_testing/specification/variable.py index 3642ea83..4fb27b13 100644 --- a/causal_testing/specification/variable.py +++ b/causal_testing/specification/variable.py @@ -1,6 +1,7 @@ """This module contains the Variable abstract class, as well as its concrete extensions: Input, Output and Meta.""" from __future__ import annotations + from abc import ABC from collections.abc import Callable from typing import TypeVar diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 8a97d2cf..1a3911dc 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -3,10 +3,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Callable + import pandas as pd + +from causal_testing.estimation.cubic_spline_estimator import CubicSplineRegressionEstimator from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.base_test_case import BaseTestCase -from causal_testing.estimation.cubic_spline_estimator import CubicSplineRegressionEstimator @dataclass diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 19e2e8f0..3b2c14ba 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -3,10 +3,11 @@ # Fitness functions are required to be iteratively defined, including all variables within. from operator import itemgetter + from pygad import GA -from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.estimation.cubic_spline_estimator import CubicSplineRegressionEstimator +from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm diff --git a/causal_testing/testing/base_test_case.py b/causal_testing/testing/base_test_case.py index 8838b932..aca1b376 100644 --- a/causal_testing/testing/base_test_case.py +++ b/causal_testing/testing/base_test_case.py @@ -1,6 +1,7 @@ """This module contains the BaseTestCase dataclass, which stores the information required for identification""" from dataclasses import dataclass + from causal_testing.specification.variable import Variable from causal_testing.testing.effect import Effect diff --git a/causal_testing/testing/causal_effect.py b/causal_testing/testing/causal_effect.py index 9d6a4c09..8ca60154 100644 --- a/causal_testing/testing/causal_effect.py +++ b/causal_testing/testing/causal_effect.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterable + import numpy as np from causal_testing.testing.causal_test_result import CausalTestResult diff --git a/causal_testing/testing/causal_test_adequacy.py b/causal_testing/testing/causal_test_adequacy.py index fcbc8f00..f64d1dea 100644 --- a/causal_testing/testing/causal_test_adequacy.py +++ b/causal_testing/testing/causal_test_adequacy.py @@ -3,14 +3,15 @@ """ import logging -from itertools import combinations from copy import deepcopy +from itertools import combinations + import pandas as pd -from numpy.linalg import LinAlgError from lifelines.exceptions import ConvergenceError +from numpy.linalg import LinAlgError -from causal_testing.specification.causal_dag import CausalDAG from causal_testing.estimation.abstract_estimator import Estimator +from causal_testing.specification.causal_dag import CausalDAG from causal_testing.testing.causal_test_case import CausalTestCase logger = logging.getLogger(__name__) diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py index 08895b76..0b75ea12 100644 --- a/causal_testing/testing/causal_test_case.py +++ b/causal_testing/testing/causal_test_case.py @@ -2,12 +2,11 @@ import logging -from causal_testing.testing.causal_effect import CausalEffect -from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.estimation.abstract_estimator import Estimator +from causal_testing.testing.base_test_case import BaseTestCase +from causal_testing.testing.causal_effect import CausalEffect from causal_testing.testing.causal_test_result import CausalTestResult - logger = logging.getLogger(__name__) diff --git a/causal_testing/testing/metamorphic_relation.py b/causal_testing/testing/metamorphic_relation.py index 5d20e3e2..09eaac86 100644 --- a/causal_testing/testing/metamorphic_relation.py +++ b/causal_testing/testing/metamorphic_relation.py @@ -3,12 +3,12 @@ defined in our ICST paper [https://eprints.whiterose.ac.uk/195317/]. """ +import json +import logging from dataclasses import dataclass -from typing import Iterable from itertools import combinations -import logging -import json from multiprocessing import Pool +from typing import Iterable import networkx as nx diff --git a/causal_testing/utils/validation.py b/causal_testing/utils/validation.py index 63df5dc3..2fd0bb66 100644 --- a/causal_testing/utils/validation.py +++ b/causal_testing/utils/validation.py @@ -1,6 +1,7 @@ """This module contains the CausalValidator class for performing Quantitive Bias Analysis techniques""" import math + import numpy as np from scipy.stats import t from statsmodels.regression.linear_model import RegressionResultsWrapper diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt index 3f2d7e9a..25212ac5 100644 --- a/docs/source/requirements.txt +++ b/docs/source/requirements.txt @@ -1,4 +1,8 @@ Sphinx==7.2.6 myst-parser==2.0.0 sphinx-autoapi==3.0.0 -sphinx-rtd-theme==2.0.0 \ No newline at end of file +sphinx-rtd-theme==2.0.0 +nbsphinx==0.9.7 +ipykernel==6.30.1 +pandoc==2.4 +sphinxcontrib-mermaid==1.0.0 diff --git a/pyproject.toml b/pyproject.toml index 9a845844..63b1f8bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,16 +11,16 @@ name = "causal_testing_framework" authors = [{ name = "The CITCOM team" }] description = "A framework for causal testing using causal directed acyclic graphs." readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.10,<3.14" license = { text = "MIT" } keywords = ["causal inference", "verification"] dependencies = [ - "lifelines~=0.29.0", + "lifelines~=0.30.0", "networkx>=3.4,<3.5", - "numpy~=1.26", + "numpy>=1.2.6, <=2.2.0", "pandas>=2.1", "scikit_learn~=1.4", - "scipy>=1.12.0,<1.14.0", + "scipy>=1.12.0,<=1.16.2", "statsmodels~=0.14", "tabulate~=0.9", "pydot~=2.0", @@ -29,22 +29,27 @@ dependencies = [ "sympy~=1.13.1", "pyarrow~=19.0.1", "fastparquet~=2024.11.0", + "tqdm~=4.67.1" ] dynamic = ["version"] [project.optional-dependencies] dev = [ - "autopep8", "isort", "pytest", "pylint", "black", "autoapi", "myst-parser", + "sphinx", "sphinx-autoapi", "sphinx_rtd_theme", - "tqdm", + "nbsphinx", + "ipykernel", + "pandoc", + "pre-commit" ] + test = [ "covasim~=3.0.7", ] @@ -64,14 +69,34 @@ find = {} line-length = 120 target-version = ["py310"] -[tool.autopep8] -max_line_length = 120 -in-place = true -recursive = true -aggressive = 3 +[tool.isort] +profile = "black" +line_length = 120 [tool.pytest.ini_options] minversion = "6.0" python_files=[ "test_*.py", "example_*.py"] + +[tool.tox] +requires = ["tox>=4.19"] +env_list = ["3.10", "3.11", "3.12", "3.13"] +skip_missing_interpreters = false # fail if devs don’t have all required Python versions + +# Base configuration for all test environments +[tool.tox.env_run_base] +description = "Run pytest under {base_python}" +extras = ["dev", "test"] +deps = ["pytest"] +commands = [["pytest"]] + +# Automatically test for type-checking (TODO: enable type checking in env_list in the future) +[tool.tox.env.type] +description = "Run type checks with mypy on the codebase" +deps = [ + "mypy==1.18.2", + "types-cachetools>=5.5.0.20240820", + "types-chardet>=5.0.4.6" +] +commands = [["mypy", "causal_testing"], ["mypy", "tests"]]