Skip to content

Commit

Permalink
Backport PR #1314: Test against mindeps (#1374)
Browse files Browse the repository at this point in the history
Co-authored-by: Isaac Virshup <ivirshup@gmail.com>
  • Loading branch information
meeseeksmachine and ivirshup committed Feb 16, 2024
1 parent 4be68f4 commit ada1ba6
Show file tree
Hide file tree
Showing 8 changed files with 152 additions and 20 deletions.
32 changes: 23 additions & 9 deletions .azure-pipelines.yml
Expand Up @@ -6,7 +6,8 @@ variables:
PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
RUN_COVERAGE: no
PYTEST_ADDOPTS: --color=yes --junitxml=test-data/test-results.xml
PRERELEASE_DEPENDENCIES: no
DEPENDENCIES_VERSION: "latest" # |"pre-release" | "minimum-version"
TEST_TYPE: "standard" # | "coverage"

jobs:
- job: PyTest
Expand All @@ -17,11 +18,17 @@ jobs:
Python3.11:
python.version: "3.11"
RUN_COVERAGE: yes
TEST_TYPE: "coverage"
Python3.9:
python.version: "3.9"
PreRelease:
python.version: "3.11"
PRERELEASE_DEPENDENCIES: yes
DEPENDENCIES_VERSION: "pre-release"
TEST_TYPE: "strict-warning"
minimum_versions:
python.version: "3.9"
DEPENDENCIES_VERSION: "minimum"
TEST_TYPE: "coverage"
steps:
- task: UsePythonVersion@0
inputs:
Expand All @@ -41,13 +48,20 @@ jobs:
python -m pip install --upgrade pip wheel
pip install .[dev,test]
displayName: "Install dependencies"
condition: eq(variables['PRERELEASE_DEPENDENCIES'], 'no')
condition: eq(variables['DEPENDENCIES_VERSION'], 'latest')
- script: |
python -m pip install pip wheel tomli packaging pytest-cov
pip install `python3 ci/scripts/min-deps.py pyproject.toml --extra dev test`
pip install --no-deps .
displayName: "Install minimum dependencies"
condition: eq(variables['DEPENDENCIES_VERSION'], 'minimum')
- script: |
python -m pip install --pre --upgrade pip wheel
pip install --pre .[dev,test]
displayName: "Install dependencies release candidates"
condition: eq(variables['PRERELEASE_DEPENDENCIES'], 'yes')
condition: eq(variables['DEPENDENCIES_VERSION'], 'pre-release')
- script: |
pip list
Expand All @@ -56,23 +70,23 @@ jobs:
- script: |
pytest
displayName: "PyTest"
condition: and(eq(variables['RUN_COVERAGE'], 'no'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no'))
condition: eq(variables['TEST_TYPE'], 'standard')
- script: |
pytest --cov --cov-report=xml --cov-context=test
displayName: "PyTest (coverage)"
condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no'))
condition: eq(variables['TEST_TYPE'], 'coverage')
- script: |
pytest --strict-warnings
displayName: "PyTest (treat warnings as errors)"
condition: and(eq(variables['RUN_COVERAGE'], 'no'), eq(variables['PRERELEASE_DEPENDENCIES'], 'yes'))
condition: eq(variables['TEST_TYPE'], 'strict-warning')
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: "test-data/coverage.xml"
condition: eq(variables['RUN_COVERAGE'], 'yes')
condition: eq(variables['TEST_TYPE'], 'coverage')

- task: PublishTestResults@2
condition: succeededOrFailed()
Expand All @@ -83,7 +97,7 @@ jobs:

- script: bash <(curl -s https://codecov.io/bash)
displayName: "Upload to codecov.io"
condition: eq(variables['RUN_COVERAGE'], 'yes')
condition: eq(variables['TEST_TYPE'], 'coverage')

- job: CheckBuild
pool:
Expand Down
2 changes: 1 addition & 1 deletion anndata/experimental/merge.py
Expand Up @@ -523,7 +523,7 @@ def concat_on_disk(
>>> adata = ad.read_h5ad('merged.h5ad', backed=True)
>>> adata.X
CSRDataset: backend hdf5, shape (490, 15585), data_dtype float32
>>> adata.obs['dataset'].value_counts()
>>> adata.obs['dataset'].value_counts() # doctest: +SKIP
dataset
fetal 344
b_cells 146
Expand Down
3 changes: 2 additions & 1 deletion anndata/tests/test_concatenate.py
Expand Up @@ -13,6 +13,7 @@
import pytest
from boltons.iterutils import default_exit, remap, research
from numpy import ma
from packaging.version import Version
from scipy import sparse

from anndata import AnnData, Raw, concat
Expand Down Expand Up @@ -1350,7 +1351,7 @@ def test_concat_size_0_dim(axis, join_type, merge_strategy, shape):
FutureWarning,
match=r"The behavior of DataFrame concatenation with empty or all-NA entries is deprecated",
)
if shape[axis] == 0
if shape[axis] == 0 and Version(pd.__version__) >= Version("2.1")
else nullcontext()
)
with ctx_concat_empty:
Expand Down
9 changes: 9 additions & 0 deletions anndata/tests/test_io_warnings.py
Expand Up @@ -5,7 +5,9 @@
from importlib.util import find_spec
from pathlib import Path

import h5py
import pytest
from packaging.version import Version

import anndata as ad
from anndata.tests.helpers import gen_adata
Expand Down Expand Up @@ -43,6 +45,13 @@ def test_old_format_warning_not_thrown(tmp_path):

with warnings.catch_warnings(record=True) as record:
warnings.simplefilter("always", ad.OldFormatWarning)
if Version(h5py.__version__) < Version("3.2"):
# https://github.com/h5py/h5py/issues/1808
warnings.filterwarnings(
"ignore",
r"Passing None into shape arguments as an alias for \(\) is deprecated\.",
category=DeprecationWarning,
)

ad.read_h5ad(pth)

Expand Down
99 changes: 99 additions & 0 deletions ci/scripts/min-deps.py
@@ -0,0 +1,99 @@
#!python3
from __future__ import annotations

import argparse
import sys
from collections import deque
from pathlib import Path
from typing import TYPE_CHECKING

if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib

from packaging.requirements import Requirement
from packaging.version import Version

if TYPE_CHECKING:
from collections.abc import Generator, Iterable


def min_dep(req: Requirement) -> Requirement:
"""
Given a requirement, return the minimum version specifier.
Example
-------
>>> min_dep(Requirement("numpy>=1.0"))
"numpy==1.0"
"""
req_name = req.name
if req.extras:
req_name = f"{req_name}[{','.join(req.extras)}]"

if not req.specifier:
return Requirement(req_name)

min_version = Version("0.0.0.a1")
for spec in req.specifier:
if spec.operator in [">", ">=", "~="]:
min_version = max(min_version, Version(spec.version))
elif spec.operator == "==":
min_version = Version(spec.version)

return Requirement(f"{req_name}=={min_version}.*")


def extract_min_deps(
dependencies: Iterable[Requirement], *, pyproject
) -> Generator[Requirement, None, None]:
dependencies = deque(dependencies) # We'll be mutating this
project_name = pyproject["project"]["name"]

while len(dependencies) > 0:
req = dependencies.pop()

# If we are referring to other optional dependency lists, resolve them
if req.name == project_name:
assert req.extras, f"Project included itself as dependency, without specifying extras: {req}"
for extra in req.extras:
extra_deps = pyproject["project"]["optional-dependencies"][extra]
dependencies += map(Requirement, extra_deps)
else:
yield min_dep(req)


def main():
parser = argparse.ArgumentParser(
prog="min-deps",
description="""Parse a pyproject.toml file and output a list of minimum dependencies.
Output is directly passable to `pip install`.""",
usage="pip install `python min-deps.py pyproject.toml`",
)
parser.add_argument(
"path", type=Path, help="pyproject.toml to parse minimum dependencies from"
)
parser.add_argument(
"--extras", type=str, nargs="*", default=(), help="extras to install"
)

args = parser.parse_args()

pyproject = tomllib.loads(args.path.read_text())

project_name = pyproject["project"]["name"]
deps = [
*map(Requirement, pyproject["project"]["dependencies"]),
*(Requirement(f"{project_name}[{extra}]") for extra in args.extras),
]

min_deps = extract_min_deps(deps, pyproject=pyproject)

print(" ".join(map(str, min_deps)))


if __name__ == "__main__":
main()
5 changes: 5 additions & 0 deletions docs/release-notes/0.10.5.md
Expand Up @@ -17,3 +17,8 @@

* `BaseCompressedSparseDataset`'s `indptr` is cached {pr}`1266` {user}`ilan-gold`
* Improved performance when indexing backed sparse matrices with boolean masks along their major axis {pr}`1233` {user}`ilan-gold`

```{rubric} Development
```

* `anndata`'s CI now tests against minimum versions of it's dependencies. As a result, several dependencies had their minimum required version bumped. See diff for details {pr}`1314` {user}`ivirshup`
5 changes: 5 additions & 0 deletions docs/release-notes/0.10.6.md
Expand Up @@ -12,3 +12,8 @@

```{rubric} Performance
```

```{rubric} Dev Process
```

* AnnData now tests against the minimum versions it's compatible with {pr}`1314` {user}`ivirshup`
17 changes: 8 additions & 9 deletions pyproject.toml
Expand Up @@ -36,16 +36,15 @@ classifiers = [
"Topic :: Scientific/Engineering :: Visualization",
]
dependencies = [
# pandas <1.1.1 has pandas/issues/35446
# pandas <1.4 has pandas/issues/35446
# pandas 2.1.0rc0 has pandas/issues/54622
# pandas 2.1.2 has pandas/issues/52927
"pandas >=1.1.1, !=2.1.0rc0, !=2.1.2",
"numpy>=1.16.5", # required by pandas 1.x
"scipy>1.4",
"h5py>=3",
"pandas >=1.4, !=2.1.0rc0, !=2.1.2",
"numpy>=1.23",
"scipy>1.8",
"h5py>=3.1",
"exceptiongroup; python_version<'3.11'",
"natsort",
"packaging>=20",
"packaging>=20.0",
"array_api_compat",
]
dynamic = ["version"]
Expand Down Expand Up @@ -81,7 +80,7 @@ doc = [
]
test = [
"loompy>=3.0.5",
"pytest >=6.0",
"pytest>=7.3",
"pytest-cov>=2.10",
"zarr",
"matplotlib",
Expand All @@ -91,7 +90,7 @@ test = [
"boltons",
"scanpy",
"httpx", # For data downloading
"dask[array,distributed]",
"dask[array,distributed]>=2022.09.2",
"awkward>=2.3",
"pyarrow",
"pytest_memray",
Expand Down

0 comments on commit ada1ba6

Please sign in to comment.