Fix mismatches in test data between 'total_results' and actual result…

…s, and other test fixes

pyinaturalist/pagination.py

@@ -107,16 +107,16 @@ def next_page(self) -> List[ResponseResult]:
 
         # Fetch results
         response = self.request_function(**self.kwargs, per_page=self.per_page)
-        self._total_results = response.get('total_results') or len(response)
         results = response.get('results', response)
+        self._total_results = response.get('total_results', 0)
         self.results_fetched += len(results)
 
         # Set params for next request, if there are more results
         # Some endpoints (like get_observation_fields) don't return total_results
         # Also check page size, in case total_results is off (race condition, outdated index, etc.)
         if (
             (self.limit and self.results_fetched >= self.limit)
-            or self.results_fetched >= self.total_results
+            or (self.total_results and self.results_fetched >= self.total_results)
             or len(results) == 0
         ):
             self.exhausted = True
@@ -165,7 +165,7 @@ def __iter__(self) -> Iterator[ResponseResult]:
                 yield result
 
     def all(self) -> JsonResponse:  # type: ignore
-        results = super().all()
+        results = list(self)
         return {
             'results': results,
             'total_results': len(results),

test/conftest.py

@@ -46,7 +46,9 @@
 @pytest.fixture(scope='function', autouse=True)
 def patch_cached_session():
     """Disable request caching and rate-limiting during test session"""
-    with patch('pyinaturalist.api_requests.get_local_session', return_value=Session()):
+    with patch('pyinaturalist.api_requests.get_local_session', return_value=Session()), patch(
+        'pyinaturalist.client.ClientSession', Session
+    ):
         yield
 
 

test/controllers/test_observation_controller.py

@@ -15,32 +15,35 @@
     TaxonSummary,
     User,
 )
-from test.conftest import load_sample_data
 from test.sample_data import *
 
 
 def test_from_id(requests_mock):
     observation_id = 57754375
     requests_mock.get(
         f'{API_V1_BASE_URL}/observations',
-        json=load_sample_data('get_observations_node_page1.json'),
-        status_code=200,
+        [
+            {'json': SAMPLE_DATA['get_observations_node_page1'], 'status_code': 200},
+            {'json': SAMPLE_DATA['get_observations_node_page2'], 'status_code': 200},
+        ],
     )
     client = iNatClient()
-    results = client.observations.from_id(observation_id)
+    results = client.observations.from_id(observation_id).all()
 
-    assert len(results) == 1 and isinstance(results[0], Observation)
+    assert len(results) == 2 and isinstance(results[0], Observation)
     assert results[0].id == observation_id
 
 
 def test_search(requests_mock):
     requests_mock.get(
         f'{API_V1_BASE_URL}/observations',
-        json=load_sample_data('get_observations_node_page1.json'),
+        json=SAMPLE_DATA['get_observations_node_page1'],
         status_code=200,
     )
     client = iNatClient()
-    results = client.observations.search(taxon_name='Danaus plexippus', created_on='2020-08-27')
+    results = client.observations.search(
+        taxon_name='Danaus plexippus', created_on='2020-08-27'
+    ).all()
 
     assert isinstance(results[0], Observation)
     assert results[0].id == 57754375
@@ -51,7 +54,7 @@ def test_search(requests_mock):
 def test_histogram(requests_mock):
     requests_mock.get(
         f'{API_V1_BASE_URL}/observations/histogram',
-        json=load_sample_data('get_observation_histogram_day.json'),
+        json=SAMPLE_DATA['get_observation_histogram_day'],
         status_code=200,
     )
     client = iNatClient()
@@ -111,7 +114,7 @@ def test_life_list(requests_mock):
 def test_species_counts(requests_mock):
     requests_mock.get(
         f'{API_V1_BASE_URL}/observations/species_counts',
-        json=load_sample_data('get_observation_species_counts.json'),
+        json=SAMPLE_DATA['get_observation_species_counts'],
         status_code=200,
     )
     client = iNatClient()

test/controllers/test_project_controller.py

@@ -14,11 +14,11 @@ def test_from_id(requests_mock):
     project_id = 8291
     requests_mock.get(
         f'{API_V1_BASE_URL}/projects/{project_id}',
-        json={'results': [j_project_1, j_project_2]},
+        json={'results': [j_project_1, j_project_2], 'total_results': 2},
         status_code=200,
     )
     client = iNatClient()
-    results = client.projects.from_id(project_id)
+    results = client.projects.from_id(project_id).all()
     project = results[0]
     assert len(results) == 2 and isinstance(project, Project)
 
@@ -47,7 +47,7 @@ def test_search(requests_mock):
         lng=-123.08,
         radius=400,
         order_by='distance',
-    )
+    ).all()
 
     project = results[0]
     assert len(results) == 5 and isinstance(project, Project)
@@ -62,11 +62,11 @@ def test_search(requests_mock):
 def test_search__with_obs_fields(requests_mock):
     requests_mock.get(
         f'{API_V1_BASE_URL}/projects',
-        json={'results': [j_project_3_obs_fields]},
+        json={'results': [j_project_3_obs_fields], 'total_results': 1},
         status_code=200,
     )
     client = iNatClient()
-    results = client.projects.search(id=1234)
+    results = client.projects.search(id=1234).all()
     obs_field = results[0].project_observation_fields[0]
 
     assert isinstance(obs_field, ProjectObservationField)

test/controllers/test_taxon_controller.py

@@ -3,44 +3,44 @@
 from pyinaturalist.client import iNatClient
 from pyinaturalist.constants import API_V1_BASE_URL
 from pyinaturalist.models import Taxon
-from test.conftest import load_sample_data
+from test.sample_data import SAMPLE_DATA
 
 
 def test_from_id(requests_mock):
     taxon_id = 70118
     requests_mock.get(
         f'{API_V1_BASE_URL}/taxa/{taxon_id}',
-        json=load_sample_data('get_taxa_by_id.json'),
+        json=SAMPLE_DATA['get_taxa_by_id'],
         status_code=200,
     )
 
     client = iNatClient()
-    results = client.taxa.from_id(taxon_id)
+    results = client.taxa.from_id(taxon_id).all()
     assert len(results) == 1 and isinstance(results[0], Taxon)
     assert results[0].id == taxon_id
 
 
 def test_autocomplete(requests_mock):
     requests_mock.get(
         f'{API_V1_BASE_URL}/taxa/autocomplete',
-        json=load_sample_data('get_taxa_autocomplete.json'),
+        json=SAMPLE_DATA['get_taxa_autocomplete'],
         status_code=200,
     )
 
     client = iNatClient()
-    results = client.taxa.autocomplete(q='vespi')
+    results = client.taxa.autocomplete(q='vespi').all()
     assert len(results) == 10 and isinstance(results[0], Taxon)
     assert results[0].id == 52747
 
 
 def test_search(requests_mock):
     requests_mock.get(
         f'{API_V1_BASE_URL}/taxa',
-        json=load_sample_data('get_taxa.json'),
+        json=SAMPLE_DATA['get_taxa'],
         status_code=200,
     )
 
     client = iNatClient()
-    results = client.taxa.search(q='vespi', rank=['genus', 'subgenus', 'species'])
+    results = client.taxa.search(q='vespi', rank=['genus', 'subgenus', 'species']).all()
     assert len(results) == 30 and isinstance(results[0], Taxon)
     assert results[0].id == 70118

test/sample_data/get_taxa.json

@@ -1,5 +1,5 @@
 {
-  "total_results": 35,
+  "total_results": 30,
   "page": 1,
   "per_page": 30,
   "results": [

test/sample_data/get_taxa_autocomplete.json

@@ -1,5 +1,5 @@
 {
-    "total_results": 47,
+    "total_results": 10,
     "page": 1,
     "per_page": 30,
     "results": [

test/v0/test_observation_fields_v0.py

@@ -9,11 +9,9 @@
 
 def test_get_observation_fields(requests_mock):
     """get_observation_fields() work as expected (basic use)"""
-    page_2_json_response = SAMPLE_DATA['get_observation_fields_page2']
-
     requests_mock.get(
         f'{API_V0_BASE_URL}/observation_fields.json?q=sex&page=2',
-        json=page_2_json_response,
+        json=SAMPLE_DATA['get_observation_fields_page2'],
         status_code=200,
     )
 
@@ -31,19 +29,12 @@ def test_get_observation_fields__all_pages(requests_mock):
     page_2_json_response = SAMPLE_DATA['get_observation_fields_page2']
 
     requests_mock.get(
-        f'{API_V0_BASE_URL}/observation_fields.json?q=sex&page=1',
-        json=page_1_json_response,
-        status_code=200,
-    )
-    requests_mock.get(
-        f'{API_V0_BASE_URL}/observation_fields.json?q=sex&page=2',
-        json=page_2_json_response,
-        status_code=200,
-    )
-    requests_mock.get(
-        f'{API_V0_BASE_URL}/observation_fields.json?q=sex&page=3',
-        json=[],
-        status_code=200,
+        f'{API_V0_BASE_URL}/observation_fields.json',
+        [
+            {'json': page_1_json_response, 'status_code': 200},
+            {'json': page_2_json_response, 'status_code': 200},
+            {'json': [], 'status_code': 200},
+        ],
     )
     expected_results = page_1_json_response + page_2_json_response
 

test/v1/test_taxa.py

@@ -23,8 +23,7 @@ def test_get_taxa(requests_mock):
     response = get_taxa(q='vespi', rank=['genus', 'subgenus', 'species'])
     first_result = response['results'][0]
 
-    assert len(response['results']) == 30
-    assert response['total_results'] == 35
+    assert len(response['results']) == response['total_results'] == 30
     assert first_result['id'] == 70118
     assert first_result['name'] == 'Nicrophorus vespilloides'
     assert first_result['rank'] == 'species'
@@ -89,8 +88,7 @@ def test_get_taxa_autocomplete(requests_mock):
     response = get_taxa_autocomplete(q='vespi')
     first_result = response['results'][0]
 
-    assert len(response['results']) == 10
-    assert response['total_results'] == 47
+    assert len(response['results']) == response['total_results'] == 10
     assert first_result['matched_term'] == 'Vespidae'
     assert first_result['id'] == 52747
     assert first_result['name'] == 'Vespidae'