-
Notifications
You must be signed in to change notification settings - Fork 9
/
plugin.py
500 lines (393 loc) · 21.7 KB
/
plugin.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
import pickle
from collections import OrderedDict
from logging import warning
from shutil import rmtree
import pytest
import six
try:
from pathlib import Path
except ImportError:
from pathlib2 import Path # python 2
try: # python 3.5+
from typing import Union, Iterable, Mapping, Any
except ImportError:
pass
from pytest_harvest.common import HARVEST_PREFIX
from pytest_harvest.results_bags import create_results_bag_fixture
from pytest_harvest.results_session import get_session_synthesis_dct, get_persistable_session_items
from pytest_harvest.xdist_api import _is_xdist_master, _is_xdist_worker, get_xdist_worker_id
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""
We use this hook to store the test execution status for each node in the 'item' object,
so that we can use it in `get_session_synthesis_dct`
It is called because the whole package is a pytest plugin (it has a pytest entry point in setup.py)
Following the example here:
https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
:param item:
:param call:
:return:
"""
# execute all other hooks to obtain the report object
outcome = yield
rep = outcome.get_result()
# set a report attribute for each phase of a call, which can
# be "setup", "call", "teardown"
setattr(item, HARVEST_PREFIX + rep.when, rep)
# ------------- To collect benchmark results ------------
FIXTURE_STORE = OrderedDict()
"""The default fixture store, that is also available through the `fixture_store` fixture. It is recommended to access
it through `get_fixture_store(session)` so as to be xdist-compliant"""
def get_fixture_store(session_or_request):
"""
pytest-xdist-compliant way to access the default fixture store.
:param session:
:return:
"""
possibly_restore_xdist_workers_structs(session_or_request)
return FIXTURE_STORE
@pytest.fixture(scope='session') # no need for autouse=True
def fixture_store(request):
"""
A "fixture store" fixture: a dictionary where fixture instances can be saved.
By default
- all fixtures decorated with `@saved_fixture` are saved in this store.
- the `results_bag` fixture is also saved in this store.
To retrieve the contents of the store, you can:
* create a test using this fixture and make sure that it is executed after all others.
* access this fixture from a dependent fixture and read its value in the setup or teardown script.
* access this fixture from the `request` fixture using the `get_fixture_value` helper method.
* access the `FIXTURE_STORE` symbol directly
This fixture has session scope so it is unique across the whole session.
"""
return get_fixture_store(request)
results_bag = create_results_bag_fixture('fixture_store', name='results_bag')
"""
A "results bag" fixture: a dictionary where you can store anything (results, context, etc.) during your tests execution.
It offers a "much"-like api: you can access all entries using the object protocol such as in `results_bag.a = 1`.
This fixture has function-scope so a new, empty instance is injected in each test node.
There are several ways to gather all results after they have been stored.
* To get the raw stored results, use the `fixture_store` fixture: `fixture_store['results_bag']` will contain all
result bags for all tests.
* If you are interested in both the stored results AND some stored fixture values (through `@saved_fixture`), you
might rather wish to leverage the following helpers:
- use one of the `session_results_dct`, `module_results_dct`, `session_results_df` or `module_results_df`
fixtures. They contain all available information, in a nicely summarized way.
- use the `get_session_synthesis_dct(session)` helper method to create a similar synthesis than the above with
more customization capabilities.
If you wish to create custom results bags similar to this one (for example to create several with different names),
use `create_results_bag_fixture`.
"""
def get_session_results_dct(session_or_request,
fixture_store=FIXTURE_STORE, # type: Union[Mapping[str, Any], Iterable[Mapping[str, Any]]]
results_bag_fixture_name='results_bag' # type: str
):
# type: (...) -> Mapping[str, Mapping[str, Any]]
"""
Helper method to get exactly the same object than the `session_results_dct` fixture, from a session object.
This can be useful to retrieve the results from the `pytest_sessionfinish` hook for example.
See `session_results_dct` fixture for details.
:param session_or_request: the pytest session or request
:param fixture_store: an optional fixture store
:param results_bag_fixture_name: an optional name for results bag fixture in the fixture store. Default is
"results_bag"
:return:
"""
# in case of xdist, make sure persisted workers results have been reloaded
possibly_restore_xdist_workers_structs(session_or_request)
results_dct = get_session_synthesis_dct(session_or_request, durations_in_ms=True,
test_id_format='full', status_details=True, pytest_prefix=False,
fixture_store=fixture_store,
flatten=False, flatten_more=results_bag_fixture_name)
# We do not want to post-process according to steps here, this fixture should have as a contract that the keys
# are the True test ids.
# try:
# # if pytest_steps is installed, separate the test ids from the step ids
# from pytest_steps import handle_steps_in_synthesis_dct
# results_dct = handle_steps_in_synthesis_dct(results_dct, is_flat=False)
# except ImportError:
# # pytest_steps is not installed, ok.
# pass
return results_dct
@pytest.fixture(scope='function')
def session_results_dct(request, fixture_store):
# type: (...) -> Mapping[str, Mapping[str, Any]]
"""
This fixture contains a synthesis dictionary for all tests completed "so far", with 'full' id format. It includes
contents from the default `fixture_store`, including `results_bag`.
Behind the scenes it relies on `get_session_synthesis_dct`.
This fixture has a function scope because we want its contents to be refreshed every time it is needed.
"""
return get_session_results_dct(request.session, fixture_store=fixture_store)
def get_module_results_dct(session_or_request,
module_name, # type: str
fixture_store=FIXTURE_STORE, # type: Union[Mapping[str, Any], Iterable[Mapping[str, Any]]]
results_bag_fixture_name='results_bag' # type: str
):
# type: (...) -> Mapping[str, Mapping[str, Any]]
"""
Helper method to get exactly the same object than the `module_results_dct` fixture, from a session object.
This can be useful to retrieve the results from the `pytest_sessionfinish` hook for example.
See `module_results_dct` fixture for details.
:param session_or_request: the pytest session or request
:param module_name: a string representing the module name to filter on
:param fixture_store: an optional fixture store
:param results_bag_fixture_name: an optional name for results bag fixture in the fixture store. Default is
"results_bag"
:return:
"""
# in case of xdist, make sure persisted workers results have been reloaded
possibly_restore_xdist_workers_structs(session_or_request)
results_dct = get_session_synthesis_dct(session_or_request, durations_in_ms=True,
filter=module_name, pytest_prefix=False,
test_id_format='function', status_details=True,
fixture_store=fixture_store,
flatten=False, flatten_more=results_bag_fixture_name)
# We do not want to post-process according to steps here, this fixture should have as a contract that the keys
# are the True test ids.
#
# try:
# # if pytest_steps is installed, separate the test ids from the step ids
# from pytest_steps import handle_steps_in_synthesis_dct
# results_dct = handle_steps_in_synthesis_dct(results_dct, is_flat=False)
# except ImportError:
# # pytest_steps is not installed, ok.
# pass
return results_dct
@pytest.fixture(scope='function')
def module_results_dct(request, fixture_store):
# type: (...) -> Mapping[str, Mapping[str, Any]]
"""
This fixture returns a synthesis dictionary for all tests completed "so far" in the module of the caller,
with 'function' id format. It includes contents from the default `fixture_store`, including `results_bag`.
Behind the scenes it relies on `get_session_synthesis_dct`.
This fixture has a function scope because we want its contents to be refreshed every time it is needed.
"""
return get_module_results_dct(request, module_name=request.module.__name__, fixture_store=fixture_store)
try:
import pandas as pd
def get_session_results_df(session_or_request,
fixture_store=FIXTURE_STORE, # type: Union[Mapping[str, Any], Iterable[Mapping[str, Any]]]
results_bag_fixture_name='results_bag' # type: str
):
# type: (...) -> pd.DataFrame
"""
Helper method to get exactly the same object than the `session_results_df` fixture, from a session object.
This can be useful to retrieve the results from the `pytest_sessionfinish` hook for example.
See `session_results_df` fixture for details.
:param session_or_request: the pytest session or request
:param fixture_store: an optional fixture store
:param results_bag_fixture_name: an optional name for results bag fixture in the fixture store. Default is
"results_bag"
:return:
"""
# in case of xdist, make sure persisted workers results have been reloaded
possibly_restore_xdist_workers_structs(session_or_request)
# get the synthesis dictionary, merged with default fixture store and flattening default results_bag
session_results_dct = get_session_synthesis_dct(session_or_request, durations_in_ms=True,
test_id_format='full', status_details=False,
fixture_store=fixture_store,
flatten=True, flatten_more=results_bag_fixture_name)
# convert to a pandas dataframe
results_df = pd.DataFrame.from_dict(session_results_dct, orient='index')
results_df = results_df.loc[list(session_results_dct.keys()), :] # fix rows order
results_df.index.name = 'test_id' # set index name
# We do not want to post-process according to steps here, this fixture should have as a contract that the keys
# are the True test ids.
#
# try:
# # if pytest_steps is installed, separate the test ids from the step ids
# from pytest_steps import handle_steps_in_results_df
# results_df = handle_steps_in_results_df(results_df, keep_orig_id=True, no_steps_policy='skip')
# except ImportError:
# # pytest_steps is not installed, ok.
# pass
# except Exception as e:
# # other issue: warn about it but continue
# warn("%s: %s" % (e, type(e)))
return results_df
except ImportError as e:
saved_e = e
def get_session_results_df(*args, **kwargs):
six.raise_from(Exception("There was an error importing `pandas` module. Fixture `session_results_df` and method"
"`get_session_results_df` can not be used in this session."), saved_e)
@pytest.fixture(scope='function')
def session_results_df(request, fixture_store):
"""
This fixture contains a synthesis dataframe for all tests completed "so far" in the module of the caller,
with 'function' id format. It includes contents from the default `fixture_store`, including `results_bag`.
It is basically just a transformation of the `session_results_dct` fixture into a pandas `DataFrame`.
If `pytest-steps` is installed, the step ids will be extracted and the dataframe index will be multi-level
(test id without step, step id).
This fixture has a function scope because we want its contents to be refreshed every time it is needed.
"""
return get_session_results_df(request, fixture_store=fixture_store)
try:
import pandas as pd
def get_filtered_results_df(session,
filter=None, # type: Any
test_id_format='full', # type: str
fixture_store=FIXTURE_STORE, # type: Union[Mapping[str, Any], Iterable[Mapping[str, Any]]]
results_bag_fixture_name='results_bag' # type: str
):
# type: (...) -> pd.DataFrame
"""
Combines `get_session_synthesis_dct` with a transformation into a pandas DataFrame.
:param session: the pytest session object
:param filter: any filter, see `get_session_synthesis_dct` for details
:param fixture_store: an optional fixture store
:param results_bag_fixture_name: an optional name for results bag fixture in the fixture store. Default is
"results_bag"
:return:
"""
# in case of xdist, make sure persisted workers results have been reloaded
possibly_restore_xdist_workers_structs(session)
# get the synthesis dictionary, merged with default fixture store and flattening default results_bag
module_results_dct = get_session_synthesis_dct(session, durations_in_ms=True,
filter=filter,
test_id_format=test_id_format, status_details=False,
fixture_store=fixture_store,
flatten=True, flatten_more=results_bag_fixture_name)
# convert to a pandas dataframe
results_df = pd.DataFrame.from_dict(module_results_dct, orient='index')
results_df = results_df.loc[list(module_results_dct.keys()), :] # fix rows order
results_df.index.name = 'test_id' # set index name
# We do not want to post-process according to steps here, this fixture should have as a contract that the keys
# are the True test ids.
#
# try:
# # if pytest_steps is installed, separate the test ids from the step ids
# from pytest_steps import handle_steps_in_results_df
# results_df = handle_steps_in_results_df(results_df, keep_orig_id=True, no_steps_policy='skip')
# except ImportError:
# # pytest_steps is not installed, ok.
# pass
# except Exception as e:
# # other issue: warn about it but continue
# warn("%s: %s" % (e, type(e)))
return results_df
except ImportError as e:
saved_e = e
def get_filtered_results_df(*args, **kwargs):
six.raise_from(Exception("There was an error importing `pandas` module. Fixture `session_results_df` and "
"methods `get_filtered_results_df` and `get_module_results_df` can not be used in this"
" session. "), saved_e)
def get_module_results_df(session,
module_name, # type: str
fixture_store=FIXTURE_STORE, # type: Union[Mapping[str, Any], Iterable[Mapping[str, Any]]]
results_bag_fixture_name='results_bag' # type: str
):
"""
Helper method to get exactly the same object than the `module_results_df` fixture, from a session object.
This can be useful to retrieve the results from the `pytest_sessionfinish` hook for example.
See `module_results_df` fixture for details.
:param session: the pytest session object
:param module_name: the name of the module
:param fixture_store: an optional fixture store
:param results_bag_fixture_name: an optional name for results bag fixture in the fixture store. Default is
"results_bag"
:return:
"""
return get_filtered_results_df(session=session, test_id_format='function', filter=module_name,
fixture_store=fixture_store, results_bag_fixture_name=results_bag_fixture_name)
@pytest.fixture(scope='function')
def module_results_df(request, fixture_store):
"""
This fixture returns a synthesis dataframe for all tests completed "so far" in the module of the caller,
with 'function' id format. It includes contents from the default `fixture_store`, including `results_bag`.
It is basically just a transformation of the `module_results_dct` fixture into a pandas DataFrame.
If `pytest-steps` is installed, the step ids will be extracted and the dataframe index will be multi-level
(test id without step, step id).
This fixture has a function scope because we want its contents to be refreshed every time it is needed.
"""
return get_module_results_df(request.session, module_name=request.module.__name__, fixture_store=fixture_store)
# ----- Support for pytest-xdist: we need to persist results across worker processes
def pytest_addhooks(pluginmanager):
from pytest_harvest import newhooks
pluginmanager.add_hookspecs(newhooks)
class DefaultXDistHarvester(object):
"""
A pytest plugin which stores results from xdist nodes and gathers everything in the final master worker session
"""
def __init__(self, config):
# Folder in which temporary worker's results will be stored
self.results_path = Path('./.xdist_harvested/')
@pytest.hookimpl(trylast=True)
def pytest_harvest_xdist_init(self):
# reset the recipient folder
if self.results_path.exists():
rmtree(self.results_path)
self.results_path.mkdir(exist_ok=False)
return True
@pytest.hookimpl(trylast=True)
def pytest_harvest_xdist_worker_dump(self, worker_id, session_items, fixture_store):
with open(self.results_path / ('%s.pkl' % worker_id), 'wb') as f:
try:
pickle.dump((session_items, fixture_store), f)
except Exception as e:
warning("Error while pickling worker %s's harvested results: [%s] %s", (worker_id, e.__class__, e))
return True
@pytest.hookimpl(trylast=True)
def pytest_harvest_xdist_load(self):
workers_saved_material = dict()
for pkl_file in self.results_path.glob('*.pkl'):
wid = pkl_file.stem
with pkl_file.open('rb') as f:
workers_saved_material[wid] = pickle.load(f)
return workers_saved_material
@pytest.hookimpl(trylast=True)
def pytest_harvest_xdist_cleanup(self):
# delete all temporary pickle files
rmtree(self.results_path)
return True
@pytest.mark.trylast
def pytest_configure(config):
config.pluginmanager.register(DefaultXDistHarvester(config))
@pytest.hookimpl(tryfirst=True)
def pytest_sessionstart(session):
""" This should run first as it creates the temporary filder when run on the xdist master."""
if _is_xdist_master(session):
# perform cleanup
session.config.hook.pytest_harvest_xdist_init()
# mark the fixture store as to be reloaded
FIXTURE_STORE.disabled = True
@pytest.hookimpl(trylast=True)
def pytest_sessionfinish(session):
""" This should run last as it deletes the persisted items when run on the xdist master."""
if _is_xdist_worker(session):
# persist fixture store and report items in a pickle file with this id
wid = get_xdist_worker_id(session)
session_items = get_persistable_session_items(session)
session.config.hook.pytest_harvest_xdist_worker_dump(worker_id=wid, session_items=session_items,
fixture_store=FIXTURE_STORE)
elif _is_xdist_master(session):
# final master cleanup
session.config.hook.pytest_harvest_xdist_cleanup()
else:
# xdist not enabled - nothing to do
pass
def possibly_restore_xdist_workers_structs(session):
"""
If this is the xdist master
:param session:
:return:
"""
if _is_xdist_master(session) and hasattr(FIXTURE_STORE, 'disabled'):
# load saved session items and fixtures
workers_saved_material = session.config.hook.pytest_harvest_xdist_load()
# restore them into the same variables used by pytest-harvest
delattr(FIXTURE_STORE, 'disabled')
assert len(FIXTURE_STORE) == 0 # make sure nothing was added in there in between
session.items = []
for wid, (session_items, store) in workers_saved_material.items():
# session items
session.items += session_items
# saved fixtures
for fixture_name, _saved_fixture_dct in store.items():
try:
saved_fixture_dct = FIXTURE_STORE[fixture_name]
except KeyError:
FIXTURE_STORE[fixture_name] = _saved_fixture_dct
else:
assert len(set(saved_fixture_dct.keys()).intersection(set(_saved_fixture_dct.keys()))) == 0
saved_fixture_dct.update(_saved_fixture_dct)