/
inspect.py
452 lines (397 loc) 路 20.6 KB
/
inspect.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
# Copyright 2020 The HuggingFace Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
""" List and inspect datasets."""
import inspect
import os
import shutil
from pathlib import PurePath
from typing import Dict, List, Mapping, Optional, Sequence, Union
import huggingface_hub
from .download.download_config import DownloadConfig
from .download.download_manager import DownloadMode
from .download.streaming_download_manager import StreamingDownloadManager
from .info import DatasetInfo
from .load import dataset_module_factory, import_main_class, load_dataset_builder, metric_module_factory
from .utils.deprecation_utils import deprecated
from .utils.file_utils import relative_to_absolute_path
from .utils.logging import get_logger
from .utils.version import Version
logger = get_logger(__name__)
class SplitsNotFoundError(ValueError):
pass
def list_datasets(with_community_datasets=True, with_details=False):
"""List all the datasets scripts available on the Hugging Face Hub.
Args:
with_community_datasets (`bool`, *optional*, defaults to `True`):
Include the community provided datasets.
with_details (`bool`, *optional*, defaults to `False`):
Return the full details on the datasets instead of only the short name.
Example:
```py
>>> from datasets import list_datasets
>>> list_datasets()
['acronym_identification',
'ade_corpus_v2',
'adversarial_qa',
'aeslc',
'afrikaans_ner_corpus',
'ag_news',
...
]
```
"""
datasets = huggingface_hub.list_datasets(full=with_details)
if not with_community_datasets:
datasets = [dataset for dataset in datasets if "/" not in dataset.id]
if not with_details:
datasets = [dataset.id for dataset in datasets]
return datasets
@deprecated(
"Use 'evaluate.list_evaluation_modules' instead, from the new library 馃 Evaluate: https://huggingface.co/docs/evaluate"
)
def list_metrics(with_community_metrics=True, with_details=False):
"""List all the metrics script available on the Hugging Face Hub.
<Deprecated version="2.5.0">
Use `evaluate.list_evaluation_modules` instead, from the new library 馃 Evaluate: https://huggingface.co/docs/evaluate
</Deprecated>
Args:
with_community_metrics (:obj:`bool`, optional, default ``True``): Include the community provided metrics.
with_details (:obj:`bool`, optional, default ``False``): Return the full details on the metrics instead of only the short name.
Example:
```py
>>> from datasets import list_metrics
>>> list_metrics()
['accuracy',
'bertscore',
'bleu',
'bleurt',
'cer',
'chrf',
...
]
```
"""
metrics = huggingface_hub.list_metrics()
if not with_community_metrics:
metrics = [metric for metric in metrics if "/" not in metric.id]
if not with_details:
metrics = [metric.id for metric in metrics]
return metrics
def inspect_dataset(path: str, local_path: str, download_config: Optional[DownloadConfig] = None, **download_kwargs):
"""
Allow inspection/modification of a dataset script by copying on local drive at local_path.
Args:
path (`str`): Path to the dataset processing script with the dataset builder. Can be either:
- a local path to processing script or the directory containing the script (if the script has the same name
as the directory),
e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`.
- a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`list_datasets`])
e.g. `'squad'`, `'glue'` or `'openai/webtext'`.
local_path (`str`):
Path to the local folder to copy the dataset script to.
download_config ([`DownloadConfig`], *optional*):
Specific download configuration parameters.
**download_kwargs (additional keyword arguments):
Optional arguments for [`DownloadConfig`] which will override
the attributes of `download_config` if supplied.
"""
dataset_module = dataset_module_factory(path, download_config=download_config, **download_kwargs)
builder_cls = import_main_class(dataset_module.module_path, dataset=True)
module_source_path = inspect.getsourcefile(builder_cls)
module_source_dirpath = os.path.dirname(module_source_path)
for dirpath, dirnames, filenames in os.walk(module_source_dirpath):
dst_dirpath = os.path.join(local_path, os.path.relpath(dirpath, module_source_dirpath))
os.makedirs(dst_dirpath, exist_ok=True)
# skipping hidden directories; prune the search
# [:] for the in-place list modification required by os.walk
dirnames[:] = [dirname for dirname in dirnames if not dirname.startswith((".", "__"))]
for filename in filenames:
shutil.copy2(os.path.join(dirpath, filename), os.path.join(dst_dirpath, filename))
shutil.copystat(dirpath, dst_dirpath)
local_path = relative_to_absolute_path(local_path)
print(
f"The processing script for dataset {path} can be inspected at {local_path}. "
f"The main class is in {module_source_dirpath}. "
f'You can modify this processing script and use it with `datasets.load_dataset("{PurePath(local_path).as_posix()}")`.'
)
@deprecated(
"Use 'evaluate.inspect_evaluation_module' instead, from the new library 馃 Evaluate: https://huggingface.co/docs/evaluate"
)
def inspect_metric(path: str, local_path: str, download_config: Optional[DownloadConfig] = None, **download_kwargs):
r"""
Allow inspection/modification of a metric script by copying it on local drive at local_path.
<Deprecated version="2.5.0">
Use `evaluate.inspect_evaluation_module` instead, from the new library 馃 Evaluate instead: https://huggingface.co/docs/evaluate
</Deprecated>
Args:
path (``str``): path to the dataset processing script with the dataset builder. Can be either:
- a local path to processing script or the directory containing the script (if the script has the same name as the directory),
e.g. ``'./dataset/squad'`` or ``'./dataset/squad/squad.py'``
- a dataset identifier on the Hugging Face Hub (list all available datasets and ids with ``datasets.list_datasets()``)
e.g. ``'squad'``, ``'glue'`` or ``'openai/webtext'``
local_path (``str``): path to the local folder to copy the datset script to.
download_config (Optional ``datasets.DownloadConfig``): specific download configuration parameters.
**download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override the attributes in download_config if supplied.
"""
metric_module = metric_module_factory(path, download_config=download_config, **download_kwargs)
builder_cls = import_main_class(metric_module.module_path, dataset=False)
module_source_path = inspect.getsourcefile(builder_cls)
module_source_dirpath = os.path.dirname(module_source_path)
for dirpath, dirnames, filenames in os.walk(module_source_dirpath):
dst_dirpath = os.path.join(local_path, os.path.relpath(dirpath, module_source_dirpath))
os.makedirs(dst_dirpath, exist_ok=True)
# skipping hidden directories; prune the search
dirnames[:] = [dirname for dirname in dirnames if not dirname.startswith((".", "__"))]
for filename in filenames:
shutil.copy2(os.path.join(dirpath, filename), os.path.join(dst_dirpath, filename))
shutil.copystat(dirpath, dst_dirpath)
local_path = relative_to_absolute_path(local_path)
print(
f"The processing scripts for metric {path} can be inspected at {local_path}. "
f"The main class is in {module_source_dirpath}. "
f'You can modify this processing scripts and use it with `datasets.load_metric("{PurePath(local_path).as_posix()}")`.'
)
def get_dataset_infos(
path: str,
data_files: Optional[Union[Dict, List, str]] = None,
download_config: Optional[DownloadConfig] = None,
download_mode: Optional[Union[DownloadMode, str]] = None,
revision: Optional[Union[str, Version]] = None,
use_auth_token: Optional[Union[bool, str]] = None,
**config_kwargs,
):
"""Get the meta information about a dataset, returned as a dict mapping config name to DatasetInfoDict.
Args:
path (`str`): path to the dataset processing script with the dataset builder. Can be either:
- a local path to processing script or the directory containing the script (if the script has the same name as the directory),
e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`
- a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`datasets.list_datasets`])
e.g. `'squad'`, `'glue'` or``'openai/webtext'`
revision (`Union[str, datasets.Version]`, *optional*):
If specified, the dataset module will be loaded from the datasets repository at this version.
By default:
- it is set to the local version of the lib.
- it will also try to load it from the main branch if it's not available at the local version of the lib.
Specifying a version that is different from your local version of the lib might cause compatibility issues.
download_config ([`DownloadConfig`], *optional*):
Specific download configuration parameters.
download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
Download/generate mode.
data_files (`Union[Dict, List, str]`, *optional*):
Defining the data_files of the dataset configuration.
use_auth_token (`str` or `bool`, *optional*):
Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
If `True`, or not specified, will get token from `"~/.huggingface"`.
**config_kwargs (additional keyword arguments):
Optional attributes for builder class which will override the attributes if supplied.
Example:
```py
>>> from datasets import get_dataset_infos
>>> get_dataset_infos('rotten_tomatoes')
{'default': DatasetInfo(description="Movie Review Dataset.\nThis is a dataset of containing 5,331 positive and 5,331 negative processed\nsentences from Rotten Tomatoes movie reviews...), ...}
```
"""
config_names = get_dataset_config_names(
path=path,
revision=revision,
download_config=download_config,
download_mode=download_mode,
data_files=data_files,
use_auth_token=use_auth_token,
)
return {
config_name: get_dataset_config_info(
path=path,
config_name=config_name,
data_files=data_files,
download_config=download_config,
download_mode=download_mode,
revision=revision,
use_auth_token=use_auth_token,
**config_kwargs,
)
for config_name in config_names
}
def get_dataset_config_names(
path: str,
revision: Optional[Union[str, Version]] = None,
download_config: Optional[DownloadConfig] = None,
download_mode: Optional[Union[DownloadMode, str]] = None,
dynamic_modules_path: Optional[str] = None,
data_files: Optional[Union[Dict, List, str]] = None,
**download_kwargs,
):
"""Get the list of available config names for a particular dataset.
Args:
path (`str`): path to the dataset processing script with the dataset builder. Can be either:
- a local path to processing script or the directory containing the script (if the script has the same name as the directory),
e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`
- a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`datasets.list_datasets`])
e.g. `'squad'`, `'glue'` or `'openai/webtext'`
revision (`Union[str, datasets.Version]`, *optional*):
If specified, the dataset module will be loaded from the datasets repository at this version.
By default:
- it is set to the local version of the lib.
- it will also try to load it from the main branch if it's not available at the local version of the lib.
Specifying a version that is different from your local version of the lib might cause compatibility issues.
download_config ([`DownloadConfig`], *optional*):
Specific download configuration parameters.
download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
Download/generate mode.
dynamic_modules_path (`str`, defaults to `~/.cache/huggingface/modules/datasets_modules`):
Optional path to the directory in which the dynamic modules are saved. It must have been initialized with `init_dynamic_modules`.
By default the datasets and metrics are stored inside the `datasets_modules` module.
data_files (`Union[Dict, List, str]`, *optional*):
Defining the data_files of the dataset configuration.
**download_kwargs (additional keyword arguments):
Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied,
for example `use_auth_token`.
Example:
```py
>>> from datasets import get_dataset_config_names
>>> get_dataset_config_names("glue")
['cola',
'sst2',
'mrpc',
'qqp',
'stsb',
'mnli',
'mnli_mismatched',
'mnli_matched',
'qnli',
'rte',
'wnli',
'ax']
```
"""
dataset_module = dataset_module_factory(
path,
revision=revision,
download_config=download_config,
download_mode=download_mode,
dynamic_modules_path=dynamic_modules_path,
data_files=data_files,
**download_kwargs,
)
builder_cls = import_main_class(dataset_module.module_path)
return list(builder_cls.builder_configs.keys()) or [dataset_module.builder_kwargs.get("config_name", "default")]
def get_dataset_config_info(
path: str,
config_name: Optional[str] = None,
data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
download_config: Optional[DownloadConfig] = None,
download_mode: Optional[Union[DownloadMode, str]] = None,
revision: Optional[Union[str, Version]] = None,
use_auth_token: Optional[Union[bool, str]] = None,
**config_kwargs,
) -> DatasetInfo:
"""Get the meta information (DatasetInfo) about a dataset for a particular config
Args:
path (``str``): path to the dataset processing script with the dataset builder. Can be either:
- a local path to processing script or the directory containing the script (if the script has the same name as the directory),
e.g. ``'./dataset/squad'`` or ``'./dataset/squad/squad.py'``
- a dataset identifier on the Hugging Face Hub (list all available datasets and ids with ``datasets.list_datasets()``)
e.g. ``'squad'``, ``'glue'`` or ``'openai/webtext'``
config_name (:obj:`str`, optional): Defining the name of the dataset configuration.
data_files (:obj:`str` or :obj:`Sequence` or :obj:`Mapping`, optional): Path(s) to source data file(s).
download_config (:class:`~download.DownloadConfig`, optional): Specific download configuration parameters.
download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode.
revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset script to load.
As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
use_auth_token (``str`` or :obj:`bool`, optional): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
If True, or not specified, will get token from `"~/.huggingface"`.
**config_kwargs (additional keyword arguments): optional attributes for builder class which will override the attributes if supplied.
"""
builder = load_dataset_builder(
path,
name=config_name,
data_files=data_files,
download_config=download_config,
download_mode=download_mode,
revision=revision,
use_auth_token=use_auth_token,
**config_kwargs,
)
info = builder.info
if info.splits is None:
download_config = download_config.copy() if download_config else DownloadConfig()
if use_auth_token is not None:
download_config.use_auth_token = use_auth_token
builder._check_manual_download(
StreamingDownloadManager(base_path=builder.base_path, download_config=download_config)
)
try:
info.splits = {
split_generator.name: {"name": split_generator.name, "dataset_name": path}
for split_generator in builder._split_generators(
StreamingDownloadManager(base_path=builder.base_path, download_config=download_config)
)
}
except Exception as err:
raise SplitsNotFoundError("The split names could not be parsed from the dataset config.") from err
return info
def get_dataset_split_names(
path: str,
config_name: Optional[str] = None,
data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
download_config: Optional[DownloadConfig] = None,
download_mode: Optional[Union[DownloadMode, str]] = None,
revision: Optional[Union[str, Version]] = None,
use_auth_token: Optional[Union[bool, str]] = None,
**config_kwargs,
):
"""Get the list of available splits for a particular config and dataset.
Args:
path (`str`): path to the dataset processing script with the dataset builder. Can be either:
- a local path to processing script or the directory containing the script (if the script has the same name as the directory),
e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`
- a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`datasets.list_datasets`])
e.g. `'squad'`, `'glue'` or `'openai/webtext'`
config_name (`str`, *optional*):
Defining the name of the dataset configuration.
data_files (`str` or `Sequence` or `Mapping`, *optional*):
Path(s) to source data file(s).
download_config ([`DownloadConfig`], *optional*):
Specific download configuration parameters.
download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
Download/generate mode.
revision ([`Version`] or `str`, *optional*):
Version of the dataset script to load.
As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
use_auth_token (`str` or `bool`, *optional*):
Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
If `True`, or not specified, will get token from `"~/.huggingface"`.
**config_kwargs (additional keyword arguments):
Optional attributes for builder class which will override the attributes if supplied.
Example:
```py
>>> from datasets import get_dataset_split_names
>>> get_dataset_split_names('rotten_tomatoes')
['train', 'validation', 'test']
```
"""
info = get_dataset_config_info(
path,
config_name=config_name,
data_files=data_files,
download_config=download_config,
download_mode=download_mode,
revision=revision,
use_auth_token=use_auth_token,
**config_kwargs,
)
return list(info.splits.keys())