/
mapping.py
126 lines (96 loc) · 3.36 KB
/
mapping.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
from typing import Dict, List, Optional, Union
from typing_extensions import Literal
from clipped.compact.pydantic import Field, PositiveInt, validator
from clipped.types.ref_or_obj import RefField
from polyaxon._contexts.params import is_template_ref
from polyaxon._flow.early_stopping import V1EarlyStopping
from polyaxon._flow.matrix.base import BaseSearchConfig
from polyaxon._flow.matrix.enums import V1MatrixKind
class V1Mapping(BaseSearchConfig):
"""Mapping is a flexible way for dynamically executing a component sequentially or in parallel
based on a list of parameter combinations.
Args:
kind: str, should be equal `mapping`
values: List[Dict]
concurrency: int, optional
early_stopping: List[[EarlyStopping](/docs/automation/helpers/early-stopping)], optional
## YAML usage
```yaml
>>> matrix:
>>> kind: mapping
>>> values:
>>> concurrency:
>>> earlyStopping:
```
## Python usage
```python
>>> from polyaxon.schemas import V1Mapping, V1FailureEarlyStopping
>>> mapping = V1Mapping(
>>> values=[{...}, {...}, ...],
>>> concurrency=4,
>>> early_stopping=[V1FailureEarlyStopping(...)]
>>> )
```
## Fields
### kind
The kind signals to the CLI, client, and other tools that this matrix is mapping.
If you are using the python client to create the mapping,
this field is not required and is set by default.
```yaml
>>> matrix:
>>> kind: mapping
```
### values
A List of dictionaries (key/value objects) that will be used to pass
those dictionaries as params to each execution.
```yaml
>>> matrix:
>>> values:
>>> - lr: 0.001
>>> dropout: 0.1
>>> - lr: 0.01
>>> dropout: 0.2
>>> - lr: 0.1
>>> dropout: 0.3
```
### concurrency
An optional value to set the number of concurrent operations.
<blockquote class="light">
This value only makes sense if less or equal to the total number of possible runs.
</blockquote>
```yaml
>>> matrix:
>>> concurrency: 2
```
For more details about concurrency management,
please check the [concurrency section](/docs/automation/helpers/concurrency/).
### earlyStopping
A list of early stopping conditions to check for terminating
all operations managed by the pipeline.
If one of the early stopping conditions is met,
a signal will be sent to terminate all running and pending operations.
```yaml
>>> matrix:
>>> earlyStopping: ...
```
For more details please check the
[early stopping section](/docs/automation/helpers/early-stopping/).
"""
_IDENTIFIER = V1MatrixKind.MAPPING
kind: Literal[_IDENTIFIER] = _IDENTIFIER
values: Union[List[Dict], RefField]
concurrency: Optional[Union[PositiveInt, RefField]]
early_stopping: Optional[Union[List[V1EarlyStopping], RefField]] = Field(
alias="earlyStopping"
)
@validator("concurrency")
def check_concurrency(cls, v):
if v and v < 1:
raise ValueError(
f"concurrency must be greater than 1, received `{v} instead."
)
return v
def has_key(self, key: str):
if is_template_ref(self.values):
return True
return self.values and key in set(self.values[0].keys())