-
Notifications
You must be signed in to change notification settings - Fork 245
/
__init__.py
262 lines (201 loc) · 7.13 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
"""
=====================
Core Flytekit
=====================
.. currentmodule:: flytekit
This package contains all of the most common abstractions you'll need to write Flyte workflows and extend Flytekit.
Basic Authoring
===============
These are the essentials needed to get started writing tasks and workflows. The elements here correspond well with :std:ref:`Basics <cookbook:sphx_glr_auto_core_flyte_basics>` section of the user guide.
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
task
workflow
kwtypes
current_context
ExecutionParameters
FlyteContext
map_task
~core.workflow.ImperativeWorkflow
~core.node_creation.create_node
FlyteContextManager
Running Locally
------------------
Tasks and Workflows can both be locally run (assuming the relevant tasks are capable of local execution). This is useful for unit testing.
Branching and Conditionals
==========================
Branches and conditionals can be expressed explicitly in Flyte. These conditions are evaluated
in the flyte engine and hence should be used for control flow. ``dynamic workflows`` can be used to perform custom conditional logic not supported by flytekit
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
conditional
Customizing Tasks & Workflows
==============================
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
TaskMetadata - Wrapper object that allows users to specify Task
Resources - Things like CPUs/Memory, etc.
WorkflowFailurePolicy - Customizes what happens when a workflow fails.
Dynamic and Nested Workflows
==============================
See the :py:mod:`Dynamic <flytekit.core.dynamic_workflow_task>` module for more information.
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
dynamic
Scheduling
============================
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
CronSchedule
FixedRate
Notifications
============================
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
Email
PagerDuty
Slack
Reference Entities
====================
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
get_reference_entity
LaunchPlanReference
TaskReference
WorkflowReference
reference_task
reference_workflow
Core Task Types
=================
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
SQLTask
ContainerTask
PythonFunctionTask
PythonInstanceTask
LaunchPlan
Secrets and SecurityContext
============================
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
Secret
SecurityContext
Common Flyte IDL Objects
=========================
.. autosummary::
:nosignatures:
:template: custom.rst
:toctree: generated/
AuthRole
Labels
Annotations
WorkflowExecutionPhase
Blob
BlobMetadata
Literal
Scalar
LiteralType
BlobType
"""
import sys
from typing import Generator
if sys.version_info < (3, 10):
from importlib_metadata import entry_points
else:
from importlib.metadata import entry_points
from flytekit.core.base_sql_task import SQLTask
from flytekit.core.base_task import SecurityContext, TaskMetadata, kwtypes
from flytekit.core.checkpointer import Checkpoint
from flytekit.core.condition import conditional
from flytekit.core.container_task import ContainerTask
from flytekit.core.context_manager import ExecutionParameters, FlyteContext, FlyteContextManager
from flytekit.core.data_persistence import DataPersistence, DataPersistencePlugins
from flytekit.core.dynamic_workflow_task import dynamic
from flytekit.core.gate import approve, sleep, wait_for_input
from flytekit.core.hash import HashMethod
from flytekit.core.launch_plan import LaunchPlan
from flytekit.core.map_task import map_task
from flytekit.core.notification import Email, PagerDuty, Slack
from flytekit.core.python_function_task import PythonFunctionTask, PythonInstanceTask
from flytekit.core.reference import get_reference_entity
from flytekit.core.reference_entity import LaunchPlanReference, TaskReference, WorkflowReference
from flytekit.core.resources import Resources
from flytekit.core.schedule import CronSchedule, FixedRate
from flytekit.core.task import Secret, reference_task, task
from flytekit.core.workflow import ImperativeWorkflow as Workflow
from flytekit.core.workflow import WorkflowFailurePolicy, reference_workflow, workflow
from flytekit.deck import Deck
from flytekit.extras import pytorch, sklearn, tensorflow
from flytekit.extras.persistence import GCSPersistence, HttpPersistence, S3Persistence
from flytekit.loggers import logger
from flytekit.models.common import Annotations, AuthRole, Labels
from flytekit.models.core.execution import WorkflowExecutionPhase
from flytekit.models.core.types import BlobType
from flytekit.models.documentation import Description, Documentation, SourceCode
from flytekit.models.literals import Blob, BlobMetadata, Literal, Scalar
from flytekit.models.types import LiteralType
from flytekit.types import directory, file, numpy, schema
from flytekit.types.structured.structured_dataset import (
StructuredDataset,
StructuredDatasetFormat,
StructuredDatasetTransformerEngine,
StructuredDatasetType,
)
__version__ = "0.0.0+develop"
def current_context() -> ExecutionParameters:
"""
Use this method to get a handle of specific parameters available in a flyte task.
Usage
.. code-block:: python
flytekit.current_context().logging.info(...)
Available params are documented in :py:class:`flytekit.core.context_manager.ExecutionParams`.
There are some special params, that should be available
"""
return FlyteContextManager.current_context().execution_state.user_space_params
def new_context() -> Generator[FlyteContext, None, None]:
return FlyteContextManager.with_context(FlyteContextManager.current_context().new_builder())
def load_implicit_plugins():
"""
This method allows loading all plugins that have the entrypoint specification. This uses the plugin loading
behavior as explained `here <>`_.
This is an opt in system and plugins that have an implicit loading requirement should add the implicit loading
entrypoint specification to their setup.py. The following example shows how we can autoload a module called fsspec
(whose init files contains the necessary plugin registration step)
.. code-block::
# note the group is always ``flytekit.plugins``
setup(
...
entry_points={'flytekit.plugins’: 'fsspec=flytekitplugins.fsspec'},
...
)
This works as long as the fsspec module has
.. code-block::
# For data persistence plugins
DataPersistencePlugins.register_plugin(f"{k}://", FSSpecPersistence, force=True)
# OR for type plugins
TypeEngine.register(PanderaTransformer())
# etc
"""
discovered_plugins = entry_points(group="flytekit.plugins")
for p in discovered_plugins:
p.load()
# Load all implicit plugins
load_implicit_plugins()