Skip to content
This repository has been archived by the owner on Jul 6, 2023. It is now read-only.

feat: Add support for notebook tasks #100

Merged
merged 2 commits into from
Sep 30, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion google/cloud/dataplex_v1/types/logs.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,8 @@ class SessionEvent(proto.Message):
The log message.
user_id (str):
The information about the user that created
the session.
the session. It will be the email address of the
user.
session_id (str):
Unique identifier for the session.
type_ (google.cloud.dataplex_v1.types.SessionEvent.EventType):
Expand All @@ -337,6 +338,13 @@ class SessionEvent(proto.Message):
The execution details of the query.

This field is a member of `oneof`_ ``detail``.
event_succeeded (bool):
The status of the event.
warm_pool_enabled (bool):
If the session is a warm pooled session.
unassigned_duration (google.protobuf.duration_pb2.Duration):
The idle duration of a warm pooled session
before it is assigned to user.
"""

class EventType(proto.Enum):
Expand All @@ -345,6 +353,7 @@ class EventType(proto.Enum):
START = 1
STOP = 2
QUERY = 3
CREATE = 4

class QueryDetail(proto.Message):
r"""Execution details of the query.
Expand Down Expand Up @@ -420,6 +429,19 @@ class Engine(proto.Enum):
oneof="detail",
message=QueryDetail,
)
event_succeeded = proto.Field(
proto.BOOL,
number=6,
)
warm_pool_enabled = proto.Field(
proto.BOOL,
number=7,
)
unassigned_duration = proto.Field(
proto.MESSAGE,
number=8,
message=duration_pb2.Duration,
)


__all__ = tuple(sorted(__protobuf__.manifest))
56 changes: 56 additions & 0 deletions google/cloud/dataplex_v1/types/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@
class Task(proto.Message):
r"""A task represents a user-visible job.

This message has `oneof`_ fields (mutually exclusive fields).
For each oneof, at most one member field can be set at the same time.
Setting any member of the oneof automatically clears all other
members.

.. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields

Attributes:
Expand Down Expand Up @@ -70,6 +75,11 @@ class Task(proto.Message):
spark (google.cloud.dataplex_v1.types.Task.SparkTaskConfig):
Config related to running custom Spark tasks.

This field is a member of `oneof`_ ``config``.
notebook (google.cloud.dataplex_v1.types.Task.NotebookTaskConfig):
Config related to running scheduled
Notebooks.

This field is a member of `oneof`_ ``config``.
"""

Expand Down Expand Up @@ -445,6 +455,46 @@ class SparkTaskConfig(proto.Message):
message="Task.InfrastructureSpec",
)

class NotebookTaskConfig(proto.Message):
r"""Config for running scheduled notebooks.

Attributes:
notebook (str):
Required. Path to input notebook. This can be the GCS URI of
the notebook file or the path to a Notebook Content. The
execution args are accessible as environment variables
(``TASK_key=value``).
infrastructure_spec (google.cloud.dataplex_v1.types.Task.InfrastructureSpec):
Optional. Infrastructure specification for
the execution.
file_uris (Sequence[str]):
Optional. GCS URIs of files to be placed in
the working directory of each executor.
archive_uris (Sequence[str]):
Optional. GCS URIs of archives to be
extracted into the working directory of each
executor. Supported file types: .jar, .tar,
.tar.gz, .tgz, and .zip.
"""

notebook = proto.Field(
proto.STRING,
number=4,
)
infrastructure_spec = proto.Field(
proto.MESSAGE,
number=3,
message="Task.InfrastructureSpec",
)
file_uris = proto.RepeatedField(
proto.STRING,
number=5,
)
archive_uris = proto.RepeatedField(
proto.STRING,
number=6,
)

class ExecutionStatus(proto.Message):
r"""Status of the task execution (e.g. Jobs).

Expand Down Expand Up @@ -523,6 +573,12 @@ class ExecutionStatus(proto.Message):
oneof="config",
message=SparkTaskConfig,
)
notebook = proto.Field(
proto.MESSAGE,
number=302,
oneof="config",
message=NotebookTaskConfig,
)


class Job(proto.Message):
Expand Down