diff --git a/docs/index.md b/docs/index.md index 0ebdc325b..d94148a3f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -456,6 +456,7 @@ auto_examples/papermill_plugin/index auto_examples/pandera_plugin/index auto_examples/kfpytorch_plugin/index auto_examples/ray_plugin/index +auto_examples/sensor/index auto_examples/snowflake_plugin/index auto_examples/k8s_spark_plugin/index auto_examples/sql_plugin/index diff --git a/docs/integrations.md b/docs/integrations.md index 83bb5e878..de04dded8 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -106,6 +106,8 @@ the Flyte task that use the respective plugin. - Run Hive jobs in your workflows. * - {doc}`MMCloud ` - Execute tasks using MemVerge Memory Machine Cloud +* - {doc}`Sensor ` + - Run Sensor jobs in your workflows. * - {doc}`Snowflake ` - Run Snowflake jobs in your workflows. * - {doc}`Databricks ` diff --git a/examples/sensor/README.md b/examples/sensor/README.md new file mode 100644 index 000000000..9bbcb5f81 --- /dev/null +++ b/examples/sensor/README.md @@ -0,0 +1,20 @@ +(sensor)= + +# Sensor + +```{eval-rst} +.. tags:: Data, Basic +``` + +## Run the example on the Flyte cluster + +To run the provided example on the Flyte cluster, use the following command: + +``` +pyflyte run --remote \ + https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/sensor/sensor/file_sensor_example.py wf +``` + +```{auto-examples-toc} +file_sensor_example +``` diff --git a/examples/sensor/sensor/__init__.py b/examples/sensor/sensor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/sensor/sensor/file_sensor_example.py b/examples/sensor/sensor/file_sensor_example.py new file mode 100644 index 000000000..aa82b1cbc --- /dev/null +++ b/examples/sensor/sensor/file_sensor_example.py @@ -0,0 +1,44 @@ +# %% [markdown] +# # File Sensor +# +# This example shows how to use the `FileSensor` to detect files appearing in your local or remote filesystem. +# +# First, import the required libraries. + +# %% +from flytekit import task, workflow +from flytekit.sensor.file_sensor import FileSensor + +# %% [markdown] +# Next, create a FileSensor task. + +# %% +sensor = FileSensor(name="test_file_sensor") + +# %% [markdown] +# To use the FileSensor created in the previous step, you must specify the path parameter. In the sandbox, you can use the S3 path. + + +# %% +@task() +def t1(): + print("SUCCEEDED") + + +@workflow() +def wf(): + sensor(path="s3://my-s3-bucket/file.txt") >> t1() + + +if __name__ == "__main__": + wf() + +# %% [markdown] +# You can also use the S3 or GCS file system. +# We have already set the minio credentials in the agent by default. If you test the sandbox example locally, you will need to set the AWS credentials in your environment variables. +# +# ```{prompt} bash +# export FLYTE_AWS_ENDPOINT="http://localhost:30002" +# export FLYTE_AWS_ACCESS_KEY_ID="minio" +# export FLYTE_AWS_SECRET_ACCESS_KEY="miniostorage" +# ```