From a81f993782de21632df730e1f3f981401e968de9 Mon Sep 17 00:00:00 2001 From: Lindy Date: Fri, 27 Jan 2023 22:50:11 +0000 Subject: [PATCH] incomplete first draft of expanded data sharing docs (#590) * incomplete first draft of expanded data sharing docs * update sharing documentation * improved data sharing docs --------- Co-authored-by: Joseph Chatelain --- .../customizing_data_sharing.rst | 36 --------- docs/managing_data/index.rst | 9 ++- docs/managing_data/stream_pub_sub.rst | 78 +++++++++++++++++++ docs/managing_data/tom_direct_sharing.rst | 32 ++++++++ 4 files changed, 117 insertions(+), 38 deletions(-) delete mode 100644 docs/managing_data/customizing_data_sharing.rst create mode 100644 docs/managing_data/stream_pub_sub.rst create mode 100644 docs/managing_data/tom_direct_sharing.rst diff --git a/docs/managing_data/customizing_data_sharing.rst b/docs/managing_data/customizing_data_sharing.rst deleted file mode 100644 index f5fdace13..000000000 --- a/docs/managing_data/customizing_data_sharing.rst +++ /dev/null @@ -1,36 +0,0 @@ -Customizing Data Sharing ---------------------------- - -Currently, data sharing is only possible with HERMES. - -You will need to add ``DATA_SHARING`` to your ``settings.py`` that will give the proper credentials for the various -streams, TOMS, etc. with which you desire to share data. - -.. code:: python - - # Define the valid data sharing destinations for your TOM. - DATA_SHARING = { - 'hermes': { - 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), - 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), - 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', - 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), - 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', - 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), - 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] - }, - 'tom-demo-dev': { - 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), - 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), - 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), - 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), - }, - 'localhost-tom': { - # for testing; share with yourself - 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), - 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), - 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), - 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), - } - - } diff --git a/docs/managing_data/index.rst b/docs/managing_data/index.rst index 37d511a29..78979a9f5 100644 --- a/docs/managing_data/index.rst +++ b/docs/managing_data/index.rst @@ -9,7 +9,8 @@ Managing Data ../api/tom_dataproducts/views plotting_data customizing_data_processing - customizing_data_sharing + tom_direct_sharing + stream_pub_sub :doc:`Creating Plots from TOM Data ` - Learn how to create plots using plot.ly and your TOM @@ -18,4 +19,8 @@ data to display anywhere in your TOM. :doc:`Adding Custom Data Processing ` - Learn how you can process data into your TOM from uploaded data products. -:doc:`Adding Custom Data Sharing ` - Learn how you can share data from your TOM. +:doc:`TOM-TOM Direct Sharing ` - Learn how you can send and receive data between your TOM and another TOM-Toolkit TOM via an API. + +:doc:`Publish and Subscribe to a Kafka Stream ` - Learn how to publish and subscribe to a Kafka stream topic. + + diff --git a/docs/managing_data/stream_pub_sub.rst b/docs/managing_data/stream_pub_sub.rst new file mode 100644 index 000000000..2e072b5a3 --- /dev/null +++ b/docs/managing_data/stream_pub_sub.rst @@ -0,0 +1,78 @@ +Publish and Subscribe to a Kafka Stream +--------------------------------------- + +Publishing data to a stream and subscribing to a stream are handled independently and we describe each below. + + +Publish Data to a Kafka Topic +############################# + +TOM Toolkit supports publishing data to a Kafka stream such as `Hermes `_ (an interface to +`HOPSKOTCH `_) and `GCNClassicOverKafka `_. + +When sharing photometry data via Hermes, the TOM publishes the data to be shared to a topic on the HOPSKOTCH +Kafka stream. At this time, only photometry data is supported. + + +Configuring your TOM to Publish Data to a stream: +************************************************* + +You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials +for the various streams with which you wish to share data. + +.. code:: python + + # Define the valid data sharing destinations for your TOM. + DATA_SHARING = { + 'hermes': { + 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), + 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), + 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', + 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), + 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', + 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), + 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] + }, + } + +Subscribe to a Kafka Topic +########################## + +TOM Toolkit allows a TOM to subscribe to a topic on a Kafka stream, ingesting messages from that topic and handling the data. +This could involve simply logging the message or extracting the data from the message and saving it if it is properly formatted. + +Configuring your TOM to subscribe to a stream: +********************************************** + +First you will need to add ``tom_alertstreams`` to your list of ``INSTALLED_APPS`` in your ``settings.py``. + +.. code:: python + + INSTALLED_APPS = [ + ... + 'tom_alertstreams', + ] + +Then you will need to add an ``ALERT_STREAMS`` configuration dictionary to your ``settings.py``. This gives the credentials +for the various streams to which you wish to subscribe. Additionally, the ``TOPIC_HANDLERS`` section of the stream ``OPTIONS`` +will include a list of handlers for each topic. + +Some alert handlers are included as examples. Below we demonstrate how to connect to a Hermes Topic. You'll want to check +out the ``tom-alertstreams`` `README `_ for more details. + +.. code:: python + + ALERT_STREAMS = [ + { + 'ACTIVE': True, + 'NAME': 'tom_alertstreams.alertstreams.hopskotch.HopskotchAlertStream', + 'OPTIONS': { + 'URL': 'kafka://kafka.scimma.org/', + 'USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), + 'PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), + 'TOPIC_HANDLERS': { + 'tomtoolkit.test': 'tom_dataproducts.alertstreams.hermes.hermes_alert_handler', + }, + }, + }, + ] diff --git a/docs/managing_data/tom_direct_sharing.rst b/docs/managing_data/tom_direct_sharing.rst new file mode 100644 index 000000000..ee04cb73e --- /dev/null +++ b/docs/managing_data/tom_direct_sharing.rst @@ -0,0 +1,32 @@ +Sharing Data with Other TOMs +############################ + +TOM Toolkit does not yet support direct sharing between TOMs, however we hope to add this functionality soon. + + +.. Configuring your TOM to submit data to another TOM: +.. *************************************************** +.. +.. You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials +.. for the various TOMs with which you wish to share data. +.. +.. .. code:: python +.. +.. # Define the valid data sharing destinations for your TOM. +.. DATA_SHARING = { +.. 'tom-demo-dev': { +.. 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), +.. 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), +.. 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), +.. 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), +.. }, +.. 'localhost-tom': { +.. # for testing; share with yourself +.. 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), +.. 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), +.. 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), +.. 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), +.. } +.. +.. } +.. \ No newline at end of file