Harrison/nasa tool (langchain-ai#14245)

Co-authored-by: Jacob Matias <88005863+matiasjacob25@users.noreply.github.com> Co-authored-by: Karam Daid <karam.daid@mail.utoronto.ca> Co-authored-by: Jumana <jumana.fanous@mail.utoronto.ca> Co-authored-by: KaramDaid <38271127+KaramDaid@users.noreply.github.com> Co-authored-by: Anna Chester <74325334+CodeMakesMeSmile@users.noreply.github.com> Co-authored-by: Jumana <144748640+jfanous@users.noreply.github.com>
andrewrreed · Dec 11, 2023 · 34db4db · 34db4db
1 parent d2ebe17
commit 34db4db
Show file tree

Hide file tree

Showing 14 changed files with 383 additions and 0 deletions.
diff --git a/docs/docs/integrations/toolkits/nasa.ipynb b/docs/docs/integrations/toolkits/nasa.ipynb
@@ -0,0 +1,108 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e6fd05db-21c2-4227-9900-0840bc62cb31",
+   "metadata": {},
+   "source": [
+    "# NASA\n",
+    "\n",
+    "This notebook shows how to use agents to interact with the NASA toolkit. The toolkit provides access to the NASA Image and Video Library API, with potential to expand and include other accessible NASA APIs in future iterations.\n",
+    "\n",
+    "**Note: NASA Image and Video Library search queries can result in large responses when the number of desired media results is not specified. Consider this prior to using the agent with LLM token credits.**"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7d93e6bd-03d7-4d3c-b915-8b73164e2ad8",
+   "metadata": {},
+   "source": [
+    "## Example Use:\n",
+    "---\n",
+    "### Initializing the agent"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "648a2cb2-308e-4b2e-9b73-37109be4e258",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.agents import AgentType, initialize_agent\n",
+    "from langchain.agents.agent_toolkits.nasa.toolkit import NasaToolkit\n",
+    "from langchain.llms import OpenAI\n",
+    "from langchain.utilities.nasa import NasaAPIWrapper\n",
+    "\n",
+    "llm = OpenAI(temperature=0, openai_api_key=\"\")\n",
+    "nasa = NasaAPIWrapper()\n",
+    "toolkit = NasaToolkit.from_nasa_api_wrapper(nasa)\n",
+    "agent = initialize_agent(\n",
+    "    toolkit.get_tools(), llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "71f05fc9-d80d-4614-b9a3-e0a5e43cbbbb",
+   "metadata": {},
+   "source": [
+    "### Querying media assets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b97409f3-dc87-425d-b555-406cf8466a28",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "agent.run(\n",
+    "    \"Can you find three pictures of the moon published between the years 2014 and 2020?\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a86ce5ff-de45-4206-86ca-07ae03f36bdf",
+   "metadata": {},
+   "source": [
+    "### Querying details about media assets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "80e86b49-749e-4026-b025-db32ed0bcc7e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "output = agent.run(\n",
+    "    \"I've just queried an image of the moon with the NASA id NHQ_2019_0311_Go Forward to the Moon.\"\n",
+    "    \" Where can I find the metadata manifest for this asset?\"\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/libs/langchain/langchain/agents/agent_toolkits/__init__.py b/libs/langchain/langchain/agents/agent_toolkits/__init__.py
@@ -34,6 +34,7 @@
 from langchain.agents.agent_toolkits.json.base import create_json_agent
 from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit
 from langchain.agents.agent_toolkits.multion.toolkit import MultionToolkit
+from langchain.agents.agent_toolkits.nasa.toolkit import NasaToolkit
 from langchain.agents.agent_toolkits.nla.toolkit import NLAToolkit
 from langchain.agents.agent_toolkits.office365.toolkit import O365Toolkit
 from langchain.agents.agent_toolkits.openapi.base import create_openapi_agent
@@ -93,6 +94,7 @@ def __getattr__(name: str) -> Any:
     "JiraToolkit",
     "JsonToolkit",
     "MultionToolkit",
+    "NasaToolkit",
     "NLAToolkit",
     "O365Toolkit",
     "OpenAPIToolkit",

diff --git a/libs/langchain/langchain/agents/agent_toolkits/nasa/__init__.py b/libs/langchain/langchain/agents/agent_toolkits/nasa/__init__.py
@@ -0,0 +1 @@
+"""NASA Toolkit"""
diff --git a/libs/langchain/langchain/agents/agent_toolkits/nasa/toolkit.py b/libs/langchain/langchain/agents/agent_toolkits/nasa/toolkit.py
@@ -0,0 +1,57 @@
+from typing import Dict, List
+
+from langchain.agents.agent_toolkits.base import BaseToolkit
+from langchain.tools import BaseTool
+from langchain.tools.nasa.prompt import (
+    NASA_CAPTIONS_PROMPT,
+    NASA_MANIFEST_PROMPT,
+    NASA_METADATA_PROMPT,
+    NASA_SEARCH_PROMPT,
+)
+from langchain.tools.nasa.tool import NasaAction
+from langchain.utilities.nasa import NasaAPIWrapper
+
+
+class NasaToolkit(BaseToolkit):
+    """Nasa Toolkit."""
+
+    tools: List[BaseTool] = []
+
+    @classmethod
+    def from_nasa_api_wrapper(cls, nasa_api_wrapper: NasaAPIWrapper) -> "NasaToolkit":
+        operations: List[Dict] = [
+            {
+                "mode": "search_media",
+                "name": "Search NASA Image and Video Library media",
+                "description": NASA_SEARCH_PROMPT,
+            },
+            {
+                "mode": "get_media_metadata_manifest",
+                "name": "Get NASA Image and Video Library media metadata manifest",
+                "description": NASA_MANIFEST_PROMPT,
+            },
+            {
+                "mode": "get_media_metadata_location",
+                "name": "Get NASA Image and Video Library media metadata location",
+                "description": NASA_METADATA_PROMPT,
+            },
+            {
+                "mode": "get_video_captions_location",
+                "name": "Get NASA Image and Video Library video captions location",
+                "description": NASA_CAPTIONS_PROMPT,
+            },
+        ]
+        tools = [
+            NasaAction(
+                name=action["name"],
+                description=action["description"],
+                mode=action["mode"],
+                api_wrapper=nasa_api_wrapper,
+            )
+            for action in operations
+        ]
+        return cls(tools=tools)
+
+    def get_tools(self) -> List[BaseTool]:
+        """Get the tools in the toolkit."""
+        return self.tools
diff --git a/libs/langchain/langchain/tools/__init__.py b/libs/langchain/langchain/tools/__init__.py
@@ -338,6 +338,12 @@ def _import_metaphor_search() -> Any:
     return MetaphorSearchResults
 
 
+def _import_nasa_tool() -> Any:
+    from langchain.tools.nasa.tool import NasaAction
+
+    return NasaAction
+
+
 def _import_office365_create_draft_message() -> Any:
     from langchain.tools.office365.create_draft_message import O365CreateDraftMessage
 
@@ -831,6 +837,8 @@ def __getattr__(name: str) -> Any:
         return _import_merriam_webster_tool()
     elif name == "MetaphorSearchResults":
         return _import_metaphor_search()
+    elif name == "NasaAction":
+        return _import_nasa_tool()
     elif name == "O365CreateDraftMessage":
         return _import_office365_create_draft_message()
     elif name == "O365SearchEvents":
@@ -1030,6 +1038,7 @@ def __getattr__(name: str) -> Any:
     "MerriamWebsterQueryRun",
     "MetaphorSearchResults",
     "MoveFileTool",
+    "NasaAction",
     "NavigateBackTool",
     "NavigateTool",
     "O365CreateDraftMessage",

diff --git a/libs/langchain/langchain/tools/nasa/__init__.py b/libs/langchain/langchain/tools/nasa/__init__.py
diff --git a/libs/langchain/langchain/tools/nasa/prompt.py b/libs/langchain/langchain/tools/nasa/prompt.py
@@ -0,0 +1,82 @@
+# flake8: noqa
+NASA_SEARCH_PROMPT = """
+    This tool is a wrapper around NASA's search API, useful when you need to search through NASA's Image and Video Library. 
+    The input to this tool is a query specified by the user, and will be passed into NASA's `search` function.
+    
+    At least one parameter must be provided.
+
+    There are optional parameters that can be passed by the user based on their query
+    specifications. Each item in this list contains pound sign (#) separated values, the first value is the parameter name, 
+    the second value is the datatype and the third value is the description: {{
+
+        - q#string#Free text search terms to compare to all indexed metadata.
+        - center#string#NASA center which published the media.
+        - description#string#Terms to search for in “Description” fields.
+        - description_508#string#Terms to search for in “508 Description” fields.
+        - keywords #string#Terms to search for in “Keywords” fields. Separate multiple values with commas.
+        - location #string#Terms to search for in “Location” fields.
+        - media_type#string#Media types to restrict the search to. Available types: [“image”,“video”, “audio”]. Separate multiple values with commas.
+        - nasa_id #string#The media asset’s NASA ID.
+        - page#integer#Page number, starting at 1, of results to get.-
+        - page_size#integer#Number of results per page. Default: 100.
+        - photographer#string#The primary photographer’s name.
+        - secondary_creator#string#A secondary photographer/videographer’s name.
+        - title #string#Terms to search for in “Title” fields.
+        - year_start#string#The start year for results. Format: YYYY.
+        - year_end #string#The end year for results. Format: YYYY.
+
+    }}
+    
+    Below are several task descriptions along with their respective input examples.
+    Task: get the 2nd page of image and video content starting from the year 2002 to 2010
+    Example Input: {{"year_start":  "2002", "year_end":  "2010", "page": 2}}
+    
+    Task: get the image and video content of saturn photographed by John Appleseed
+    Example Input: {{"q": "saturn", "photographer": "John Appleseed"}}
+    
+    Task: search for Meteor Showers with description "Search Description" with media type image
+    Example Input: {{"q": "Meteor Shower", "description": "Search Description", "media_type": "image"}}
+    
+    Task: get the image and video content from year 2008 to 2010 from Kennedy Center
+    Example Input: {{"year_start":  "2002", "year_end":  "2010", "location": "Kennedy Center}}
+    """
+
+
+NASA_MANIFEST_PROMPT = """
+    This tool is a wrapper around NASA's media asset manifest API, useful when you need to retrieve a media 
+    asset's manifest. The input to this tool should include a string representing a NASA ID for a media asset that the user is trying to get the media asset manifest data for. The NASA ID will be passed as a string into NASA's `get_media_metadata_manifest` function.
+
+    The following list are some examples of NASA IDs for a media asset that you can use to better extract the NASA ID from the input string to the tool.
+    - GSFC_20171102_Archive_e000579
+    - Launch-Sound_Delta-PAM-Random-Commentary
+    - iss066m260341519_Expedition_66_Education_Inflight_with_Random_Lake_School_District_220203
+    - 6973610
+    - GRC-2020-CM-0167.4
+    - Expedition_55_Inflight_Japan_VIP_Event_May_31_2018_659970
+    - NASA 60th_SEAL_SLIVER_150DPI
+"""
+
+NASA_METADATA_PROMPT = """
+    This tool is a wrapper around NASA's media asset metadata location API, useful when you need to retrieve the media asset's metadata. The input to this tool should include a string representing a NASA ID for a media asset that the user is trying to get the media asset metadata location for. The NASA ID will be passed as a string into NASA's `get_media_metadata_manifest` function.
+
+    The following list are some examples of NASA IDs for a media asset that you can use to better extract the NASA ID from the input string to the tool.
+    - GSFC_20171102_Archive_e000579
+    - Launch-Sound_Delta-PAM-Random-Commentary
+    - iss066m260341519_Expedition_66_Education_Inflight_with_Random_Lake_School_District_220203
+    - 6973610
+    - GRC-2020-CM-0167.4
+    - Expedition_55_Inflight_Japan_VIP_Event_May_31_2018_659970
+    - NASA 60th_SEAL_SLIVER_150DPI
+"""
+
+NASA_CAPTIONS_PROMPT = """
+    This tool is a wrapper around NASA's video assests caption location API, useful when you need 
+    to retrieve the location of the captions of a specific video. The input to this tool should include a string representing a NASA ID for a video media asset that the user is trying to get the get the location of the captions for. The NASA ID will be passed as a string into NASA's `get_media_metadata_manifest` function.
+
+    The following list are some examples of NASA IDs for a video asset that you can use to better extract the NASA ID from the input string to the tool.
+    - 2017-08-09 - Video File RS-25 Engine Test
+    - 20180415-TESS_Social_Briefing
+    - 201_TakingWildOutOfWildfire
+    - 2022-H1_V_EuropaClipper-4
+    - 2022_0429_Recientemente
+"""
diff --git a/libs/langchain/langchain/tools/nasa/tool.py b/libs/langchain/langchain/tools/nasa/tool.py
@@ -0,0 +1,28 @@
+"""
+This tool allows agents to interact with the NASA API, specifically 
+the the NASA Image & Video Library and Exoplanet
+"""
+
+from typing import Optional
+
+from langchain.callbacks.manager import CallbackManagerForToolRun
+from langchain.pydantic_v1 import Field
+from langchain.tools.base import BaseTool
+from langchain.utilities.nasa import NasaAPIWrapper
+
+
+class NasaAction(BaseTool):
+    """Tool that queries the Atlassian Jira API."""
+
+    api_wrapper: NasaAPIWrapper = Field(default_factory=NasaAPIWrapper)
+    mode: str
+    name: str = ""
+    description: str = ""
+
+    def _run(
+        self,
+        instructions: str,
+        run_manager: Optional[CallbackManagerForToolRun] = None,
+    ) -> str:
+        """Use the NASA API to run an operation."""
+        return self.api_wrapper.run(self.mode, instructions)
diff --git a/libs/langchain/langchain/utilities/__init__.py b/libs/langchain/langchain/utilities/__init__.py
@@ -260,6 +260,12 @@ def _import_zapier() -> Any:
     return ZapierNLAWrapper
 
 
+def _import_nasa() -> Any:
+    from langchain.utilities.nasa import NasaAPIWrapper
+
+    return NasaAPIWrapper
+
+
 def __getattr__(name: str) -> Any:
     if name == "AlphaVantageAPIWrapper":
         return _import_alpha_vantage()
@@ -307,6 +313,8 @@ def __getattr__(name: str) -> Any:
         return _import_merriam_webster()
     elif name == "MetaphorSearchAPIWrapper":
         return _import_metaphor_search()
+    elif name == "NasaAPIWrapper":
+        return _import_nasa()
     elif name == "OpenWeatherMapAPIWrapper":
         return _import_openweathermap()
     elif name == "OutlineAPIWrapper":
@@ -373,6 +381,7 @@ def __getattr__(name: str) -> Any:
     "MaxComputeAPIWrapper",
     "MerriamWebsterAPIWrapper",
     "MetaphorSearchAPIWrapper",
+    "NasaAPIWrapper",
     "OpenWeatherMapAPIWrapper",
     "OutlineAPIWrapper",
     "Portkey",