diff --git a/docs/docs/integrations/toolkits/nasa.ipynb b/docs/docs/integrations/toolkits/nasa.ipynb new file mode 100644 index 00000000000000..09580d2396e363 --- /dev/null +++ b/docs/docs/integrations/toolkits/nasa.ipynb @@ -0,0 +1,108 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e6fd05db-21c2-4227-9900-0840bc62cb31", + "metadata": {}, + "source": [ + "# NASA\n", + "\n", + "This notebook shows how to use agents to interact with the NASA toolkit. The toolkit provides access to the NASA Image and Video Library API, with potential to expand and include other accessible NASA APIs in future iterations.\n", + "\n", + "**Note: NASA Image and Video Library search queries can result in large responses when the number of desired media results is not specified. Consider this prior to using the agent with LLM token credits.**" + ] + }, + { + "cell_type": "markdown", + "id": "7d93e6bd-03d7-4d3c-b915-8b73164e2ad8", + "metadata": {}, + "source": [ + "## Example Use:\n", + "---\n", + "### Initializing the agent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "648a2cb2-308e-4b2e-9b73-37109be4e258", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.agents import AgentType, initialize_agent\n", + "from langchain.agents.agent_toolkits.nasa.toolkit import NasaToolkit\n", + "from langchain.llms import OpenAI\n", + "from langchain.utilities.nasa import NasaAPIWrapper\n", + "\n", + "llm = OpenAI(temperature=0, openai_api_key=\"\")\n", + "nasa = NasaAPIWrapper()\n", + "toolkit = NasaToolkit.from_nasa_api_wrapper(nasa)\n", + "agent = initialize_agent(\n", + " toolkit.get_tools(), llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "71f05fc9-d80d-4614-b9a3-e0a5e43cbbbb", + "metadata": {}, + "source": [ + "### Querying media assets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b97409f3-dc87-425d-b555-406cf8466a28", + "metadata": {}, + "outputs": [], + "source": [ + "agent.run(\n", + " \"Can you find three pictures of the moon published between the years 2014 and 2020?\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a86ce5ff-de45-4206-86ca-07ae03f36bdf", + "metadata": {}, + "source": [ + "### Querying details about media assets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80e86b49-749e-4026-b025-db32ed0bcc7e", + "metadata": {}, + "outputs": [], + "source": [ + "output = agent.run(\n", + " \"I've just queried an image of the moon with the NASA id NHQ_2019_0311_Go Forward to the Moon.\"\n", + " \" Where can I find the metadata manifest for this asset?\"\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/libs/langchain/langchain/agents/agent_toolkits/__init__.py b/libs/langchain/langchain/agents/agent_toolkits/__init__.py index b40975cc732997..062ab1f3176331 100644 --- a/libs/langchain/langchain/agents/agent_toolkits/__init__.py +++ b/libs/langchain/langchain/agents/agent_toolkits/__init__.py @@ -34,6 +34,7 @@ from langchain.agents.agent_toolkits.json.base import create_json_agent from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit from langchain.agents.agent_toolkits.multion.toolkit import MultionToolkit +from langchain.agents.agent_toolkits.nasa.toolkit import NasaToolkit from langchain.agents.agent_toolkits.nla.toolkit import NLAToolkit from langchain.agents.agent_toolkits.office365.toolkit import O365Toolkit from langchain.agents.agent_toolkits.openapi.base import create_openapi_agent @@ -93,6 +94,7 @@ def __getattr__(name: str) -> Any: "JiraToolkit", "JsonToolkit", "MultionToolkit", + "NasaToolkit", "NLAToolkit", "O365Toolkit", "OpenAPIToolkit", diff --git a/libs/langchain/langchain/agents/agent_toolkits/nasa/__init__.py b/libs/langchain/langchain/agents/agent_toolkits/nasa/__init__.py new file mode 100644 index 00000000000000..a13c3ec706c6d8 --- /dev/null +++ b/libs/langchain/langchain/agents/agent_toolkits/nasa/__init__.py @@ -0,0 +1 @@ +"""NASA Toolkit""" diff --git a/libs/langchain/langchain/agents/agent_toolkits/nasa/toolkit.py b/libs/langchain/langchain/agents/agent_toolkits/nasa/toolkit.py new file mode 100644 index 00000000000000..0410507e0f7f5f --- /dev/null +++ b/libs/langchain/langchain/agents/agent_toolkits/nasa/toolkit.py @@ -0,0 +1,57 @@ +from typing import Dict, List + +from langchain.agents.agent_toolkits.base import BaseToolkit +from langchain.tools import BaseTool +from langchain.tools.nasa.prompt import ( + NASA_CAPTIONS_PROMPT, + NASA_MANIFEST_PROMPT, + NASA_METADATA_PROMPT, + NASA_SEARCH_PROMPT, +) +from langchain.tools.nasa.tool import NasaAction +from langchain.utilities.nasa import NasaAPIWrapper + + +class NasaToolkit(BaseToolkit): + """Nasa Toolkit.""" + + tools: List[BaseTool] = [] + + @classmethod + def from_nasa_api_wrapper(cls, nasa_api_wrapper: NasaAPIWrapper) -> "NasaToolkit": + operations: List[Dict] = [ + { + "mode": "search_media", + "name": "Search NASA Image and Video Library media", + "description": NASA_SEARCH_PROMPT, + }, + { + "mode": "get_media_metadata_manifest", + "name": "Get NASA Image and Video Library media metadata manifest", + "description": NASA_MANIFEST_PROMPT, + }, + { + "mode": "get_media_metadata_location", + "name": "Get NASA Image and Video Library media metadata location", + "description": NASA_METADATA_PROMPT, + }, + { + "mode": "get_video_captions_location", + "name": "Get NASA Image and Video Library video captions location", + "description": NASA_CAPTIONS_PROMPT, + }, + ] + tools = [ + NasaAction( + name=action["name"], + description=action["description"], + mode=action["mode"], + api_wrapper=nasa_api_wrapper, + ) + for action in operations + ] + return cls(tools=tools) + + def get_tools(self) -> List[BaseTool]: + """Get the tools in the toolkit.""" + return self.tools diff --git a/libs/langchain/langchain/tools/__init__.py b/libs/langchain/langchain/tools/__init__.py index 7422c136620953..cc8c37ee6a6326 100644 --- a/libs/langchain/langchain/tools/__init__.py +++ b/libs/langchain/langchain/tools/__init__.py @@ -338,6 +338,12 @@ def _import_metaphor_search() -> Any: return MetaphorSearchResults +def _import_nasa_tool() -> Any: + from langchain.tools.nasa.tool import NasaAction + + return NasaAction + + def _import_office365_create_draft_message() -> Any: from langchain.tools.office365.create_draft_message import O365CreateDraftMessage @@ -831,6 +837,8 @@ def __getattr__(name: str) -> Any: return _import_merriam_webster_tool() elif name == "MetaphorSearchResults": return _import_metaphor_search() + elif name == "NasaAction": + return _import_nasa_tool() elif name == "O365CreateDraftMessage": return _import_office365_create_draft_message() elif name == "O365SearchEvents": @@ -1030,6 +1038,7 @@ def __getattr__(name: str) -> Any: "MerriamWebsterQueryRun", "MetaphorSearchResults", "MoveFileTool", + "NasaAction", "NavigateBackTool", "NavigateTool", "O365CreateDraftMessage", diff --git a/libs/langchain/langchain/tools/nasa/__init__.py b/libs/langchain/langchain/tools/nasa/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/libs/langchain/langchain/tools/nasa/prompt.py b/libs/langchain/langchain/tools/nasa/prompt.py new file mode 100644 index 00000000000000..4c7a3846a7e839 --- /dev/null +++ b/libs/langchain/langchain/tools/nasa/prompt.py @@ -0,0 +1,82 @@ +# flake8: noqa +NASA_SEARCH_PROMPT = """ + This tool is a wrapper around NASA's search API, useful when you need to search through NASA's Image and Video Library. + The input to this tool is a query specified by the user, and will be passed into NASA's `search` function. + + At least one parameter must be provided. + + There are optional parameters that can be passed by the user based on their query + specifications. Each item in this list contains pound sign (#) separated values, the first value is the parameter name, + the second value is the datatype and the third value is the description: {{ + + - q#string#Free text search terms to compare to all indexed metadata. + - center#string#NASA center which published the media. + - description#string#Terms to search for in “Description” fields. + - description_508#string#Terms to search for in “508 Description” fields. + - keywords #string#Terms to search for in “Keywords” fields. Separate multiple values with commas. + - location #string#Terms to search for in “Location” fields. + - media_type#string#Media types to restrict the search to. Available types: [“image”,“video”, “audio”]. Separate multiple values with commas. + - nasa_id #string#The media asset’s NASA ID. + - page#integer#Page number, starting at 1, of results to get.- + - page_size#integer#Number of results per page. Default: 100. + - photographer#string#The primary photographer’s name. + - secondary_creator#string#A secondary photographer/videographer’s name. + - title #string#Terms to search for in “Title” fields. + - year_start#string#The start year for results. Format: YYYY. + - year_end #string#The end year for results. Format: YYYY. + + }} + + Below are several task descriptions along with their respective input examples. + Task: get the 2nd page of image and video content starting from the year 2002 to 2010 + Example Input: {{"year_start": "2002", "year_end": "2010", "page": 2}} + + Task: get the image and video content of saturn photographed by John Appleseed + Example Input: {{"q": "saturn", "photographer": "John Appleseed"}} + + Task: search for Meteor Showers with description "Search Description" with media type image + Example Input: {{"q": "Meteor Shower", "description": "Search Description", "media_type": "image"}} + + Task: get the image and video content from year 2008 to 2010 from Kennedy Center + Example Input: {{"year_start": "2002", "year_end": "2010", "location": "Kennedy Center}} + """ + + +NASA_MANIFEST_PROMPT = """ + This tool is a wrapper around NASA's media asset manifest API, useful when you need to retrieve a media + asset's manifest. The input to this tool should include a string representing a NASA ID for a media asset that the user is trying to get the media asset manifest data for. The NASA ID will be passed as a string into NASA's `get_media_metadata_manifest` function. + + The following list are some examples of NASA IDs for a media asset that you can use to better extract the NASA ID from the input string to the tool. + - GSFC_20171102_Archive_e000579 + - Launch-Sound_Delta-PAM-Random-Commentary + - iss066m260341519_Expedition_66_Education_Inflight_with_Random_Lake_School_District_220203 + - 6973610 + - GRC-2020-CM-0167.4 + - Expedition_55_Inflight_Japan_VIP_Event_May_31_2018_659970 + - NASA 60th_SEAL_SLIVER_150DPI +""" + +NASA_METADATA_PROMPT = """ + This tool is a wrapper around NASA's media asset metadata location API, useful when you need to retrieve the media asset's metadata. The input to this tool should include a string representing a NASA ID for a media asset that the user is trying to get the media asset metadata location for. The NASA ID will be passed as a string into NASA's `get_media_metadata_manifest` function. + + The following list are some examples of NASA IDs for a media asset that you can use to better extract the NASA ID from the input string to the tool. + - GSFC_20171102_Archive_e000579 + - Launch-Sound_Delta-PAM-Random-Commentary + - iss066m260341519_Expedition_66_Education_Inflight_with_Random_Lake_School_District_220203 + - 6973610 + - GRC-2020-CM-0167.4 + - Expedition_55_Inflight_Japan_VIP_Event_May_31_2018_659970 + - NASA 60th_SEAL_SLIVER_150DPI +""" + +NASA_CAPTIONS_PROMPT = """ + This tool is a wrapper around NASA's video assests caption location API, useful when you need + to retrieve the location of the captions of a specific video. The input to this tool should include a string representing a NASA ID for a video media asset that the user is trying to get the get the location of the captions for. The NASA ID will be passed as a string into NASA's `get_media_metadata_manifest` function. + + The following list are some examples of NASA IDs for a video asset that you can use to better extract the NASA ID from the input string to the tool. + - 2017-08-09 - Video File RS-25 Engine Test + - 20180415-TESS_Social_Briefing + - 201_TakingWildOutOfWildfire + - 2022-H1_V_EuropaClipper-4 + - 2022_0429_Recientemente +""" diff --git a/libs/langchain/langchain/tools/nasa/tool.py b/libs/langchain/langchain/tools/nasa/tool.py new file mode 100644 index 00000000000000..058944858cbebc --- /dev/null +++ b/libs/langchain/langchain/tools/nasa/tool.py @@ -0,0 +1,28 @@ +""" +This tool allows agents to interact with the NASA API, specifically +the the NASA Image & Video Library and Exoplanet +""" + +from typing import Optional + +from langchain.callbacks.manager import CallbackManagerForToolRun +from langchain.pydantic_v1 import Field +from langchain.tools.base import BaseTool +from langchain.utilities.nasa import NasaAPIWrapper + + +class NasaAction(BaseTool): + """Tool that queries the Atlassian Jira API.""" + + api_wrapper: NasaAPIWrapper = Field(default_factory=NasaAPIWrapper) + mode: str + name: str = "" + description: str = "" + + def _run( + self, + instructions: str, + run_manager: Optional[CallbackManagerForToolRun] = None, + ) -> str: + """Use the NASA API to run an operation.""" + return self.api_wrapper.run(self.mode, instructions) diff --git a/libs/langchain/langchain/utilities/__init__.py b/libs/langchain/langchain/utilities/__init__.py index 8c8242e3a6a9c3..986f99de295bfb 100644 --- a/libs/langchain/langchain/utilities/__init__.py +++ b/libs/langchain/langchain/utilities/__init__.py @@ -260,6 +260,12 @@ def _import_zapier() -> Any: return ZapierNLAWrapper +def _import_nasa() -> Any: + from langchain.utilities.nasa import NasaAPIWrapper + + return NasaAPIWrapper + + def __getattr__(name: str) -> Any: if name == "AlphaVantageAPIWrapper": return _import_alpha_vantage() @@ -307,6 +313,8 @@ def __getattr__(name: str) -> Any: return _import_merriam_webster() elif name == "MetaphorSearchAPIWrapper": return _import_metaphor_search() + elif name == "NasaAPIWrapper": + return _import_nasa() elif name == "OpenWeatherMapAPIWrapper": return _import_openweathermap() elif name == "OutlineAPIWrapper": @@ -373,6 +381,7 @@ def __getattr__(name: str) -> Any: "MaxComputeAPIWrapper", "MerriamWebsterAPIWrapper", "MetaphorSearchAPIWrapper", + "NasaAPIWrapper", "OpenWeatherMapAPIWrapper", "OutlineAPIWrapper", "Portkey", diff --git a/libs/langchain/langchain/utilities/nasa.py b/libs/langchain/langchain/utilities/nasa.py new file mode 100644 index 00000000000000..609a854f17db99 --- /dev/null +++ b/libs/langchain/langchain/utilities/nasa.py @@ -0,0 +1,52 @@ +"""Util that calls several NASA APIs.""" +import json + +import requests + +from langchain.pydantic_v1 import BaseModel + +IMAGE_AND_VIDEO_LIBRARY_URL = "https://images-api.nasa.gov" + + +class NasaAPIWrapper(BaseModel): + def get_media(self, query: str) -> str: + params = json.loads(query) + if params.get("q"): + queryText = params["q"] + params.pop("q") + else: + queryText = "" + response = requests.get( + IMAGE_AND_VIDEO_LIBRARY_URL + "/search?q=" + queryText, params=params + ) + data = response.json() + return data + + def get_media_metadata_manifest(self, query: str) -> str: + response = requests.get(IMAGE_AND_VIDEO_LIBRARY_URL + "/asset/" + query) + return response.json() + + def get_media_metadata_location(self, query: str) -> str: + response = requests.get(IMAGE_AND_VIDEO_LIBRARY_URL + "/metadata/" + query) + return response.json() + + def get_video_captions_location(self, query: str) -> str: + response = requests.get(IMAGE_AND_VIDEO_LIBRARY_URL + "/captions/" + query) + return response.json() + + def run(self, mode: str, query: str) -> str: + if mode == "search_media": + output = self.get_media(query) + elif mode == "get_media_metadata_manifest": + output = self.get_media_metadata_manifest(query) + elif mode == "get_media_metadata_location": + output = self.get_media_metadata_location(query) + elif mode == "get_video_captions_location": + output = self.get_video_captions_location(query) + else: + output = f"ModeError: Got unexpected mode {mode}." + + try: + return json.dumps(output) + except Exception: + return str(output) diff --git a/libs/langchain/tests/integration_tests/utilities/test_nasa.py b/libs/langchain/tests/integration_tests/utilities/test_nasa.py new file mode 100644 index 00000000000000..c605626afd8658 --- /dev/null +++ b/libs/langchain/tests/integration_tests/utilities/test_nasa.py @@ -0,0 +1,32 @@ +"""Integration test for NASA API Wrapper.""" +from langchain.utilities.nasa import NasaAPIWrapper + + +def test_media_search() -> None: + """Test for NASA Image and Video Library media search""" + nasa = NasaAPIWrapper() + query = '{"q": "saturn", + "year_start": "2002", "year_end": "2010", "page": 2}' + output = nasa.run("search_media", query) + assert output is not None + assert "collection" in output + + +def test_get_media_metadata_manifest() -> None: + """Test for retrieving media metadata manifest from NASA Image and Video Library""" + nasa = NasaAPIWrapper() + output = nasa.run("get_media_metadata_manifest", "2022_0707_Recientemente") + assert output is not None + + +def test_get_media_metadata_location() -> None: + """Test for retrieving media metadata location from NASA Image and Video Library""" + nasa = NasaAPIWrapper() + output = nasa.run("get_media_metadata_location", "as11-40-5874") + assert output is not None + + +def test_get_video_captions_location() -> None: + """Test for retrieving video captions location from NASA Image and Video Library""" + nasa = NasaAPIWrapper() + output = nasa.run("get_video_captions_location", "172_ISS-Slosh.sr") + assert output is not None diff --git a/libs/langchain/tests/unit_tests/tools/test_imports.py b/libs/langchain/tests/unit_tests/tools/test_imports.py index fb30a00393c0cb..dbd9bd48555ff6 100644 --- a/libs/langchain/tests/unit_tests/tools/test_imports.py +++ b/libs/langchain/tests/unit_tests/tools/test_imports.py @@ -68,6 +68,7 @@ "ListSparkSQLTool", "MetaphorSearchResults", "MoveFileTool", + "NasaAction", "NavigateBackTool", "NavigateTool", "O365CreateDraftMessage", diff --git a/libs/langchain/tests/unit_tests/tools/test_public_api.py b/libs/langchain/tests/unit_tests/tools/test_public_api.py index b0b3844c964892..4d7cf8f4d0a861 100644 --- a/libs/langchain/tests/unit_tests/tools/test_public_api.py +++ b/libs/langchain/tests/unit_tests/tools/test_public_api.py @@ -70,6 +70,7 @@ "MerriamWebsterQueryRun", "MetaphorSearchResults", "MoveFileTool", + "NasaAction", "NavigateBackTool", "NavigateTool", "O365CreateDraftMessage", diff --git a/libs/langchain/tests/unit_tests/utilities/test_imports.py b/libs/langchain/tests/unit_tests/utilities/test_imports.py index 9650889869f551..49fddcff655221 100644 --- a/libs/langchain/tests/unit_tests/utilities/test_imports.py +++ b/libs/langchain/tests/unit_tests/utilities/test_imports.py @@ -23,6 +23,7 @@ "LambdaWrapper", "MaxComputeAPIWrapper", "MetaphorSearchAPIWrapper", + "NasaAPIWrapper", "OpenWeatherMapAPIWrapper", "OutlineAPIWrapper", "Portkey",