opensearch-multimodalsearch/multimodal_search_with_opensearch_service.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "082aa845",
   "metadata": {},
   "source": [
    "# Build Multimodal Search with Amazon OpenSearch service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9d38375",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Install python packages\n",
    "%pip install requests_aws4auth boto3 pillow opensearch_py ipywidgets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "647587db",
   "metadata": {},
   "source": [
    "## 1. Setup OpenSearch client\n",
    "\n",
    "NOTE: Values enclosed within <> are the parameters that you should configure\n",
    "\n",
    "You should also setup opensearch authentication in the code "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57ed12ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from requests_aws4auth import AWS4Auth\n",
    "import boto3\n",
    "import json\n",
    "from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth\n",
    "from requests.auth import HTTPBasicAuth\n",
    "\n",
    "\n",
    "headers = {\"Content-Type\": \"application/json\"}\n",
    "host = <opensearch domain endpoint without 'https://'> #example: \"opensearch-domain-endpoint.us-east-1.es.amazonaws.com\"\n",
    "service = 'es'\n",
    "region = <aws-region> #example: \"us-east-1\"\n",
    "\n",
    "\n",
    "# 3.Use OpenSearch master credentials that you created while creating the OpenSearch domain\n",
    "awsauth = HTTPBasicAuth(<master username>,<master password>)\n",
    "\n",
    "\n",
    "#Initialise OpenSearch-py client\n",
    "aos_client = OpenSearch(\n",
    "    hosts = [{'host': host, 'port': 443}],\n",
    "    http_auth = awsauth,\n",
    "    use_ssl = True,\n",
    "    connection_class = RequestsHttpConnection\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b636855",
   "metadata": {},
   "source": [
    "## 2. Download the dataset (.gz) and extract the .gz file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aca8ce36",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import urllib.request\n",
    "import tarfile\n",
    "\n",
    "os.makedirs('tmp/images', exist_ok = True)\n",
    "metadata_file = urllib.request.urlretrieve('https://aws-blogs-artifacts-public.s3.amazonaws.com/BDB-3144/products-data.yml', 'tmp/images/products.yaml')\n",
    "img_filename,headers= urllib.request.urlretrieve('https://aws-blogs-artifacts-public.s3.amazonaws.com/BDB-3144/images.tar.gz', 'tmp/images/images.tar.gz')              \n",
    "print(img_filename)\n",
    "file = tarfile.open('tmp/images/images.tar.gz')\n",
    "file.extractall('tmp/images/')\n",
    "file.close()\n",
    "#remove images.tar.gz\n",
    "os.remove('tmp/images/images.tar.gz')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31b73212",
   "metadata": {},
   "source": [
    "# Do you have the model_id already ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80a08450",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ipywidgets as widgets\n",
    "from ipywidgets import Dropdown\n",
    "\n",
    "model_id_selection = [\n",
    "    \"I have the model_id already from the cloudformation\",\n",
    "    \"I don't have the model_id already fromthe cloudformation\",\n",
    "]\n",
    "\n",
    "model_id_dropdown = widgets.Dropdown(\n",
    "    options=model_id_selection,\n",
    "    value=\"I have the model_id already from the cloudformation\",\n",
    "    description=\"Is model_id already created ?\",\n",
    "    style={\"description_width\": \"initial\"},\n",
    "    layout={\"width\": \"max-content\"},\n",
    ")\n",
    "\n",
    "display(model_id_dropdown)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c7eb014",
   "metadata": {},
   "outputs": [],
   "source": [
    "if(model_id_dropdown.value == \"I have the model_id already from the cloudformation\"):\n",
    "    model_id = (input(\"Enter the model_id and press ENTER : \"))\n",
    "    print(\"model_id: \"+model_id)\n",
    "else:\n",
    "    model_id = \"\"\n",
    "    print(\"model_id: '' \\nCreate the model id by running next step\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f2c5a23",
   "metadata": {},
   "source": [
    "# Run the below step 3 only if you DO NOT have the model_id \n",
    "\n",
    "# If you have the model_id already created from the cloudformation template, skip step 3 and proceed from step 4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f468131",
   "metadata": {},
   "source": [
    "## 3. Create the OpenSearch Bedrock ML connector\n",
    "\n",
    "you need to change **\"iam-role-arn\"** below with the ARN of the IAM role that has permissions to talk to OpenSearch and mapped as back-end role in OpenSearch dashboards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a113cb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Register repository\n",
    "if(model_id == ''):\n",
    "    path = '_plugins/_ml/connectors/_create'\n",
    "    url = 'https://'+host + '/' + path\n",
    "\n",
    "    payload = {\n",
    "       \"name\": \"sagemaker: embedding\",\n",
    "       \"description\": \"Test connector for Sagemaker embedding model\",\n",
    "       \"version\": 1,\n",
    "       \"protocol\": \"aws_sigv4\",\n",
    "       \"credential\": {\n",
    "          \"roleArn\": \"<iam-role-arn>\"\n",
    "       },\n",
    "       \"parameters\": {\n",
    "          \"region\": region,\n",
    "          \"service_name\": \"bedrock\"\n",
    "       },\n",
    "       \"actions\": [\n",
    "          {\n",
    "             \"action_type\": \"predict\",\n",
    "             \"method\": \"POST\",\n",
    "           \"headers\": {\n",
    "            \"content-type\": \"application/json\",\n",
    "            \"x-amz-content-sha256\": \"required\"\n",
    "          },\n",
    "\n",
    "        \"url\": \"https://bedrock-runtime.\"+region+\".amazonaws.com/model/amazon.titan-embed-image-v1/invoke\",\n",
    "         \"request_body\": \"{ \\\"inputText\\\": \\\"${parameters.inputText:-null}\\\", \\\"inputImage\\\": \\\"${parameters.inputImage:-null}\\\" }\",\n",
    "          \"pre_process_function\": \"\\n    StringBuilder parametersBuilder = new StringBuilder(\\\"{\\\");\\n    if (params.text_docs.length > 0 && params.text_docs[0] != null) {\\n      parametersBuilder.append(\\\"\\\\\\\"inputText\\\\\\\":\\\");\\n      parametersBuilder.append(\\\"\\\\\\\"\\\");\\n      parametersBuilder.append(params.text_docs[0]);\\n      parametersBuilder.append(\\\"\\\\\\\"\\\");\\n      \\n      if (params.text_docs.length > 1 && params.text_docs[1] != null) {\\n        parametersBuilder.append(\\\",\\\");\\n      }\\n    }\\n    \\n    \\n    if (params.text_docs.length > 1 && params.text_docs[1] != null) {\\n      parametersBuilder.append(\\\"\\\\\\\"inputImage\\\\\\\":\\\");\\n      parametersBuilder.append(\\\"\\\\\\\"\\\");\\n      parametersBuilder.append(params.text_docs[1]);\\n      parametersBuilder.append(\\\"\\\\\\\"\\\");\\n    }\\n    parametersBuilder.append(\\\"}\\\");\\n    \\n    return  \\\"{\\\" +\\\"\\\\\\\"parameters\\\\\\\":\\\" + parametersBuilder + \\\"}\\\";\",\n",
    "          \"post_process_function\": \"\\n      def name = \\\"sentence_embedding\\\";\\n      def dataType = \\\"FLOAT32\\\";\\n      if (params.embedding == null || params.embedding.length == 0) {\\n          return null;\\n      }\\n      def shape = [params.embedding.length];\\n      def json = \\\"{\\\" +\\n                 \\\"\\\\\\\"name\\\\\\\":\\\\\\\"\\\" + name + \\\"\\\\\\\",\\\" +\\n                 \\\"\\\\\\\"data_type\\\\\\\":\\\\\\\"\\\" + dataType + \\\"\\\\\\\",\\\" +\\n                 \\\"\\\\\\\"shape\\\\\\\":\\\" + shape + \\\",\\\" +\\n                 \\\"\\\\\\\"data\\\\\\\":\\\" + params.embedding +\\n                 \\\"}\\\";\\n      return json;\\n    \"\n",
    "          }\n",
    "       ]\n",
    "    }\n",
    "    headers = {\"Content-Type\": \"application/json\"}\n",
    "\n",
    "    r = requests.post(url, auth=awsauth, json=payload, headers=headers)\n",
    "    print(r.status_code)\n",
    "    print(r.text)\n",
    "    connector_id = json.loads(r.text)[\"connector_id\"]\n",
    "\n",
    "    # Register the model\n",
    "    path = '_plugins/_ml/models/_register'\n",
    "    url = 'https://'+host + '/' + path\n",
    "    payload = { \"name\": \"Bedrock Multimodal embeddings model\",\n",
    "    \"function_name\": \"remote\",\n",
    "    \"description\": \"Bedrock Multimodal embeddings model\",\n",
    "    \"connector_id\": connector_id}\n",
    "    r = requests.post(url, auth=awsauth, json=payload, headers=headers)\n",
    "    model_id = json.loads(r.text)[\"model_id\"]\n",
    "    print(\"Model registered under model_id: \"+model_id)\n",
    "\n",
    "    # Deploy the model\n",
    "    path = '_plugins/_ml/models/'+model_id+'/_deploy'\n",
    "    url = 'https://'+host + '/' + path\n",
    "    r = requests.post(url, auth=awsauth, headers=headers)\n",
    "    deploy_status = json.loads(r.text)[\"status\"]\n",
    "    print(\"Deployment status of the model, \"+model_id+\" : \"+deploy_status)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "852500c8",
   "metadata": {},
   "source": [
    "## 4. Test the OpenSearch - Bedrock integration with a test input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39fac4d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import base64\n",
    "\n",
    "path = '_plugins/_ml/models/'+model_id+'/_predict'\n",
    "url = 'https://'+host + '/' + path\n",
    "img = \"tmp/images/footwear/2d2d8ec8-4806-42a7-b8ba-ceb15c1c7e84.jpg\"\n",
    "with open(img, \"rb\") as image_file:\n",
    "    input_image_binary = base64.b64encode(image_file.read()).decode(\"utf8\")\n",
    "    \n",
    "payload = {\n",
    "\"parameters\": {\n",
    "\"inputText\": \"Sleek, stylish black sneakers made for urban exploration. With fashionable looks and comfortable design, these sneakers keep your feet looking great while you walk the city streets in style\",\n",
    "\"inputImage\":input_image_binary\n",
    "}\n",
    "}\n",
    "r = requests.post(url, auth=awsauth, json=payload, headers=headers)\n",
    "embed = json.loads(r.text)['inference_results'][0]['output'][0]['data'][0:10]\n",
    "shape = json.loads(r.text)['inference_results'][0]['output'][0]['shape'][0]\n",
    "print(\"First 10 dimensions:\")\n",
    "print(str(embed))\n",
    "print(\"\\n\")\n",
    "print(\"Total: \"+str(shape)+\" dimensions\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05cd7e48",
   "metadata": {},
   "source": [
    "## 5. Create the OpenSearch ingest pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5951fc54",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"_ingest/pipeline/bedrock-multimodal-ingest-pipeline\"\n",
    "url = 'https://'+host + '/' + path\n",
    "payload = {\n",
    "\"description\": \"A text/image embedding pipeline\",\n",
    "\"processors\": [\n",
    "{\n",
    "\"text_image_embedding\": {\n",
    "\"model_id\":model_id,\n",
    "\"embedding\": \"vector_embedding\",\n",
    "\"field_map\": {\n",
    "\"text\": \"product_description\",\n",
    "\"image\": \"image_binary\"\n",
    "}}}]}\n",
    "r = requests.put(url, auth=awsauth, json=payload, headers=headers)\n",
    "print(r.status_code)\n",
    "print(r.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f358ebf",
   "metadata": {},
   "source": [
    "## 6. Create the k-NN index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f85afd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"bedrock-multimodal-demostore-search-index\"\n",
    "url = 'https://'+host + '/' + path\n",
    "\n",
    "#this will delete the index if already exists\n",
    "requests.delete(url, auth=awsauth, json=payload, headers=headers)\n",
    "\n",
    "payload = {\n",
    "  \"settings\": {\n",
    "    \"index.knn\": True,\n",
    "    \"default_pipeline\": \"bedrock-multimodal-ingest-pipeline\"\n",
    "  },\n",
    "  \"mappings\": {\n",
    "      \n",
    "    \"_source\": {\n",
    "     \n",
    "      \"excludes\": [\"image_binary\"]\n",
    "    },\n",
    "    \"properties\": {\n",
    "      \"vector_embedding\": {\n",
    "        \"type\": \"knn_vector\",\n",
    "        \"dimension\": shape,\n",
    "        \"method\": {\n",
    "          \"name\": \"hnsw\",\n",
    "          \"engine\": \"faiss\",\n",
    "          \"parameters\": {}\n",
    "        }\n",
    "      },\n",
    "      \"product_description\": {\n",
    "        \"type\": \"text\"\n",
    "      },\n",
    "        \"image_url\": {\n",
    "        \"type\": \"text\"\n",
    "      },\n",
    "      \"image_binary\": {\n",
    "        \"type\": \"binary\"\n",
    "      }\n",
    "    }\n",
    "  }\n",
    "}\n",
    "r = requests.put(url, auth=awsauth, json=payload, headers=headers)\n",
    "print(r.status_code)\n",
    "print(r.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f42df6cf",
   "metadata": {},
   "source": [
    "## 7. Ingest the dataset into k-NN index usig Bulk request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ceea00f0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from ruamel.yaml import YAML\n",
    "from PIL import Image\n",
    "import os\n",
    "from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth\n",
    "\n",
    "def resize_image(photo, width, height):\n",
    "    Image.MAX_IMAGE_PIXELS = 100000000\n",
    "    \n",
    "    with Image.open(photo) as image:\n",
    "        image.verify()\n",
    "    with Image.open(photo) as image:    \n",
    "        \n",
    "        if image.format in [\"JPEG\", \"PNG\"]:\n",
    "            file_type = image.format.lower()\n",
    "            path = image.filename.rsplit(\".\", 1)[0]\n",
    "\n",
    "            image.thumbnail((width, height))\n",
    "            image.save(f\"{path}-resized.{file_type}\")\n",
    "    return file_type, path\n",
    "\n",
    "# Load the products from the dataset\n",
    "yaml = YAML()\n",
    "items_ = yaml.load(open('tmp/images/products.yaml'))\n",
    "\n",
    "batch = 0\n",
    "count = 0\n",
    "body_ = ''\n",
    "batch_size = 100\n",
    "last_batch = int(len(items_)/batch_size)\n",
    "action = json.dumps({ 'index': { '_index': 'bedrock-multimodal-demostore-search-index' } })\n",
    "\n",
    "for item in items_:\n",
    "    count+=1\n",
    "    fileshort = \"tmp/images/\"+item[\"category\"]+\"/\"+item[\"image\"]\n",
    "    payload = {}\n",
    "    payload['image_url'] = fileshort\n",
    "    payload['product_description'] = item['description']\n",
    "    \n",
    "    #resize the image and generate image binary\n",
    "    file_type, path = resize_image(fileshort, 2048, 2048)\n",
    "\n",
    "    with open(fileshort.split(\".\")[0]+\"-resized.\"+file_type, \"rb\") as image_file:\n",
    "        input_image = base64.b64encode(image_file.read()).decode(\"utf8\")\n",
    "    \n",
    "    os.remove(fileshort.split(\".\")[0]+\"-resized.\"+file_type)\n",
    "    payload['image_binary'] = input_image\n",
    "    \n",
    "    body_ = body_ + action + \"\\n\" + json.dumps(payload) + \"\\n\"\n",
    "    \n",
    "    if(count == batch_size):\n",
    "        response = aos_client.bulk(\n",
    "        index = 'bedrock-multimodal-demostore-search-index',\n",
    "        body = body_\n",
    "        )\n",
    "        batch += 1\n",
    "        count = 0\n",
    "        print(\"batch \"+str(batch) + \" ingestion done!\")\n",
    "        if(batch != last_batch):\n",
    "            body_ = \"\"\n",
    "        \n",
    "            \n",
    "#ingest the remaining rows\n",
    "response = aos_client.bulk(\n",
    "        index = 'bedrock-multimodal-demostore-search-index',\n",
    "        body = body_\n",
    "        )\n",
    "        \n",
    "print(\"All \"+str(last_batch)+\" batches ingested into index\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0b08140",
   "metadata": {},
   "source": [
    "## 8. Experiment 1: Keyword search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58dbb176",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Keyword Search\n",
    "query = \"trendy footwear for women\"\n",
    "url = 'https://' + host + \"/bedrock-multimodal-demostore-search-index/_search\"\n",
    "keyword_payload = {\"_source\": {\n",
    "        \"exclude\": [\n",
    "            \"vector_embedding\"\n",
    "        ]\n",
    "        },\n",
    "        \"query\": {    \"match\": {\n",
    "                        \"product_description\": {\n",
    "                            \"query\": query\n",
    "                        }\n",
    "                        }\n",
    "                    }\n",
    "        \n",
    "        ,\"size\":5,\n",
    "  }\n",
    "\n",
    "r = requests.get(url, auth=awsauth, json=keyword_payload, headers=headers)\n",
    "response_ = json.loads(r.text)\n",
    "docs = response_['hits']['hits']\n",
    "\n",
    "for i,doc in enumerate(docs):\n",
    "    print(str(i+1)+ \". \"+doc[\"_source\"][\"product_description\"])\n",
    "    image = Image.open(doc[\"_source\"][\"image_url\"])\n",
    "    image.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "809face1",
   "metadata": {},
   "source": [
    "## 9. Experiment 2: Multimodal search with only text caption as input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13db1381",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Multimodal Search\n",
    "#Text as input\n",
    "\n",
    "query = \"trendy footwear for women\"\n",
    "url = 'https://'+host+\"/bedrock-multimodal-demostore-search-index/_search\"\n",
    "keyword_payload = {\"_source\": {\n",
    "        \"exclude\": [\n",
    "            \"vector_embedding\"\n",
    "        ]\n",
    "        },\n",
    "        \"query\": {    \n",
    "       \n",
    "        \"neural\": {\n",
    "            \"vector_embedding\": {\n",
    "                \n",
    "            #\"query_image\":query_image_binary,\n",
    "            \"query_text\":query,\n",
    "                \n",
    "            \"model_id\": model_id,\n",
    "            \"k\": 3\n",
    "            }\n",
    "            }\n",
    "            \n",
    "                    }\n",
    "        \n",
    "        ,\"size\":5,\n",
    "  }\n",
    "\n",
    "r = requests.get(url, auth=awsauth, json=keyword_payload, headers=headers)\n",
    "response_ = json.loads(r.text)\n",
    "docs = response_['hits']['hits']\n",
    "\n",
    "for i,doc in enumerate(docs):\n",
    "    print(doc[\"_source\"][\"product_description\"])\n",
    "    image = Image.open(doc[\"_source\"][\"image_url\"])\n",
    "    image.show()\n",
    "   \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6ccfd57",
   "metadata": {},
   "source": [
    "## 10. Experiment 3: Multimodal search with only image as input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87c2cc82",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Multimodal Search\n",
    "#image as input\n",
    "import urllib.request\n",
    "s3 = boto3.client('s3')\n",
    "url = 'https://'+host + \"/bedrock-multimodal-demostore-search-index/_search\"\n",
    "query = \"trendy black footwear for women\"\n",
    "print(\"Input text query: \"+query)\n",
    "# urllib.request.urlretrieve( \n",
    "#   'https://cdn.pixabay.com/photo/2014/09/03/20/15/shoes-434918_1280.jpg',\"tmp/women-footwear.jpg\") \n",
    "image_file = urllib.request.urlretrieve('https://aws-blogs-artifacts-public.s3.amazonaws.com/BDB-3144/women_wear.jpg', 'tmp/women-footwear-1.jpg')\n",
    "img = Image.open(\"tmp/women-footwear-1.jpg\") \n",
    "print(\"Input query Image:\")\n",
    "img.show()\n",
    "with open(\"tmp/women-footwear-1.jpg\", \"rb\") as image_file:\n",
    "    query_image_binary = base64.b64encode(image_file.read()).decode(\"utf8\")\n",
    "keyword_payload = {\"_source\": {\n",
    "        \"exclude\": [\n",
    "            \"vector_embedding\"\n",
    "        ]\n",
    "        },\n",
    "        \"query\": {    \n",
    "       \n",
    "        \"neural\": {\n",
    "            \"vector_embedding\": {\n",
    "                \n",
    "            \"query_image\":query_image_binary,\n",
    "            \"model_id\": model_id,\n",
    "            \"k\": 5\n",
    "            }\n",
    "            \n",
    "            }\n",
    "                    }\n",
    "        \n",
    "        ,\"size\":5,\n",
    "  }\n",
    "\n",
    "r = requests.get(url, auth=awsauth, json=keyword_payload, headers=headers)\n",
    "response_ = json.loads(r.text)\n",
    "docs = response_['hits']['hits']\n",
    "\n",
    "for i,doc in enumerate(docs):\n",
    "    print(doc[\"_source\"][\"product_description\"])\n",
    "    image = Image.open(doc[\"_source\"][\"image_url\"])\n",
    "    image.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6a1cb55",
   "metadata": {},
   "source": [
    "## 11. Experiment 4: Multimodal search with both image and text caption as inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bce08e5f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Multimodal Search\n",
    "#Text and image as inputs\n",
    "import urllib.request\n",
    "s3 = boto3.client('s3')\n",
    "url = 'https://'+host + \"/bedrock-multimodal-demostore-search-index/_search\"\n",
    "query = \"trendy footwear for women\"\n",
    "print(\"Input text query: \"+query)\n",
    "# urllib.request.urlretrieve( \n",
    "#   'https://cdn.pixabay.com/photo/2014/09/03/20/15/shoes-434918_1280.jpg',\"tmp/women-footwear.jpg\") \n",
    "img = Image.open(\"tmp/women-footwear-1.jpg\") \n",
    "print(\"Input query Image:\")\n",
    "img.show()\n",
    "with open(\"tmp/women-footwear-1.jpg\", \"rb\") as image_file:\n",
    "    query_image_binary = base64.b64encode(image_file.read()).decode(\"utf8\")\n",
    "keyword_payload = {\"_source\": {\n",
    "        \"exclude\": [\n",
    "            \"vector_embedding\"\n",
    "        ]\n",
    "        },\n",
    "        \"query\": {    \n",
    "       \n",
    "        \"neural\": {\n",
    "            \"vector_embedding\": {\n",
    "                \n",
    "            \"query_image\":query_image_binary,\n",
    "            \"query_text\":query,\n",
    "                \n",
    "            \"model_id\": model_id,\n",
    "            \"k\": 5\n",
    "            }\n",
    "            \n",
    "            }\n",
    "                    }\n",
    "        \n",
    "        ,\"size\":5,\n",
    "  }\n",
    "\n",
    "r = requests.get(url, auth=awsauth, json=keyword_payload, headers=headers)\n",
    "response_ = json.loads(r.text)\n",
    "docs = response_['hits']['hits']\n",
    "\n",
    "for i,doc in enumerate(docs):\n",
    "    print(doc[\"_source\"][\"product_description\"])\n",
    "    image = Image.open(doc[\"_source\"][\"image_url\"])\n",
    "    image.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c75bc076",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p310",
   "language": "python",
   "name": "conda_pytorch_p310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}