Katrina_RAG_LACounty_Rerank.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: PyPDF2 in /usr/local/python/3.10.8/lib/python3.10/site-packages (3.0.1)\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "!pip install PyPDF2\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cohere.client import Client\n",
    "\n",
    "COHERE_API_KEY = \"your key\"\n",
    "\n",
    "co = Client(COHERE_API_KEY)\n",
    "\n",
    "WEAVIATE_API_KEY = \"your key\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file contains a list of PDFs to load into Weaviate\n",
    "file_path = 'LACountyFiles/LACountyWasteDocs.csv'\n",
    "\n",
    "# Read the file into a Pandas DataFrame\n",
    "df = pd.read_csv(file_path, header=None, names=['Filename'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/codespace/.python/current/lib/python3.10/site-packages/PyPDF2/__init__.py:21: DeprecationWarning: PyPDF2 is deprecated. Please move to the pypdf library instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                      Filename  \\\n",
      "0                    RMIS_Textiles_5_final.pdf   \n",
      "1                RMIS_Landscaping_17_final.pdf   \n",
      "2  RMIS_FoodServiceWasteDiversion_11_final.pdf   \n",
      "3           RMIS_FoodServiceWaste_18_final.pdf   \n",
      "4                   Organicsfactsheet_2022.pdf   \n",
      "\n",
      "                                                Text  \n",
      "0  Laundry and Textile \\nManagement\\n1.2 MILLION ...  \n",
      "1  PRUNINGS\\nGRASS\\nFLOWERSVEGETABLE \\nSCRAPSFOOD...  \n",
      "2  Food Service \\nWaste Diversion\\nOTHER\\nAccordi...  \n",
      "3  Food Service Waste \\nReduction, Reuse and Dona...  \n",
      "4  MANDATORY ORGANIC WASTE RECYCLING  \\nFOR COUNT...  \n"
     ]
    }
   ],
   "source": [
    "import PyPDF2\n",
    "\n",
    "# Function to extract text from a PDF file\n",
    "def extract_text_from_pdf(filename):\n",
    "    try:\n",
    "        with open(filename, 'rb') as pdf_file:\n",
    "            pdf_reader = PyPDF2.PdfReader(pdf_file)\n",
    "            text = \"\"\n",
    "            for page in pdf_reader.pages:\n",
    "                page_text = page.extract_text()\n",
    "                text += page_text\n",
    "            return text\n",
    "    except Exception as e:\n",
    "        print(f\"Error processing file {filename}: {e}\")\n",
    "        return None\n",
    "    \n",
    "\n",
    "full_path = 'LACountyFiles/'+df['Filename'] \n",
    "df['Text'] = full_path.apply(extract_text_from_pdf)\n",
    "\n",
    "# Display the updated DataFrame\n",
    "print(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                Text\n",
      "0  \\n \\n \\n Los Angeles, CA Zero Waste Plan - Gre...\n"
     ]
    }
   ],
   "source": [
    "# webscrape a relevant article - if more than one is desired, need to modify this code to be like the above - to read a list of URLs from a file\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "\n",
    "url = 'https://www.greenpolicy360.net/w/Los_Angeles,_CA_Zero_Waste_Plan'\n",
    "\n",
    "# Send a GET request to the URL\n",
    "response = requests.get(url)\n",
    "\n",
    "# Check if the request was successful (status code 200)\n",
    "if response.status_code == 200:\n",
    "    # Parse the HTML content using BeautifulSoup\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "\n",
    "    # Extract all text from the HTML\n",
    "    all_text = soup.get_text(separator=' ')\n",
    "\n",
    "    # Create a DataFrame with a single column containing all the text\n",
    "    df2 = pd.DataFrame({'Text': [all_text]})\n",
    "\n",
    "    # Display the DataFrame\n",
    "    print(df2)\n",
    "else:\n",
    "    print(f\"Failed to fetch the URL. Status Code: {response.status_code}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                            Filename  \\\n",
      "0                          RMIS_Textiles_5_final.pdf   \n",
      "1                      RMIS_Landscaping_17_final.pdf   \n",
      "2        RMIS_FoodServiceWasteDiversion_11_final.pdf   \n",
      "3                 RMIS_FoodServiceWaste_18_final.pdf   \n",
      "4                         Organicsfactsheet_2022.pdf   \n",
      "5  https://www.greenpolicy360.net/w/Los_Angeles,_...   \n",
      "\n",
      "                                                Text  \n",
      "0  Laundry and Textile \\nManagement\\n1.2 MILLION ...  \n",
      "1  PRUNINGS\\nGRASS\\nFLOWERSVEGETABLE \\nSCRAPSFOOD...  \n",
      "2  Food Service \\nWaste Diversion\\nOTHER\\nAccordi...  \n",
      "3  Food Service Waste \\nReduction, Reuse and Dona...  \n",
      "4  MANDATORY ORGANIC WASTE RECYCLING  \\nFOR COUNT...  \n",
      "5  \\n \\n \\n Los Angeles, CA Zero Waste Plan - Gre...  \n"
     ]
    }
   ],
   "source": [
    "# add the new row with the web scraped article to the existing DataFrame\n",
    "new_row = {'Filename': url, 'Text': all_text}\n",
    "\n",
    "# Convert dict to DataFrame\n",
    "new_row_df = pd.DataFrame([new_row]) \n",
    "\n",
    "# add the new row to existing DataFrame and then print it to make sure it got added\n",
    "df = pd.concat([df, new_row_df], ignore_index=True)\n",
    "print (df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                            Filename  \\\n",
      "0                          RMIS_Textiles_5_final.pdf   \n",
      "1                      RMIS_Landscaping_17_final.pdf   \n",
      "2        RMIS_FoodServiceWasteDiversion_11_final.pdf   \n",
      "3                 RMIS_FoodServiceWaste_18_final.pdf   \n",
      "4                         Organicsfactsheet_2022.pdf   \n",
      "5  https://www.greenpolicy360.net/w/Los_Angeles,_...   \n",
      "\n",
      "                                                Text  \\\n",
      "0  Laundry and Textile \\nManagement\\n1.2 MILLION ...   \n",
      "1  PRUNINGS\\nGRASS\\nFLOWERSVEGETABLE \\nSCRAPSFOOD...   \n",
      "2  Food Service \\nWaste Diversion\\nOTHER\\nAccordi...   \n",
      "3  Food Service Waste \\nReduction, Reuse and Dona...   \n",
      "4  MANDATORY ORGANIC WASTE RECYCLING  \\nFOR COUNT...   \n",
      "5  \\n \\n \\n Los Angeles, CA Zero Waste Plan - Gre...   \n",
      "\n",
      "                                           Summaries  \n",
      "0  Textiles are a major source of waste and there...  \n",
      "1  As of 2016, AB 1826 required businesses genera...  \n",
      "2  Organic waste, including food waste, comprises...  \n",
      "3  Food waste is a significant problem in the Uni...  \n",
      "4  In order to reduce climate change and generate...  \n",
      "5  For the first time, Los Angeles is developing ...  \n"
     ]
    }
   ],
   "source": [
    "# create summaries using Cohere Client, specified 'short' because it was recommended the Weaviate object minimize vector size, otherwise we can get a 400 error from Cohere\n",
    "summaries = []\n",
    "\n",
    "for text in df['Text']:\n",
    "    summary = co.summarize(text, length=\"short\")\n",
    "  \n",
    "    summarized_text = summary.summary\n",
    "    summaries.append(summarized_text)\n",
    "\n",
    "# create a new column in the DataFrame for summaries and check if summaries loaded properly  \n",
    "df['Summaries'] = summaries\n",
    "print (df)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                            Filename  \\\n",
      "0                          RMIS_Textiles_5_final.pdf   \n",
      "1                      RMIS_Landscaping_17_final.pdf   \n",
      "2        RMIS_FoodServiceWasteDiversion_11_final.pdf   \n",
      "3                 RMIS_FoodServiceWaste_18_final.pdf   \n",
      "4                         Organicsfactsheet_2022.pdf   \n",
      "5  https://www.greenpolicy360.net/w/Los_Angeles,_...   \n",
      "\n",
      "                                           Summaries  \n",
      "0  Textiles are a major source of waste and there...  \n",
      "1  As of 2016, AB 1826 required businesses genera...  \n",
      "2  Organic waste, including food waste, comprises...  \n",
      "3  Food waste is a significant problem in the Uni...  \n",
      "4  In order to reduce climate change and generate...  \n",
      "5  For the first time, Los Angeles is developing ...  \n"
     ]
    }
   ],
   "source": [
    "# need to drop the Text from the DataFrame because we don't want to load it into the Weaviate object\n",
    "df.drop('Text', axis=1, inplace=True)\n",
    "\n",
    "# check to see if column dropped successfully\n",
    "print (df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{\"Filename\":\"RMIS_Textiles_5_final.pdf\",\"Summaries\":\"Textiles are a major source of waste and there are many ways to reduce textile waste. Laundry repair and reuse services can help maintain and extend the life of garments and bedding. Reusable tablecloths, napkins and table skirts can be used instead of single-use options and rags can be used to wash and dry vehicles. Linens are reused in hospitals, rehabilitation centres and fire stations. Donating used textiles to thrift stores or using a textile recycling service can prevent textiles from ending up in landfills.\"},{\"Filename\":\"RMIS_Landscaping_17_final.pdf\",\"Summaries\":\"As of 2016, AB 1826 required businesses generating a certain amount of organic waste per week to arrange for recycling services, and Los Angeles County has implemented programs to divert organic waste from landfills. Organic waste, including green waste, makes up a significant percentage of California's waste stream and can be repurposed through mulching, chipping, grinding, composting, or recycling. Landscape management contracts can require landscapers to take green waste to facilities for reuse, and leaving grass clippings on the lawn after mowing, known as grasscycling, can provide nutrients for fertilization while reducing green waste. Utilizing drought-tolerant plants native to California can reduce green waste, conserve water, reduce pesticide use, and lower maintenance costs.\"},{\"Filename\":\"RMIS_FoodServiceWasteDiversion_11_final.pdf\",\"Summaries\":\"Organic waste, including food waste, comprises a significant portion of the waste stream and emits harmful greenhouse gases when disposed of in landfills. Organic recycling programs can help mitigate these emissions by turning food waste into feedstock for livestock, fuel for vehicles, and nutrient-rich soil. Food service operations can establish recycling and organic waste collection programs to reduce waste disposal and comply with regulations such as California's Senate Bill 1383, which requires businesses to arrange for organic waste recycling. This article provides guidance on setting up organic recycling programs, tips for waste reduction, and case studies from Los Angeles County facilities.\"},{\"Filename\":\"RMIS_FoodServiceWaste_18_final.pdf\",\"Summaries\":\"Food waste is a significant problem in the United States, with over a third of all food produced being lost or wasted each year. Food waste has a negative impact on both the environment and society, as it increases pollution and exacerbates hunger. This article provides strategies that food service facilities can use to reduce their food waste, including the use of reusable trays and crates, implementing food supply rotation practices, and repurposing leftover food and scraps. These strategies can help facilities save money, feed hungry people in their community, and protect the environment by reducing the resources used to produce food that is ultimately wasted.\"},{\"Filename\":\"Organicsfactsheet_2022.pdf\",\"Summaries\":\"In order to reduce climate change and generate renewable products, Los Angeles county proposes implementing a mandatory organic waste recycling for all county departments. Organic waste, which is comprised of food waste and food-soiled paper products, accounts for nearly 15% of all disposal in the county. Therefore, recycling such waste would be extremely beneficial to the county. This proposal will require all county departments to arrange for organic waste recycling, including donating excess edible food to local food recovery organizations.\"},{\"Filename\":\"https:\\/\\/www.greenpolicy360.net\\/w\\/Los_Angeles,_CA_Zero_Waste_Plan\",\"Summaries\":\"For the first time, Los Angeles is developing the Solid Waste Integrated Resources Plan, also known as SWIRP, seeks input from a broad section of the community, and will result in the development of a 20-year master plan for the City\\u2019s solid waste and recycling programs.  Los Angeles is looking to achieve zero waste and this plan outlines the City\\u2019s objectives to provide sustainability, resource conservation, source reduction, recycling, renewable energy, maximum material recovery, and environmental protection. \\n\\nWould you like me to go into more details regarding any particular section of the plan?\"}]\n"
     ]
    }
   ],
   "source": [
    "# This converts the DataFrame contents into JSON in preparation for loading into Weaviate\n",
    "json_output = df.to_json(orient='records')\n",
    "print (json_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'classes': [{'class': 'Question',\n",
       "   'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n",
       "    'cleanupIntervalSeconds': 60,\n",
       "    'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n",
       "   'moduleConfig': {'generative-cohere': {},\n",
       "    'text2vec-cohere': {},\n",
       "    'text2vec-openai': {'baseURL': 'https://api.openai.com',\n",
       "     'model': 'ada',\n",
       "     'modelVersion': '002',\n",
       "     'type': 'text',\n",
       "     'vectorizeClassName': True}},\n",
       "   'multiTenancyConfig': {'enabled': False},\n",
       "   'properties': [],\n",
       "   'replicationConfig': {'factor': 1},\n",
       "   'shardingConfig': {'virtualPerPhysical': 128,\n",
       "    'desiredCount': 1,\n",
       "    'actualCount': 1,\n",
       "    'desiredVirtualCount': 128,\n",
       "    'actualVirtualCount': 128,\n",
       "    'key': '_id',\n",
       "    'strategy': 'hash',\n",
       "    'function': 'murmur3'},\n",
       "   'vectorIndexConfig': {'skip': False,\n",
       "    'cleanupIntervalSeconds': 300,\n",
       "    'maxConnections': 64,\n",
       "    'efConstruction': 128,\n",
       "    'ef': -1,\n",
       "    'dynamicEfMin': 100,\n",
       "    'dynamicEfMax': 500,\n",
       "    'dynamicEfFactor': 8,\n",
       "    'vectorCacheMaxObjects': 1000000000000,\n",
       "    'flatSearchCutoff': 40000,\n",
       "    'distance': 'cosine',\n",
       "    'pq': {'enabled': False,\n",
       "     'bitCompression': False,\n",
       "     'segments': 0,\n",
       "     'centroids': 256,\n",
       "     'trainingLimit': 100000,\n",
       "     'encoder': {'type': 'kmeans', 'distribution': 'log-normal'}}},\n",
       "   'vectorIndexType': 'hnsw',\n",
       "   'vectorizer': 'text2vec-openai'},\n",
       "  {'class': 'ExOProfiles',\n",
       "   'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n",
       "    'cleanupIntervalSeconds': 60,\n",
       "    'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n",
       "   'moduleConfig': {'generative-cohere': {},\n",
       "    'text2vec-cohere': {},\n",
       "    'text2vec-openai': {'baseURL': 'https://api.openai.com',\n",
       "     'model': 'ada',\n",
       "     'modelVersion': '002',\n",
       "     'type': 'text',\n",
       "     'vectorizeClassName': True}},\n",
       "   'multiTenancyConfig': {'enabled': False},\n",
       "   'properties': [{'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Sat Nov 11 16:06:31 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-openai': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'textprofiles',\n",
       "     'tokenization': 'word'},\n",
       "    {'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Sat Nov 11 16:06:31 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-openai': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'summaries',\n",
       "     'tokenization': 'word'}],\n",
       "   'replicationConfig': {'factor': 1},\n",
       "   'shardingConfig': {'virtualPerPhysical': 128,\n",
       "    'desiredCount': 1,\n",
       "    'actualCount': 1,\n",
       "    'desiredVirtualCount': 128,\n",
       "    'actualVirtualCount': 128,\n",
       "    'key': '_id',\n",
       "    'strategy': 'hash',\n",
       "    'function': 'murmur3'},\n",
       "   'vectorIndexConfig': {'skip': False,\n",
       "    'cleanupIntervalSeconds': 300,\n",
       "    'maxConnections': 64,\n",
       "    'efConstruction': 128,\n",
       "    'ef': -1,\n",
       "    'dynamicEfMin': 100,\n",
       "    'dynamicEfMax': 500,\n",
       "    'dynamicEfFactor': 8,\n",
       "    'vectorCacheMaxObjects': 1000000000000,\n",
       "    'flatSearchCutoff': 40000,\n",
       "    'distance': 'cosine',\n",
       "    'pq': {'enabled': False,\n",
       "     'bitCompression': False,\n",
       "     'segments': 0,\n",
       "     'centroids': 256,\n",
       "     'trainingLimit': 100000,\n",
       "     'encoder': {'type': 'kmeans', 'distribution': 'log-normal'}}},\n",
       "   'vectorIndexType': 'hnsw',\n",
       "   'vectorizer': 'text2vec-openai'},\n",
       "  {'class': 'Profiles',\n",
       "   'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n",
       "    'cleanupIntervalSeconds': 60,\n",
       "    'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n",
       "   'moduleConfig': {'generative-cohere': {},\n",
       "    'text2vec-cohere': {'vectorizeClassName': True}},\n",
       "   'multiTenancyConfig': {'enabled': False},\n",
       "   'properties': [{'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Sat Nov 11 16:16:53 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'textprofiles',\n",
       "     'tokenization': 'word'},\n",
       "    {'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Sat Nov 11 16:16:53 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'summaries',\n",
       "     'tokenization': 'word'}],\n",
       "   'replicationConfig': {'factor': 1},\n",
       "   'shardingConfig': {'virtualPerPhysical': 128,\n",
       "    'desiredCount': 1,\n",
       "    'actualCount': 1,\n",
       "    'desiredVirtualCount': 128,\n",
       "    'actualVirtualCount': 128,\n",
       "    'key': '_id',\n",
       "    'strategy': 'hash',\n",
       "    'function': 'murmur3'},\n",
       "   'vectorIndexConfig': {'skip': False,\n",
       "    'cleanupIntervalSeconds': 300,\n",
       "    'maxConnections': 64,\n",
       "    'efConstruction': 128,\n",
       "    'ef': -1,\n",
       "    'dynamicEfMin': 100,\n",
       "    'dynamicEfMax': 500,\n",
       "    'dynamicEfFactor': 8,\n",
       "    'vectorCacheMaxObjects': 1000000000000,\n",
       "    'flatSearchCutoff': 40000,\n",
       "    'distance': 'cosine',\n",
       "    'pq': {'enabled': False,\n",
       "     'bitCompression': False,\n",
       "     'segments': 0,\n",
       "     'centroids': 256,\n",
       "     'trainingLimit': 100000,\n",
       "     'encoder': {'type': 'kmeans', 'distribution': 'log-normal'}}},\n",
       "   'vectorIndexType': 'hnsw',\n",
       "   'vectorizer': 'text2vec-cohere'},\n",
       "  {'class': 'Profiles2',\n",
       "   'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n",
       "    'cleanupIntervalSeconds': 60,\n",
       "    'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n",
       "   'moduleConfig': {'generative-cohere': {},\n",
       "    'text2vec-cohere': {'vectorizeClassName': True}},\n",
       "   'multiTenancyConfig': {'enabled': False},\n",
       "   'properties': [{'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Sun Nov 12 02:26:14 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'textprofiles',\n",
       "     'tokenization': 'word'},\n",
       "    {'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Sun Nov 12 02:26:14 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'summaries',\n",
       "     'tokenization': 'word'}],\n",
       "   'replicationConfig': {'factor': 1},\n",
       "   'shardingConfig': {'virtualPerPhysical': 128,\n",
       "    'desiredCount': 1,\n",
       "    'actualCount': 1,\n",
       "    'desiredVirtualCount': 128,\n",
       "    'actualVirtualCount': 128,\n",
       "    'key': '_id',\n",
       "    'strategy': 'hash',\n",
       "    'function': 'murmur3'},\n",
       "   'vectorIndexConfig': {'skip': False,\n",
       "    'cleanupIntervalSeconds': 300,\n",
       "    'maxConnections': 64,\n",
       "    'efConstruction': 128,\n",
       "    'ef': -1,\n",
       "    'dynamicEfMin': 100,\n",
       "    'dynamicEfMax': 500,\n",
       "    'dynamicEfFactor': 8,\n",
       "    'vectorCacheMaxObjects': 1000000000000,\n",
       "    'flatSearchCutoff': 40000,\n",
       "    'distance': 'cosine',\n",
       "    'pq': {'enabled': False,\n",
       "     'bitCompression': False,\n",
       "     'segments': 0,\n",
       "     'centroids': 256,\n",
       "     'trainingLimit': 100000,\n",
       "     'encoder': {'type': 'kmeans', 'distribution': 'log-normal'}}},\n",
       "   'vectorIndexType': 'hnsw',\n",
       "   'vectorizer': 'text2vec-cohere'},\n",
       "  {'class': 'LACountyWasteResources',\n",
       "   'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n",
       "    'cleanupIntervalSeconds': 60,\n",
       "    'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n",
       "   'moduleConfig': {'generative-cohere': {},\n",
       "    'text2vec-cohere': {'vectorizeClassName': True}},\n",
       "   'multiTenancyConfig': {'enabled': False},\n",
       "   'properties': [{'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Thu Nov 16 18:12:55 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'source',\n",
       "     'tokenization': 'word'},\n",
       "    {'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Thu Nov 16 18:12:55 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'summary',\n",
       "     'tokenization': 'word'}],\n",
       "   'replicationConfig': {'factor': 1},\n",
       "   'shardingConfig': {'virtualPerPhysical': 128,\n",
       "    'desiredCount': 1,\n",
       "    'actualCount': 1,\n",
       "    'desiredVirtualCount': 128,\n",
       "    'actualVirtualCount': 128,\n",
       "    'key': '_id',\n",
       "    'strategy': 'hash',\n",
       "    'function': 'murmur3'},\n",
       "   'vectorIndexConfig': {'skip': False,\n",
       "    'cleanupIntervalSeconds': 300,\n",
       "    'maxConnections': 64,\n",
       "    'efConstruction': 128,\n",
       "    'ef': -1,\n",
       "    'dynamicEfMin': 100,\n",
       "    'dynamicEfMax': 500,\n",
       "    'dynamicEfFactor': 8,\n",
       "    'vectorCacheMaxObjects': 1000000000000,\n",
       "    'flatSearchCutoff': 40000,\n",
       "    'distance': 'cosine',\n",
       "    'pq': {'enabled': False,\n",
       "     'bitCompression': False,\n",
       "     'segments': 0,\n",
       "     'centroids': 256,\n",
       "     'trainingLimit': 100000,\n",
       "     'encoder': {'type': 'kmeans', 'distribution': 'log-normal'}}},\n",
       "   'vectorIndexType': 'hnsw',\n",
       "   'vectorizer': 'text2vec-cohere'},\n",
       "  {'class': 'OpenAIProfiles',\n",
       "   'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n",
       "    'cleanupIntervalSeconds': 60,\n",
       "    'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n",
       "   'moduleConfig': {'generative-openai': {},\n",
       "    'text2vec-openai': {'baseURL': 'https://api.openai.com',\n",
       "     'model': 'ada',\n",
       "     'modelVersion': '002',\n",
       "     'type': 'text',\n",
       "     'vectorizeClassName': True}},\n",
       "   'multiTenancyConfig': {'enabled': False},\n",
       "   'properties': [{'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Wed Nov 15 22:24:50 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-openai': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'textprofiles',\n",
       "     'tokenization': 'word'}],\n",
       "   'replicationConfig': {'factor': 1},\n",
       "   'shardingConfig': {'virtualPerPhysical': 128,\n",
       "    'desiredCount': 1,\n",
       "    'actualCount': 1,\n",
       "    'desiredVirtualCount': 128,\n",
       "    'actualVirtualCount': 128,\n",
       "    'key': '_id',\n",
       "    'strategy': 'hash',\n",
       "    'function': 'murmur3'},\n",
       "   'vectorIndexConfig': {'skip': False,\n",
       "    'cleanupIntervalSeconds': 300,\n",
       "    'maxConnections': 64,\n",
       "    'efConstruction': 128,\n",
       "    'ef': -1,\n",
       "    'dynamicEfMin': 100,\n",
       "    'dynamicEfMax': 500,\n",
       "    'dynamicEfFactor': 8,\n",
       "    'vectorCacheMaxObjects': 1000000000000,\n",
       "    'flatSearchCutoff': 40000,\n",
       "    'distance': 'cosine',\n",
       "    'pq': {'enabled': False,\n",
       "     'bitCompression': False,\n",
       "     'segments': 0,\n",
       "     'centroids': 256,\n",
       "     'trainingLimit': 100000,\n",
       "     'encoder': {'type': 'kmeans', 'distribution': 'log-normal'}}},\n",
       "   'vectorIndexType': 'hnsw',\n",
       "   'vectorizer': 'text2vec-openai'},\n",
       "  {'class': 'LACountyWasteResourcesTest',\n",
       "   'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n",
       "    'cleanupIntervalSeconds': 60,\n",
       "    'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n",
       "   'moduleConfig': {'generative-cohere': {},\n",
       "    'text2vec-cohere': {'vectorizeClassName': True}},\n",
       "   'multiTenancyConfig': {'enabled': False},\n",
       "   'properties': [{'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Thu Nov 16 03:48:26 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'source',\n",
       "     'tokenization': 'word'},\n",
       "    {'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Thu Nov 16 03:48:26 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'summary',\n",
       "     'tokenization': 'word'}],\n",
       "   'replicationConfig': {'factor': 1},\n",
       "   'shardingConfig': {'virtualPerPhysical': 128,\n",
       "    'desiredCount': 1,\n",
       "    'actualCount': 1,\n",
       "    'desiredVirtualCount': 128,\n",
       "    'actualVirtualCount': 128,\n",
       "    'key': '_id',\n",
       "    'strategy': 'hash',\n",
       "    'function': 'murmur3'},\n",
       "   'vectorIndexConfig': {'skip': False,\n",
       "    'cleanupIntervalSeconds': 300,\n",
       "    'maxConnections': 64,\n",
       "    'efConstruction': 128,\n",
       "    'ef': -1,\n",
       "    'dynamicEfMin': 100,\n",
       "    'dynamicEfMax': 500,\n",
       "    'dynamicEfFactor': 8,\n",
       "    'vectorCacheMaxObjects': 1000000000000,\n",
       "    'flatSearchCutoff': 40000,\n",
       "    'distance': 'cosine',\n",
       "    'pq': {'enabled': False,\n",
       "     'bitCompression': False,\n",
       "     'segments': 0,\n",
       "     'centroids': 256,\n",
       "     'trainingLimit': 100000,\n",
       "     'encoder': {'type': 'kmeans', 'distribution': 'log-normal'}}},\n",
       "   'vectorIndexType': 'hnsw',\n",
       "   'vectorizer': 'text2vec-cohere'},\n",
       "  {'class': 'LACountyWasteResources2',\n",
       "   'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n",
       "    'cleanupIntervalSeconds': 60,\n",
       "    'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n",
       "   'moduleConfig': {'generative-cohere': {},\n",
       "    'reranker-cohere': {},\n",
       "    'text2vec-cohere': {'vectorizeClassName': True}},\n",
       "   'multiTenancyConfig': {'enabled': False},\n",
       "   'properties': [{'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Fri Nov 17 21:55:07 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'summary',\n",
       "     'tokenization': 'word'},\n",
       "    {'dataType': ['text'],\n",
       "     'description': \"This property was generated by Weaviate's auto-schema feature on Fri Nov 17 21:55:07 2023\",\n",
       "     'indexFilterable': True,\n",
       "     'indexSearchable': True,\n",
       "     'moduleConfig': {'text2vec-cohere': {'skip': False,\n",
       "       'vectorizePropertyName': False}},\n",
       "     'name': 'source',\n",
       "     'tokenization': 'word'}],\n",
       "   'replicationConfig': {'factor': 1},\n",
       "   'shardingConfig': {'virtualPerPhysical': 128,\n",
       "    'desiredCount': 1,\n",
       "    'actualCount': 1,\n",
       "    'desiredVirtualCount': 128,\n",
       "    'actualVirtualCount': 128,\n",
       "    'key': '_id',\n",
       "    'strategy': 'hash',\n",
       "    'function': 'murmur3'},\n",
       "   'vectorIndexConfig': {'skip': False,\n",
       "    'cleanupIntervalSeconds': 300,\n",
       "    'maxConnections': 64,\n",
       "    'efConstruction': 128,\n",
       "    'ef': -1,\n",
       "    'dynamicEfMin': 100,\n",
       "    'dynamicEfMax': 500,\n",
       "    'dynamicEfFactor': 8,\n",
       "    'vectorCacheMaxObjects': 1000000000000,\n",
       "    'flatSearchCutoff': 40000,\n",
       "    'distance': 'cosine',\n",
       "    'pq': {'enabled': False,\n",
       "     'bitCompression': False,\n",
       "     'segments': 0,\n",
       "     'centroids': 256,\n",
       "     'trainingLimit': 100000,\n",
       "     'encoder': {'type': 'kmeans', 'distribution': 'log-normal'}}},\n",
       "   'vectorIndexType': 'hnsw',\n",
       "   'vectorizer': 'text2vec-cohere'}]}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import weaviate\n",
    "\n",
    "\n",
    "client = weaviate.Client(\n",
    "  url=\"https://sts-climatechange-h81u8pr2.weaviate.network\",  # URL of your Weaviate instance\n",
    "  auth_client_secret=weaviate.AuthApiKey(api_key=WEAVIATE_API_KEY),  # (Optional) If the Weaviate instance requires authentication\n",
    "  additional_headers={  # (Optional) Any additional headers; e.g. keys for API inference services\n",
    "    \"X-Cohere-Api-Key\": COHERE_API_KEY,            # Replace with your Cohere key\n",
    "  }\n",
    ")\n",
    "\n",
    "client.schema.get()  # Get the schema to test connection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.schema.delete_class(\"LACountyWasteResources2\")  # Replace with your class name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a class object - this only needs to be done once.  Object persists in the Weaviate cloud\n",
    "class_obj = {\n",
    "    \"class\": \"LACountyWasteResources2\",\n",
    "    \"vectorizer\": \"text2vec-cohere\",  # If set to \"none\" you must always provide vectors yourself. Could be any other \"text2vec-*\" also.\n",
    "    \"moduleConfig\": {\n",
    "        \"text2vec-cohere\": {},\n",
    "        \"reranker-cohere\": {},\n",
    "        \"generative-cohere\": {}  # Ensure the `generative-cohere` module is used for generative queries\n",
    "    }\n",
    "}\n",
    "\n",
    "client.schema.create_class(class_obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing profile:  0\n",
      "importing profile:  1\n",
      "importing profile:  2\n",
      "importing profile:  3\n",
      "importing profile:  4\n",
      "importing profile:  5\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "json_data = json.loads(json_output)\n",
    "client.batch.configure(batch_size=100)  # Configure batch\n",
    "with client.batch as batch:  # Initialize a batch process\n",
    "    for i,d in enumerate(json_data):  # Batch import data\n",
    "        print(\"importing profile: \", i)\n",
    "        properties = {\n",
    "            \"source\": d[\"Filename\"],\n",
    "            \"summary\": d[\"Summaries\"]      \n",
    "        }\n",
    "        batch.add_data_object(\n",
    "            data_object=properties,\n",
    "            class_name=\"LACountyWasteResources2\"\n",
    "        )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"data\": {\n",
      "        \"Get\": {\n",
      "            \"LACountyWasteResources2\": [\n",
      "                {\n",
      "                    \"source\": \"RMIS_Landscaping_17_final.pdf\",\n",
      "                    \"summary\": \"As of 2016, AB 1826 required businesses generating a certain amount of organic waste per week to arrange for recycling services, and Los Angeles County has implemented programs to divert organic waste from landfills. Organic waste, including green waste, makes up a significant percentage of California's waste stream and can be repurposed through mulching, chipping, grinding, composting, or recycling. Landscape management contracts can require landscapers to take green waste to facilities for reuse, and leaving grass clippings on the lawn after mowing, known as grasscycling, can provide nutrients for fertilization while reducing green waste. Utilizing drought-tolerant plants native to California can reduce green waste, conserve water, reduce pesticide use, and lower maintenance costs.\"\n",
      "                },\n",
      "                {\n",
      "                    \"source\": \"RMIS_FoodServiceWaste_18_final.pdf\",\n",
      "                    \"summary\": \"Food waste is a significant problem in the United States, with over a third of all food produced being lost or wasted each year. Food waste has a negative impact on both the environment and society, as it increases pollution and exacerbates hunger. This article provides strategies that food service facilities can use to reduce their food waste, including the use of reusable trays and crates, implementing food supply rotation practices, and repurposing leftover food and scraps. These strategies can help facilities save money, feed hungry people in their community, and protect the environment by reducing the resources used to produce food that is ultimately wasted.\"\n",
      "                },\n",
      "                {\n",
      "                    \"source\": \"Organicsfactsheet_2022.pdf\",\n",
      "                    \"summary\": \"In order to reduce climate change and generate renewable products, Los Angeles county proposes implementing a mandatory organic waste recycling for all county departments. Organic waste, which is comprised of food waste and food-soiled paper products, accounts for nearly 15% of all disposal in the county. Therefore, recycling such waste would be extremely beneficial to the county. This proposal will require all county departments to arrange for organic waste recycling, including donating excess edible food to local food recovery organizations.\"\n",
      "                },\n",
      "                {\n",
      "                    \"source\": \"RMIS_Textiles_5_final.pdf\",\n",
      "                    \"summary\": \"Textiles are a major source of waste and there are many ways to reduce textile waste. Laundry repair and reuse services can help maintain and extend the life of garments and bedding. Reusable tablecloths, napkins and table skirts can be used instead of single-use options and rags can be used to wash and dry vehicles. Linens are reused in hospitals, rehabilitation centres and fire stations. Donating used textiles to thrift stores or using a textile recycling service can prevent textiles from ending up in landfills.\"\n",
      "                },\n",
      "                {\n",
      "                    \"source\": \"RMIS_FoodServiceWasteDiversion_11_final.pdf\",\n",
      "                    \"summary\": \"Organic waste, including food waste, comprises a significant portion of the waste stream and emits harmful greenhouse gases when disposed of in landfills. Organic recycling programs can help mitigate these emissions by turning food waste into feedstock for livestock, fuel for vehicles, and nutrient-rich soil. Food service operations can establish recycling and organic waste collection programs to reduce waste disposal and comply with regulations such as California's Senate Bill 1383, which requires businesses to arrange for organic waste recycling. This article provides guidance on setting up organic recycling programs, tips for waste reduction, and case studies from Los Angeles County facilities.\"\n",
      "                },\n",
      "                {\n",
      "                    \"source\": \"https://www.greenpolicy360.net/w/Los_Angeles,_CA_Zero_Waste_Plan\",\n",
      "                    \"summary\": \"For the first time, Los Angeles is developing the Solid Waste Integrated Resources Plan, also known as SWIRP, seeks input from a broad section of the community, and will result in the development of a 20-year master plan for the City\\u2019s solid waste and recycling programs.  Los Angeles is looking to achieve zero waste and this plan outlines the City\\u2019s objectives to provide sustainability, resource conservation, source reduction, recycling, renewable energy, maximum material recovery, and environmental protection. \\n\\nWould you like me to go into more details regarding any particular section of the plan?\"\n",
      "                }\n",
      "            ]\n",
      "        }\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# check the contents of the class\n",
    "response = (\n",
    "    client.query\n",
    "    .get(\"LACountyWasteResources2\", [\"source\", \"summary\"])\n",
    "    .do()\n",
    ")\n",
    "\n",
    "print(json.dumps(response, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"data\": {\n",
      "        \"Get\": {\n",
      "            \"LACountyWasteResources2\": [\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.7615385\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"RMIS_Textiles_5_final.pdf\",\n",
      "                    \"summary\": \"Textiles are a major source of waste and there are many ways to reduce textile waste. Laundry repair and reuse services can help maintain and extend the life of garments and bedding. Reusable tablecloths, napkins and table skirts can be used instead of single-use options and rags can be used to wash and dry vehicles. Linens are reused in hospitals, rehabilitation centres and fire stations. Donating used textiles to thrift stores or using a textile recycling service can prevent textiles from ending up in landfills.\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.0006538156\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"RMIS_FoodServiceWasteDiversion_11_final.pdf\",\n",
      "                    \"summary\": \"Organic waste, including food waste, comprises a significant portion of the waste stream and emits harmful greenhouse gases when disposed of in landfills. Organic recycling programs can help mitigate these emissions by turning food waste into feedstock for livestock, fuel for vehicles, and nutrient-rich soil. Food service operations can establish recycling and organic waste collection programs to reduce waste disposal and comply with regulations such as California's Senate Bill 1383, which requires businesses to arrange for organic waste recycling. This article provides guidance on setting up organic recycling programs, tips for waste reduction, and case studies from Los Angeles County facilities.\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.00034196206\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"RMIS_FoodServiceWaste_18_final.pdf\",\n",
      "                    \"summary\": \"Food waste is a significant problem in the United States, with over a third of all food produced being lost or wasted each year. Food waste has a negative impact on both the environment and society, as it increases pollution and exacerbates hunger. This article provides strategies that food service facilities can use to reduce their food waste, including the use of reusable trays and crates, implementing food supply rotation practices, and repurposing leftover food and scraps. These strategies can help facilities save money, feed hungry people in their community, and protect the environment by reducing the resources used to produce food that is ultimately wasted.\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.00019563534\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"Organicsfactsheet_2022.pdf\",\n",
      "                    \"summary\": \"In order to reduce climate change and generate renewable products, Los Angeles county proposes implementing a mandatory organic waste recycling for all county departments. Organic waste, which is comprised of food waste and food-soiled paper products, accounts for nearly 15% of all disposal in the county. Therefore, recycling such waste would be extremely beneficial to the county. This proposal will require all county departments to arrange for organic waste recycling, including donating excess edible food to local food recovery organizations.\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.00017674513\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"https://www.greenpolicy360.net/w/Los_Angeles,_CA_Zero_Waste_Plan\",\n",
      "                    \"summary\": \"For the first time, Los Angeles is developing the Solid Waste Integrated Resources Plan, also known as SWIRP, seeks input from a broad section of the community, and will result in the development of a 20-year master plan for the City\\u2019s solid waste and recycling programs.  Los Angeles is looking to achieve zero waste and this plan outlines the City\\u2019s objectives to provide sustainability, resource conservation, source reduction, recycling, renewable energy, maximum material recovery, and environmental protection. \\n\\nWould you like me to go into more details regarding any particular section of the plan?\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 4.6474397e-05\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"RMIS_Landscaping_17_final.pdf\",\n",
      "                    \"summary\": \"As of 2016, AB 1826 required businesses generating a certain amount of organic waste per week to arrange for recycling services, and Los Angeles County has implemented programs to divert organic waste from landfills. Organic waste, including green waste, makes up a significant percentage of California's waste stream and can be repurposed through mulching, chipping, grinding, composting, or recycling. Landscape management contracts can require landscapers to take green waste to facilities for reuse, and leaving grass clippings on the lawn after mowing, known as grasscycling, can provide nutrients for fertilization while reducing green waste. Utilizing drought-tolerant plants native to California can reduce green waste, conserve water, reduce pesticide use, and lower maintenance costs.\"\n",
      "                }\n",
      "            ]\n",
      "        }\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Semantic search\n",
    "query = \"Find best practices on waste removal\"\n",
    "response = (\n",
    "    client.query\n",
    "    .get(\"LACountyWasteResources2\", [\"source\", \"summary\"])\n",
    "    .with_bm25(query=query)\n",
    "    #.with_near_text({\"concepts\": [\"best practices\"]})\n",
    "    .with_additional('rerank(property: \"summary\" query: \"textiles\") { score }')\n",
    "    #.with_limit(1)\n",
    "    .do()\n",
    ")\n",
    "\n",
    "print(json.dumps(response, indent=4))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"data\": {\n",
      "        \"Get\": {\n",
      "            \"LACountyWasteResources2\": [\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"generate\": null,\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.20275873\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"RMIS_FoodServiceWasteDiversion_11_final.pdf\",\n",
      "                    \"summary\": \"Organic waste, including food waste, comprises a significant portion of the waste stream and emits harmful greenhouse gases when disposed of in landfills. Organic recycling programs can help mitigate these emissions by turning food waste into feedstock for livestock, fuel for vehicles, and nutrient-rich soil. Food service operations can establish recycling and organic waste collection programs to reduce waste disposal and comply with regulations such as California's Senate Bill 1383, which requires businesses to arrange for organic waste recycling. This article provides guidance on setting up organic recycling programs, tips for waste reduction, and case studies from Los Angeles County facilities.\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"generate\": {\n",
      "                            \"error\": null,\n",
      "                            \"groupedResult\": \" Here are some best practices for organic waste removal:\\n\\n1. **Compliance with Regulations**: Ensure that you are complying with local regulations regarding organic waste removal. Many areas, including California, have implemented laws requiring businesses to arrange for organic waste recycling. Non-compliance may result in fines or other penalties. \\n\\n2. **Source Separation**: Implement a system for separating organic waste from other waste streams at the source. This can include placing different colored bins for different types of waste in key areas around your business premises to make it easy for your staff to differentiate between organic waste, recyclables, and other waste items. Having different colored or labeled bins helps with streamlining the sorting process once the waste is collected. \\n\\n3. **Utilization of Recycling Programs**: Take advantage of available recycling programs for organic waste, such as composting or mulching programs. These programs can help to reduce the amount of organic waste that is sent to landfills and can repurpose organic waste into useful products.\\n\\n4. **Donation of Edible Food**: Donate excess edible food to local food recovery organizations to support the community and reduce organic waste. This is a form of organic waste removal that complies with organic waste recycling while aiding the community. \\n\\n5. **Education and Training**: Provide education and training to your staff on proper organic waste removal and recycling practices. This can include information on how to identify organic waste, how to properly separate it from other waste streams, and how to participate in organic recycling programs such as composting. \\n\\n6. **Waste Reduction**: Implement strategies to reduce organic waste generation in the first place, such as using reusable trays and crates, implementing food supply rotation practices, and repurposing leftover food and scraps. \\n\\n7. **Utilization of Grasscycling**: Grasscycling involves leaving grass clippings on the lawn after mowing, which provides nutrients for fertilization while reducing green waste. \\n\\n8. **Use of Drought-Tolerant Plants**: Utilize drought-tolerant plants native to your area to reduce the amount of green waste generated and conserve water. \\n\\n9. **Textile Waste Reduction**: Implement strategies to reduce textile waste, such as using laundry repair and reuse services, donating used textiles to thrift stores or using a textile recycling service, and using reusable tablecloths, napkins, and table skirts instead of single-use options. \\n\\n10. **Organic Recycling Programs**: Food service operations can establish recycling and organic waste collection programs to reduce waste disposal and comply with regulations. These programs can turn food waste into feedstock for livestock, fuel for vehicles, and nutrient-rich soil. \\n\\nRemember, proper organic waste removal and recycling practices can help to reduce the amount of waste sent to landfills, mitigate greenhouse gas emissions, and contribute to a more sustainable future. \\n\\nWould you like me to go into more detail about any of the practices listed above? \"\n",
      "                        },\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.047691282\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"RMIS_Landscaping_17_final.pdf\",\n",
      "                    \"summary\": \"As of 2016, AB 1826 required businesses generating a certain amount of organic waste per week to arrange for recycling services, and Los Angeles County has implemented programs to divert organic waste from landfills. Organic waste, including green waste, makes up a significant percentage of California's waste stream and can be repurposed through mulching, chipping, grinding, composting, or recycling. Landscape management contracts can require landscapers to take green waste to facilities for reuse, and leaving grass clippings on the lawn after mowing, known as grasscycling, can provide nutrients for fertilization while reducing green waste. Utilizing drought-tolerant plants native to California can reduce green waste, conserve water, reduce pesticide use, and lower maintenance costs.\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"generate\": null,\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.0029926652\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"Organicsfactsheet_2022.pdf\",\n",
      "                    \"summary\": \"In order to reduce climate change and generate renewable products, Los Angeles county proposes implementing a mandatory organic waste recycling for all county departments. Organic waste, which is comprised of food waste and food-soiled paper products, accounts for nearly 15% of all disposal in the county. Therefore, recycling such waste would be extremely beneficial to the county. This proposal will require all county departments to arrange for organic waste recycling, including donating excess edible food to local food recovery organizations.\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"generate\": null,\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.0008231386\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"RMIS_FoodServiceWaste_18_final.pdf\",\n",
      "                    \"summary\": \"Food waste is a significant problem in the United States, with over a third of all food produced being lost or wasted each year. Food waste has a negative impact on both the environment and society, as it increases pollution and exacerbates hunger. This article provides strategies that food service facilities can use to reduce their food waste, including the use of reusable trays and crates, implementing food supply rotation practices, and repurposing leftover food and scraps. These strategies can help facilities save money, feed hungry people in their community, and protect the environment by reducing the resources used to produce food that is ultimately wasted.\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"generate\": null,\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 0.00030534892\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"RMIS_Textiles_5_final.pdf\",\n",
      "                    \"summary\": \"Textiles are a major source of waste and there are many ways to reduce textile waste. Laundry repair and reuse services can help maintain and extend the life of garments and bedding. Reusable tablecloths, napkins and table skirts can be used instead of single-use options and rags can be used to wash and dry vehicles. Linens are reused in hospitals, rehabilitation centres and fire stations. Donating used textiles to thrift stores or using a textile recycling service can prevent textiles from ending up in landfills.\"\n",
      "                },\n",
      "                {\n",
      "                    \"_additional\": {\n",
      "                        \"generate\": null,\n",
      "                        \"rerank\": [\n",
      "                            {\n",
      "                                \"score\": 9.4612624e-05\n",
      "                            }\n",
      "                        ]\n",
      "                    },\n",
      "                    \"source\": \"https://www.greenpolicy360.net/w/Los_Angeles,_CA_Zero_Waste_Plan\",\n",
      "                    \"summary\": \"For the first time, Los Angeles is developing the Solid Waste Integrated Resources Plan, also known as SWIRP, seeks input from a broad section of the community, and will result in the development of a 20-year master plan for the City\\u2019s solid waste and recycling programs.  Los Angeles is looking to achieve zero waste and this plan outlines the City\\u2019s objectives to provide sustainability, resource conservation, source reduction, recycling, renewable energy, maximum material recovery, and environmental protection. \\n\\nWould you like me to go into more details regarding any particular section of the plan?\"\n",
      "                }\n",
      "            ]\n",
      "        }\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# RAG search\n",
    "query_text = \"Find best practices for organic waste removal in {summary}.\"\n",
    "response = (\n",
    "    client.query\n",
    "    .get(\"LACountyWasteResources2\", [\"source\", \"summary\"])\n",
    "    .with_generate(grouped_task=query_text)\n",
    "    .with_additional('rerank(property: \"summary\" query: \"Find best practices for organic waste removal\") { score }')\n",
    "    .do()\n",
    ")\n",
    "\n",
    "print(json.dumps(response, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results been saved to CohereGroupedTaskRankedLACounty_1.txt\n"
     ]
    }
   ],
   "source": [
    "# Write to a file to save results for later inspection\n",
    "import os\n",
    "\n",
    "# Specify the name where you want to save the text file to save results\n",
    "file_name = 'CohereGroupedTaskRankedLACounty.txt'\n",
    "new_filename = file_name\n",
    "\n",
    "# Save the response along with the query that generated the response\n",
    "header = \"Query: \", query_text + '\\n'\n",
    "\n",
    "if os.path.exists(file_name):\n",
    "    # File exists, append a version number before extension\n",
    "    version = 1\n",
    "    while True:\n",
    "        new_filename = f'{file_name[:-4]}_{version}{file_name[-4:]}'\n",
    "        if not os.path.exists(new_filename):\n",
    "            filename = new_filename\n",
    "            break\n",
    "        version += 1\n",
    "\n",
    "with open(new_filename, 'w') as f:\n",
    "    f.write(str(header))\n",
    "    f.write(json.dumps(response, indent=4))\n",
    "\n",
    "print(f'Results been saved to {new_filename}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rerank_responses(query_text, responses, num_responses):\n",
    "  reranked_responses = co.rerank(\n",
    "    model = 'rerank-english-v2.0',\n",
    "    query = query,\n",
    "    documents = responses,\n",
    "    top_n = num_responses\n",
    "  )\n",
    "  return reranked_responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Find best practices in {summary}.\n",
      "{'data': {'Get': {'LACountyWasteResources': [{'_additional': {'generate': {'error': None, 'groupedResult': ' I was able to find many best practices in the provided summaries. Here is a summary of the key points I found:\\n\\n- Organic waste recycling is mandatory for businesses generating more than two cubic yards of waste per week in Los Angeles County, as this can be successfully diverted and converted into mulch, compost, and biofuel.\\n- In 2020, it is estimated that over 1.6 million tons of food was disposed of in Los Angeles County, which represents nearly 15% of all disposal. Therefore, recycling organic waste is not only beneficial for the environment but also for the county as a whole.\\n- Textiles are a major source of waste, with over one million tons disposed of in California landfills each year. However, methods such as repair and reuse can reduce the environmental impact of textile manufacturing. Donating used textiles to charities or repurposing them for other uses can also reduce waste.\\n- Food waste is a significant problem in the United States, with over a third of all food produced being lost or wasted every year. Reducing food waste saves money, conserves resources, and helps feed hungry people in local communities. \\n\\nWould you like me to extract best practices from a different summary? '}, 'rerank': [{'score': 0.0005274784}]}, 'source': 'RMIS_Landscaping_17_final.pdf', 'summary': 'As of 2016, businesses generating two cubic yards or more of waste per week must arrange for organic waste recycling. Los Angeles County unincorporated areas have successfully diverted 37% of organic waste and converted it into mulch, compost and biofuel, with a State goal of reaching a 75% reduction of 2014 organic waste disposal levels by 2025. To reduce green waste, landscape with native and drought-tolerant plants, grasscycle by leaving grass clippings on the lawn after mowing, and use recycled organic products on site.'}, {'_additional': {'generate': None, 'rerank': [{'score': 0.000111479036}]}, 'source': 'RMIS_FoodServiceWasteDiversion_11_final.pdf', 'summary': 'Organic waste, including food waste, comprises a significant portion of the waste stream and emits harmful greenhouse gases when disposed of in landfills. Organic recycling programs can help mitigate this by converting waste organic materials into useful products like livestock feed, vehicle fuel, and soil nourishment. Food service operations can significantly reduce their waste output by implementing recycling and organic waste collection programs. This guide provides strategies for effective organic waste management in food service settings, including establishing source separation of recyclables and organic waste, finding creative uses for leftovers, implementing on-site compost or vermicompost programs, and collecting clean and dry paper, cardboard, bottles, and plastic containers. Case studies from Los Angeles County facilities demonstrate best practices in implementing these strategies. By following these tips, food service operations can help reduce their environmental impact and contribute to a more sustainable future.'}, {'_additional': {'generate': None, 'rerank': [{'score': 9.387641e-05}]}, 'source': 'https://www.greenpolicy360.net/w/Los_Angeles,_CA_Zero_Waste_Plan', 'summary': 'Los Angeles is developing its Solid Waste Integrated Resources Plan, also known as SWIRP, to create a visionary 20-year master plan for the city\\'s solid waste and recycling programs. The plan seeks input from a broad range of stakeholders to develop goals for sustainability, resource conservation, source reduction, recycling, renewable energy, and maximum material recovery. The city aims to lead Los Angeles toward becoming a \"zero waste\" city through these initiatives. Would you like to know more about the city of Los Angeles\\' environmental policies?'}, {'_additional': {'generate': None, 'rerank': [{'score': 8.415109e-05}]}, 'source': 'RMIS_Textiles_5_final.pdf', 'summary': 'Textiles are a major source of waste, with over one million tons disposed of in California landfills each year. However, methods such as repair and reuse can reduce the environmental impact of textile manufacturing. Examples of textile reuse include washable linens at event venues, fire station cleaning rags, and hospital linens. Laundry repair and reuse services can extend the life of garments and bedding, reducing the amount of waste sent to landfill. Donating used textiles to charities or repurposing them for other uses can also reduce waste. Overall, textile waste is a problem that can be mitigated through creative solutions and increased reuse. \\n\\nWould you like to know more about textile recycling?'}, {'_additional': {'generate': None, 'rerank': [{'score': 8.220189e-05}]}, 'source': 'Organicsfactsheet_2022.pdf', 'summary': 'In 2020, it is estimated that over 1.6 million tons of food was disposed of in Los Angeles County, which represents nearly 15% of all disposal. Therefore, recycling organic waste is not only beneficial for the environment but also for the county as a whole. To facilitate this, the county has arranged for organic waste collection and recycling services through waste haulers such as Athens Services, Southland Disposal, Universal Waste Systems, and USA Waste, to name a few. Identifying where organic waste is generated, setting up an organic waste collection program, distributing organic waste collection bins, posting signage, and distributing educational materials are some of the basics of setting up an organic waste recycling program.'}, {'_additional': {'generate': None, 'rerank': [{'score': 5.0250615e-05}]}, 'source': 'RMIS_FoodServiceWaste_18_final.pdf', 'summary': \"Food waste is a significant problem in the United States, with over a third of all food produced being lost or wasted every year. Food waste has negative impacts on both the environment and society. This article outlines programs that aim to reduce food waste, including the EPA's Food Recovery Hierarchy, the USDA's FoodKeeper app, and the Bill Emerson Good Samaritan Food Donation Act. It also provides case studies of successful food waste reduction and donation programs in Los Angeles County, such as reusable food service trays, deliveries in reusable crates, rotation and organization of meal supplies, repurposing leftover food and scraps, and summer lunch food donations. Reducing food waste saves money, conserves resources, and helps feed hungry people in local communities. \\n\\nWould you like to know more about food waste reduction and donation programs?\"}]}}}\n"
     ]
    }
   ],
   "source": [
    "print (query_text)\n",
    "print (response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "reranked_RAG_responses = rerank_responses(query_text, response, num_responses=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[RerankResult<document['text']: data, index: 0, relevance_score: 0.00930395>]\n",
      "<class 'cohere.responses.rerank.Reranking'>\n",
      "top_text:  data\n"
     ]
    }
   ],
   "source": [
    "print (reranked_RAG_responses)\n",
    "print(type(reranked_RAG_responses))\n",
    "top_text = reranked_RAG_responses[0].document['text']\n",
    "print (\"top_text: \", top_text)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}