minio-weaviate-backups/s3_backup_module.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2bf7e43b",
   "metadata": {},
   "source": [
    "Install the Weaviate-Client library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f456ae22-34bc-4aa4-9cfb-b9a34ffdc5a9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: weaviate-client in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (4.4.2)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.30.0 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from weaviate-client) (2.31.0)\n",
      "Requirement already satisfied: httpx==0.26.0 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from weaviate-client) (0.26.0)\n",
      "Requirement already satisfied: validators==0.22.0 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from weaviate-client) (0.22.0)\n",
      "Requirement already satisfied: authlib<2.0.0,>=1.2.1 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from weaviate-client) (1.2.1)\n",
      "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from weaviate-client) (2.6.0)\n",
      "Requirement already satisfied: grpcio<2.0.0,>=1.57.0 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from weaviate-client) (1.60.0)\n",
      "Requirement already satisfied: grpcio-tools<2.0.0,>=1.57.0 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from weaviate-client) (1.60.0)\n",
      "Requirement already satisfied: grpcio-health-checking<2.0.0,>=1.57.0 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from weaviate-client) (1.60.0)\n",
      "Requirement already satisfied: anyio in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from httpx==0.26.0->weaviate-client) (3.7.1)\n",
      "Requirement already satisfied: certifi in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from httpx==0.26.0->weaviate-client) (2022.12.7)\n",
      "Requirement already satisfied: httpcore==1.* in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from httpx==0.26.0->weaviate-client) (1.0.2)\n",
      "Requirement already satisfied: idna in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from httpx==0.26.0->weaviate-client) (2.10)\n",
      "Requirement already satisfied: sniffio in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from httpx==0.26.0->weaviate-client) (1.3.0)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from httpcore==1.*->httpx==0.26.0->weaviate-client) (0.14.0)\n",
      "Requirement already satisfied: cryptography>=3.2 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from authlib<2.0.0,>=1.2.1->weaviate-client) (3.4.8)\n",
      "Requirement already satisfied: protobuf>=4.21.6 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from grpcio-health-checking<2.0.0,>=1.57.0->weaviate-client) (4.25.1)\n",
      "Requirement already satisfied: setuptools in c:\\program files\\windowsapps\\pythonsoftwarefoundation.python.3.9_3.9.3568.0_x64__qbz5n2kfra8p0\\lib\\site-packages (from grpcio-tools<2.0.0,>=1.57.0->weaviate-client) (58.1.0)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from pydantic<3.0.0,>=2.5.0->weaviate-client) (0.5.0)\n",
      "Requirement already satisfied: pydantic-core==2.16.1 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from pydantic<3.0.0,>=2.5.0->weaviate-client) (2.16.1)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from pydantic<3.0.0,>=2.5.0->weaviate-client) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from requests<3.0.0,>=2.30.0->weaviate-client) (3.1.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from requests<3.0.0,>=2.30.0->weaviate-client) (1.26.15)\n",
      "Requirement already satisfied: cffi>=1.12 in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from cryptography>=3.2->authlib<2.0.0,>=1.2.1->weaviate-client) (1.15.1)\n",
      "Requirement already satisfied: exceptiongroup in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from anyio->httpx==0.26.0->weaviate-client) (1.1.1)\n",
      "Requirement already satisfied: pycparser in c:\\users\\david\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\\localcache\\local-packages\\python39\\site-packages (from cffi>=1.12->cryptography>=3.2->authlib<2.0.0,>=1.2.1->weaviate-client) (2.21)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEPRECATION: torchsde 0.2.5 has a non-standard dependency specifier numpy>=1.19.*; python_version >= \"3.7\". pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of torchsde or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\n"
     ]
    }
   ],
   "source": [
    "!pip install -U weaviate-client"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8871cdab",
   "metadata": {},
   "source": [
    "# Import the Schema Classes for Article & Author"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4eea272-8c56-4ad1-b9a3-199d33da96aa",
   "metadata": {},
   "source": [
    "## Setup of Schema and Data\n",
    "\n",
    "This cell does the following:\n",
    "- initializes the client\n",
    "- defines the schema using json\n",
    "- creates the objects using json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7a49347a-a329-45ff-ac5c-76a2b804c604",
   "metadata": {},
   "outputs": [],
   "source": [
    "import weaviate\n",
    "client = weaviate.Client(\"http://localhost:8080\")\n",
    "\n",
    "schema = {\n",
    "  \"classes\": [\n",
    "    {\n",
    "      \"class\": \"Article\",\n",
    "      \"description\": \"A class to store articles\",\n",
    "      \"properties\": [\n",
    "        {\"name\": \"title\", \"dataType\": [\"string\"], \"description\": \"The title of the article\"},\n",
    "        {\"name\": \"content\", \"dataType\": [\"text\"], \"description\": \"The content of the article\"},\n",
    "        {\"name\": \"datePublished\", \"dataType\": [\"date\"], \"description\": \"The date the article was published\"},\n",
    "        {\"name\": \"url\", \"dataType\": [\"string\"], \"description\": \"The URL of the article\"}\n",
    "      ]\n",
    "    },\n",
    "    {\n",
    "      \"class\": \"Author\",\n",
    "      \"description\": \"A class to store authors\",\n",
    "      \"properties\": [\n",
    "        {\"name\": \"name\", \"dataType\": [\"string\"], \"description\": \"The name of the author\"},\n",
    "        {\"name\": \"articles\", \"dataType\": [\"Article\"], \"description\": \"The articles written by the author\"}\n",
    "      ]\n",
    "    }\n",
    "  ]\n",
    "}\n",
    "client.schema.delete_class('Article')\n",
    "client.schema.delete_class('Author')\n",
    "client.schema.create(schema)\n",
    "\n",
    "# JSON data to be Ingested\n",
    "\n",
    "data = [\n",
    "    {\n",
    "        \"class\": \"Article\",\n",
    "        \"properties\": {\n",
    "            \"title\": \"LangChain: OpenAI + S3 Loader\",\n",
    "            \"content\": \"This article discusses the integration of LangChain with OpenAI and S3 Loader...\",\n",
    "            \"url\": \"https://blog.min.io/langchain-openai-s3-loader/\"\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        \"class\": \"Article\",\n",
    "        \"properties\": {\n",
    "            \"title\": \"MinIO Webhook Event Notifications\",\n",
    "            \"content\": \"Exploring the webhook event notification system in MinIO...\",\n",
    "            \"url\": \"https://blog.min.io/minio-webhook-event-notifications/\"\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        \"class\": \"Article\",\n",
    "        \"properties\": {\n",
    "            \"title\": \"MinIO Postgres Event Notifications\",\n",
    "            \"content\": \"An in-depth look at Postgres event notifications in MinIO...\",\n",
    "            \"url\": \"https://blog.min.io/minio-postgres-event-notifications/\"\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        \"class\": \"Article\",\n",
    "        \"properties\": {\n",
    "            \"title\": \"From Docker to Localhost\",\n",
    "            \"content\": \"A guide on transitioning from Docker to localhost environments...\",\n",
    "            \"url\": \"https://blog.min.io/from-docker-to-localhost/\"\n",
    "        }\n",
    "    }\n",
    "]\n",
    "\n",
    "for item in data:\n",
    "    client.data_object.create(\n",
    "        data_object=item[\"properties\"],\n",
    "        class_name=item[\"class\"]\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fee0a3f-f1fe-4cee-8ceb-effa3c870f99",
   "metadata": {},
   "source": [
    "# Creating a Backup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b8b5479f-4d7b-47bd-84f8-f17379c399e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'backend': 's3', 'classes': ['Article', 'Author'], 'id': 'backup-id-5', 'path': 's3://weaviate-backups/backup-id-5', 'status': 'SUCCESS'}\n"
     ]
    }
   ],
   "source": [
    "import weaviate\n",
    "\n",
    "client = weaviate.Client(\"http://localhost:8080\")\n",
    "result = client.backup.create(\n",
    "  backup_id=\"backup-id-5\",\n",
    "  backend=\"s3\",\n",
    "  include_classes=[\"Article\", \"Author\"],  # specify classes to include or omit this for all classes\n",
    "  wait_for_completion=True,\n",
    ")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19b4f015-7445-4a74-a97e-7b51770f401e",
   "metadata": {},
   "source": [
    "## Deleting the Schema Classes for Restoring Purposes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a08feed1-c376-4449-a033-8be81a311cfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "client.schema.delete_class(\"Article\")\n",
    "client.schema.delete_class(\"Author\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2aa5bac4-d28f-48e9-a570-1d4c35c5ccba",
   "metadata": {},
   "source": [
    "# Restoring the Backup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "17c8c3ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'backend': 's3', 'classes': ['Article', 'Author'], 'id': 'backup-id-2', 'path': 's3://weaviate-backups/backup-id-2', 'status': 'SUCCESS'}\n"
     ]
    }
   ],
   "source": [
    "result = client.backup.restore(\n",
    "  backup_id=\"backup-id-2\",\n",
    "  backend=\"s3\",\n",
    "  wait_for_completion=True,\n",
    ")\n",
    "\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10cdb9ed-2b7e-494b-bdc5-b932253b0c85",
   "metadata": {},
   "source": [
    "## Deleting the Schema Classes for Restoring Purposes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "649a6123-10c1-4aed-8663-30f629fe2d25",
   "metadata": {},
   "outputs": [],
   "source": [
    "client.schema.delete_class(\"Article\")\n",
    "client.schema.delete_class(\"Author\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e7789c4-d9ab-4a95-b5e4-d2b74953cd21",
   "metadata": {},
   "source": [
    "# Restoring the Backup with error handling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "92c5dea2-44a9-4dad-90b5-6d7f90b19255",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Backup restored successfully: {'backend': 's3', 'classes': ['Author', 'Article'], 'id': 'backup-id-2', 'path': 's3://weaviate-backups/backup-id-2', 'status': 'SUCCESS'}\n"
     ]
    }
   ],
   "source": [
    "import weaviate\n",
    "from weaviate.exceptions import BackupFailedError\n",
    "\n",
    "client = weaviate.Client(\"http://192.168.0.25:8080\")\n",
    "\n",
    "try:\n",
    "    result = client.backup.restore(\n",
    "        backup_id=\"backup-id-2\",\n",
    "        backend=\"s3\",\n",
    "        wait_for_completion=True,\n",
    "    )\n",
    "    print(\"Backup restored successfully:\", result)\n",
    "\n",
    "except BackupFailedError as e:\n",
    "    print(\"Backup restore failed with error:\", e)\n",
    "    # Here you can add logic to handle the failure, such as retrying the operation or logging the error.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f47880a3-6fb5-47ed-94fd-d45db58fb212",
   "metadata": {},
   "source": [
    "# Get the Schema Class for \"Article\" to Prove the Backup/Restore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d279e4d9-9e85-484b-bf7b-333737c23dab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class': 'Article',\n",
       " 'description': 'A class to store articles',\n",
       " 'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n",
       "  'cleanupIntervalSeconds': 60,\n",
       "  'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n",
       " 'moduleConfig': {'text2vec-transformers': {'poolingStrategy': 'masked_mean',\n",
       "   'vectorizeClassName': True}},\n",
       " 'multiTenancyConfig': {'enabled': False},\n",
       " 'properties': [{'dataType': ['text'],\n",
       "   'description': 'The title of the article',\n",
       "   'indexFilterable': True,\n",
       "   'indexSearchable': True,\n",
       "   'moduleConfig': {'text2vec-transformers': {'skip': False,\n",
       "     'vectorizePropertyName': False}},\n",
       "   'name': 'title',\n",
       "   'tokenization': 'whitespace'},\n",
       "  {'dataType': ['text'],\n",
       "   'description': 'The content of the article',\n",
       "   'indexFilterable': True,\n",
       "   'indexSearchable': True,\n",
       "   'moduleConfig': {'text2vec-transformers': {'skip': False,\n",
       "     'vectorizePropertyName': False}},\n",
       "   'name': 'content',\n",
       "   'tokenization': 'word'},\n",
       "  {'dataType': ['date'],\n",
       "   'description': 'The date the article was published',\n",
       "   'indexFilterable': True,\n",
       "   'indexSearchable': False,\n",
       "   'moduleConfig': {'text2vec-transformers': {'skip': False,\n",
       "     'vectorizePropertyName': False}},\n",
       "   'name': 'datePublished'},\n",
       "  {'dataType': ['text'],\n",
       "   'description': 'The URL of the article',\n",
       "   'indexFilterable': True,\n",
       "   'indexSearchable': True,\n",
       "   'moduleConfig': {'text2vec-transformers': {'skip': False,\n",
       "     'vectorizePropertyName': False}},\n",
       "   'name': 'url',\n",
       "   'tokenization': 'whitespace'}],\n",
       " 'replicationConfig': {'factor': 1},\n",
       " 'shardingConfig': {'virtualPerPhysical': 128,\n",
       "  'desiredCount': 1,\n",
       "  'actualCount': 1,\n",
       "  'desiredVirtualCount': 128,\n",
       "  'actualVirtualCount': 128,\n",
       "  'key': '_id',\n",
       "  'strategy': 'hash',\n",
       "  'function': 'murmur3'},\n",
       " 'vectorIndexConfig': {'skip': False,\n",
       "  'cleanupIntervalSeconds': 300,\n",
       "  'maxConnections': 64,\n",
       "  'efConstruction': 128,\n",
       "  'ef': -1,\n",
       "  'dynamicEfMin': 100,\n",
       "  'dynamicEfMax': 500,\n",
       "  'dynamicEfFactor': 8,\n",
       "  'vectorCacheMaxObjects': 1000000000000,\n",
       "  'flatSearchCutoff': 40000,\n",
       "  'distance': 'cosine',\n",
       "  'pq': {'enabled': False,\n",
       "   'bitCompression': False,\n",
       "   'segments': 0,\n",
       "   'centroids': 256,\n",
       "   'trainingLimit': 100000,\n",
       "   'encoder': {'type': 'kmeans', 'distribution': 'log-normal'}},\n",
       "  'bq': {'enabled': False}},\n",
       " 'vectorIndexType': 'hnsw',\n",
       " 'vectorizer': 'text2vec-transformers'}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.schema.get(\"Article\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d789f5ae-8810-4062-8721-308f2b6231b6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}