query_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install lancedb\n",
    "%pip install pandas\n",
    "%pip install pyarrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lancedb\n",
    "import pandas as pd\n",
    "import pyarrow as pa\n",
    "\n",
    "uri = \"data/vecdb1\"\n",
    "db = lancedb.connect(uri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(db.table_names())\n",
    "tbl = db.open_table(\"vectors_table_1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tbl.count_rows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tbl.search(None).select([\"filename\"]).to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install sentence_transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "\n",
    "# The sentences to encode\n",
    "sentences = [\n",
    "    \"I really like animals and especially dogs.\",\n",
    "    \"It's so sunny outside!\",\n",
    "]\n",
    "\n",
    "# 2. Calculate embeddings by calling model.encode()\n",
    "embeddings = model.encode(sentences, normalize_embeddings=True)\n",
    "print(embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(embeddings[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tbl.search(embeddings[0]) \\\n",
    "    .select([\"filename\"]) \\\n",
    "    .limit(3).to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read the content of this file: embedding_files/embedding_content_99996.txt\n",
    "filename = \"embedding_files_test/embedding_content_99996.txt\"\n",
    "with open(filename , \"r\") as f:\n",
    "    content = f.read()\n",
    "    print(content)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"embedding_files/embedding_content_10350.txt\"\n",
    "# count the number of words in the document without a counter\n",
    "with open(filename , \"r\") as f:\n",
    "    content = f.read()\n",
    "    print(len(content.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(Counter(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py312",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}