docs/content/use_cases/fine_tune_llm_on_database.ipynb

{"metadata": {"kernelspec": {"display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.2"}}, "nbformat": 4, "nbformat_minor": 5, "cells": [{"cell_type": "markdown", "id": "1d92f42b-7160-4965-a0ef-5e4ce46bd529", "metadata": {}, "source": ["<!-- TABS -->\n", "# Fine tune LLM on database"]}, {"cell_type": "markdown", "id": "f7a4aab8-86eb-4e1c-9200-0a16ba75b2e6", "metadata": {}, "source": ["<!-- TABS -->\n", "## Configure your production system"]}, {"cell_type": "markdown", "id": "81e7cd59-67d0-4776-aea1-4864aa768f95", "metadata": {}, "source": [":::note\n", "If you would like to use the production features \n", "of SuperDuperDB, then you should set the relevant \n", "connections and configurations in a configuration \n", "file. Otherwise you are welcome to use \"development\" mode \n", "to get going with SuperDuperDB quickly.\n", ":::"]}, {"cell_type": "code", "execution_count": null, "id": "62014646-ccd4-4d10-ac26-1c470f88f2f2", "metadata": {}, "outputs": [], "source": ["import os\n", "\n", "os.makedirs('.superduperdb', exist_ok=True)\n", "os.environ['SUPERDUPERDB_CONFIG'] = '.superduperdb/config.yaml'"]}, {"cell_type": "code", "execution_count": null, "id": "8e50edd2-438d-44ab-9da0-0b72197df262", "metadata": {}, "outputs": [], "source": ["# <tab: MongoDB Community>\n", "CFG = '''\n", "data_backend: mongodb://127.0.0.1:27017/documents\n", "artifact_store: filesystem://./artifact_store\n", "cluster:\n", "  cdc:\n", "    strategy: null\n", "    uri: ray://127.0.0.1:20000\n", "  compute:\n", "    uri: ray://127.0.0.1:10001\n", "  vector_search:\n", "    backfill_batch_size: 100\n", "    type: in_memory\n", "    uri: http://127.0.0.1:21000\n", "'''"]}, {"cell_type": "code", "execution_count": null, "id": "1ad9ee67-6402-45ea-8311-3efb039b5df3", "metadata": {}, "outputs": [], "source": ["# <tab: MongoDB Atlas>\n", "CFG = '''\n", "artifact_store: filesystem://<path-to-artifact-store>\n", "cluster: \n", "    compute: ray://<ray-host>\n", "    cdc:    \n", "        uri: http://<cdc-host>:<cdc-port>\n", "    vector_search:\n", "        uri: http://<vector-search-host>:<vector-search-port>\n", "        type: native\n", "databackend: mongodb+srv://<user>:<password>@<mongo-host>:27017/documents\n", "'''"]}, {"cell_type": "code", "execution_count": null, "id": "9c9e8351-b17f-4882-bda6-5ad51dbc7e1f", "metadata": {}, "outputs": [], "source": ["# <tab: SQLite>\n", "CFG = '''\n", "artifact_store: filesystem://<path-to-artifact-store>\n", "cluster: \n", "    compute: ray://<ray-host>\n", "    cdc:    \n", "        uri: http://<cdc-host>:<cdc-port>\n", "    vector_search:\n", "        uri: http://<vector-search-host>:<vector-search-port>\n", "databackend: sqlite://<path-to-db>.db\n", "'''"]}, {"cell_type": "code", "execution_count": null, "id": "d16c66bb-6ff2-4cea-b11c-0a65bf86c7ad", "metadata": {}, "outputs": [], "source": ["# <tab: MySQL>\n", "CFG = '''\n", "artifact_store: filesystem://<path-to-artifact-store>\n", "cluster: \n", "    compute: ray://<ray-host>\n", "    cdc:    \n", "        uri: http://<cdc-host>:<cdc-port>\n", "    vector_search:\n", "        uri: http://<vector-search-host>:<vector-search-port>\n", "databackend: mysql://<user>:<password>@<host>:<port>/database\n", "'''"]}, {"cell_type": "code", "execution_count": null, "id": "9b7ac715-712c-4ec7-be90-0aaa22518977", "metadata": {}, "outputs": [], "source": ["# <tab: Oracle>\n", "CFG = '''\n", "artifact_store: filesystem://<path-to-artifact-store>\n", "cluster: \n", "    compute: ray://<ray-host>\n", "    cdc:    \n", "        uri: http://<cdc-host>:<cdc-port>\n", "    vector_search:\n", "        uri: http://<vector-search-host>:<vector-search-port>\n", "databackend: mssql://<user>:<password>@<host>:<port>\n", "'''"]}, {"cell_type": "code", "execution_count": null, "id": "f21fad9c-cc0e-4cf5-83f0-41a3a614c6af", "metadata": {}, "outputs": [], "source": ["# <tab: PostgreSQL>\n", "CFG = '''\n", "artifact_store: filesystem://<path-to-artifact-store>\n", "cluster: \n", "    compute: ray://<ray-host>\n", "    cdc:    \n", "        uri: http://<cdc-host>:<cdc-port>\n", "    vector_search:\n", "        uri: http://<vector-search-host>:<vector-search-port>\n", "databackend: postgres://<user>:<password>@<host>:<port</<database>\n", "'''"]}, {"cell_type": "code", "execution_count": null, "id": "1badb5a3-823c-4463-ab79-6f4f9239dabe", "metadata": {}, "outputs": [], "source": ["# <tab: Snowflake>\n", "CFG = '''\n", "artifact_store: filesystem://<path-to-artifact-store>\n", "metadata_store: sqlite://<path-to-sqlite-db>.db\n", "cluster: \n", "    compute: ray://<ray-host>\n", "    cdc:    \n", "        uri: http://<cdc-host>:<cdc-port>\n", "    vector_search:\n", "        uri: http://<vector-search-host>:<vector-search-port>\n", "databackend: snowflake://<user>:<password>@<account>/<database>\n", "'''"]}, {"cell_type": "code", "execution_count": null, "id": "ae7807d9-9fc1-4c18-8027-a512f827783d", "metadata": {}, "outputs": [], "source": ["# <tab: Clickhouse>\n", "CFG = '''\n", "artifact_store: filesystem://<path-to-artifact-store>\n", "metadata_store: sqlite://<path-to-sqlite-db>.db\n", "cluster: \n", "    compute: ray://<ray-host>\n", "    cdc:    \n", "        uri: http://<cdc-host>:<cdc-port>\n", "    vector_search:\n", "        uri: http://<vector-search-host>:<vector-search-port>\n", "databackend: clickhouse://<user>:<password>@<host>:<port>\n", "'''"]}, {"cell_type": "code", "execution_count": null, "id": "fc40c13b-9bc5-47ac-86d6-ef7a379c45ee", "metadata": {}, "outputs": [], "source": ["with open(os.environ['SUPERDUPERDB_CONFIG'], 'w') as f:\n", "    f.write(CFG)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["<!-- TABS -->\n", "## Start your cluster"]}, {"cell_type": "markdown", "metadata": {}, "source": [":::note\n", "Starting a SuperDuperDB cluster is useful in production and model development\n", "if you want to enable scalable compute, access to the models by multiple users for collaboration, \n", "monitoring.\n", "\n", "If you don't need this, then it is simpler to start in development mode.\n", ":::"]}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["# <tab: Experimental Cluster>\n", "!python -m superduperdb local-cluster up"]}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["# <tab: Docker-Compose>\n", "!make build_sandbox\n", "!make testenv_init"]}, {"cell_type": "markdown", "id": "32f8484d-2e35-472a-9b24-1a30ec1d144b", "metadata": {}, "source": ["<!-- TABS -->\n", "## Connect to SuperDuperDB"]}, {"cell_type": "markdown", "id": "06d66021-ce62-4021-a2c5-158dee92b3bb", "metadata": {}, "source": [":::note\n", "Note that this is only relevant if you are running SuperDuperDB in development mode.\n", "Otherwise refer to \"Configuring your production system\".\n", ":::"]}, {"cell_type": "code", "execution_count": null, "id": "61976f44-8139-41c0-a73e-569c6d16c4b1", "metadata": {}, "outputs": [], "source": ["# <tab: MongoDB>\n", "from superduperdb import superduper\n", "\n", "db = superduper('mongodb://localhost:27017/documents')"]}, {"cell_type": "code", "execution_count": null, "id": "e981a457", "metadata": {}, "outputs": [], "source": ["# <tab: SQLite>\n", "from superduperdb import superduper\n", "db = superduper('sqlite://my_db.db')"]}, {"cell_type": "code", "execution_count": null, "id": "19ecf7c0-b730-4503-9b5d-e97697b3bcee", "metadata": {}, "outputs": [], "source": ["# <tab: MySQL>\n", "from superduperdb import superduper\n", "\n", "user = 'superduper'\n", "password = 'superduper'\n", "port = 3306\n", "host = 'localhost'\n", "database = 'test_db'\n", "\n", "db = superduper(f\"mysql://{user}:{password}@{host}:{port}/{database}\")"]}, {"cell_type": "code", "execution_count": null, "id": "df208e8c-4fd0-438f-af29-22a763a2aebd", "metadata": {}, "outputs": [], "source": ["# <tab: Oracle>\n", "from superduperdb import superduper\n", "\n", "user = 'sa'\n", "password = 'Superduper#1'\n", "port = 1433\n", "host = 'localhost'\n", "\n", "db = superduper(f\"mssql://{user}:{password}@{host}:{port}\")"]}, {"cell_type": "code", "execution_count": null, "id": "d2297295", "metadata": {}, "outputs": [], "source": ["# <tab: PostgreSQL>\n", "!pip install psycopg2\n", "from superduperdb import superduper\n", "\n", "user = 'postgres'\n", "password = 'postgres'\n", "port = 5432\n", "host = 'localhost'\n", "database = 'test_db'\n", "db_uri = f\"postgres://{user}:{password}@{host}:{port}/{database}\"\n", "\n", "db = superduper(db_uri, metadata_store=db_uri.replace('postgres://', 'postgresql://'))"]}, {"cell_type": "code", "execution_count": null, "id": "cc6c8517", "metadata": {}, "outputs": [], "source": ["# <tab: Snowflake>\n", "from superduperdb import superduper\n", "\n", "user = \"superduperuser\"\n", "password = \"superduperpassword\"\n", "account = \"XXXX-XXXX\"  # ORGANIZATIONID-USERID\n", "database = \"FREE_COMPANY_DATASET/PUBLIC\"\n", "\n", "snowflake_uri = f\"snowflake://{user}:{password}@{account}/{database}\"\n", "\n", "db = superduper(\n", "    snowflake_uri, \n", "    metadata_store='sqlite:///your_database_name.db',\n", ")"]}, {"cell_type": "code", "execution_count": null, "id": "05da45e3-d9e4-49ca-b9ee-db1b8bf4eb44", "metadata": {}, "outputs": [], "source": ["# <tab: Clickhouse>\n", "from superduperdb import superduper\n", "\n", "user = 'default'\n", "password = ''\n", "port = 8123\n", "host = 'localhost'\n", "\n", "db = superduper(f\"clickhouse://{user}:{password}@{host}:{port}\", metadata_store=f'mongomock://meta')"]}, {"cell_type": "code", "execution_count": null, "id": "0e89c8dd-d845-423a-9acc-97e3360d370c", "metadata": {}, "outputs": [], "source": ["# <tab: DuckDB>\n", "from superduperdb import superduper\n", "\n", "db = superduper('duckdb://mydb.duckdb')"]}, {"cell_type": "code", "execution_count": null, "id": "2de71562", "metadata": {}, "outputs": [], "source": ["# <tab: Pandas>\n", "from superduperdb import superduper\n", "\n", "db = superduper(['my.csv'], metadata_store=f'mongomock://meta')"]}, {"cell_type": "code", "execution_count": null, "id": "cb029a5e-fedf-4f07-8a31-d220cfbfbb3d", "metadata": {}, "outputs": [], "source": ["# <tab: MongoMock>\n", "from superduperdb import superduper\n", "\n", "db = superduper('mongomock:///test_db')"]}, {"cell_type": "markdown", "id": "511b9f96-cccd-471c-86e2-b10da1dc1ad5", "metadata": {}, "source": ["## Install related dependencies"]}, {"cell_type": "code", "execution_count": null, "id": "f3a2f58c-db34-4808-ab2f-8933bd452853", "metadata": {}, "outputs": [], "source": ["!pip install transformers torch accelerate trl peft datasets"]}, {"cell_type": "markdown", "id": "032c2e7b-3f54-4263-b778-0fef60596efb", "metadata": {}, "source": ["<!-- TABS -->\n", "## Get LLM Finetuning Data"]}, {"cell_type": "markdown", "id": "33f5169e-ab2f-4eac-bd3f-30fd845f2a1b", "metadata": {}, "source": ["The following are examples of training data in different formats."]}, {"cell_type": "code", "execution_count": null, "id": "9b37c7dc-390a-428b-916a-09d191678cbc", "metadata": {}, "outputs": [], "source": ["# <tab: Text>\n", "from datasets import load_dataset\n", "from superduperdb.base.document import Document\n", "dataset_name = \"timdettmers/openassistant-guanaco\"\n", "dataset = load_dataset(dataset_name)\n", "\n", "train_dataset = dataset[\"train\"]\n", "eval_dataset = dataset[\"test\"]\n", "\n", "train_documents = [\n", "    Document({**example, \"_fold\": \"train\"})\n", "    for example in train_dataset\n", "]\n", "eval_documents = [\n", "    Document({**example, \"_fold\": \"valid\"})\n", "    for example in eval_dataset\n", "]\n", "\n", "datas = train_documents + eval_documents"]}, {"cell_type": "code", "execution_count": null, "id": "4e7902bd", "metadata": {}, "outputs": [], "source": ["# <tab: Prompt-Response>\n", "from datasets import load_dataset\n", "from superduperdb.base.document import Document\n", "dataset_name = \"mosaicml/instruct-v3\"\n", "dataset = load_dataset(dataset_name)\n", "\n", "train_dataset = dataset[\"train\"]\n", "eval_dataset = dataset[\"test\"]\n", "\n", "train_documents = [\n", "    Document({**example, \"_fold\": \"train\"})\n", "    for example in train_dataset\n", "]\n", "eval_documents = [\n", "    Document({**example, \"_fold\": \"valid\"})\n", "    for example in eval_dataset\n", "]\n", "\n", "datas = train_documents + eval_documents"]}, {"cell_type": "code", "execution_count": null, "id": "a9c05195-3372-48c2-95c8-5ef51d65bcfe", "metadata": {}, "outputs": [], "source": ["# <tab: Chat>\n", "from datasets import load_dataset\n", "from superduperdb.base.document import Document\n", "dataset_name = \"philschmid/dolly-15k-oai-style\"\n", "dataset = load_dataset(dataset_name)['train'].train_test_split(0.9)\n", "\n", "train_dataset = dataset[\"train\"]\n", "eval_dataset = dataset[\"test\"]\n", "\n", "train_documents = [\n", "    Document({**example, \"_fold\": \"train\"})\n", "    for example in train_dataset\n", "]\n", "eval_documents = [\n", "    Document({**example, \"_fold\": \"valid\"})\n", "    for example in eval_dataset\n", "]\n", "\n", "datas = train_documents + eval_documents"]}, {"cell_type": "markdown", "id": "361a4705-e7d6-4244-9150-bfa8372f85ba", "metadata": {}, "source": ["We can define different training parameters to handle this type of data."]}, {"cell_type": "code", "execution_count": null, "id": "c824212e-0c4f-4b93-b3fa-4d2a105fc655", "metadata": {}, "outputs": [], "source": ["# <tab: Text>\n", "# Function for transformation after extracting data from the database\n", "transform = None\n", "key = ('text')\n", "training_kwargs=dict(dataset_text_field=\"text\")"]}, {"cell_type": "code", "execution_count": null, "id": "1c2d583a-a0f3-432d-b737-356ab3cd4378", "metadata": {}, "outputs": [], "source": ["# <tab: Prompt-Response>\n", "# Function for transformation after extracting data from the database\n", "def transform(prompt, response):\n", "    return {'text': prompt + response + \"</s>\"}\n", "\n", "key = ('prompt', 'response')\n", "training_kwargs=dict(dataset_text_field=\"text\")"]}, {"cell_type": "code", "execution_count": null, "id": "225cdb09-d060-4d45-bcf3-cae92fb22ee8", "metadata": {}, "outputs": [], "source": ["# <tab: Chat>\n", "# Function for transformation after extracting data from the database\n", "transform = None\n", "\n", "key = ('messages')\n", "training_kwargs=None"]}, {"cell_type": "markdown", "id": "ca7a1ec0-bf28-4b59-8be1-e7bcfd4eeccc", "metadata": {}, "source": ["Example input_text and output_text"]}, {"cell_type": "code", "execution_count": null, "id": "7eb8c36c-97f8-40f4-8b8d-736d55352138", "metadata": {}, "outputs": [], "source": ["# <tab: Text>\n", "data = datas[0]\n", "input_text, output_text = data[\"text\"].rsplit(\"### Assistant: \", maxsplit=1)\n", "input_text += \"### Assistant: \"\n", "output_text = output_text.rsplit(\"### Human:\")[0]\n", "print(\"Input: --------------\")\n", "print(input_text)\n", "print(\"Response: --------------\")\n", "print(output_text)"]}, {"cell_type": "code", "execution_count": null, "id": "6dbef4a4-478d-43f1-8f40-b3b1e5923639", "metadata": {}, "outputs": [], "source": ["# <tab: Prompt-Response>\n", "data = datas[0]\n", "input_text = data[\"prompt\"]\n", "output_text = data[\"response\"]\n", "print(\"Input: --------------\")\n", "print(input_text)\n", "print(\"Response: --------------\")\n", "print(output_text)"]}, {"cell_type": "code", "execution_count": null, "id": "983e8612-9c58-4688-a3af-8c408f9b3063", "metadata": {}, "outputs": [], "source": ["# <tab: Chat>\n", "data = datas[0]\n", "messages = data[\"messages\"]\n", "input_text = messages[:-1]\n", "output_text = messages[-1][\"content\"]\n", "print(\"Input: --------------\")\n", "print(input_text)\n", "print(\"Response: --------------\")\n", "print(output_text)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["<!-- TABS -->\n", "## Insert simple data\n", "\n", "After turning on auto_schema, we can directly insert data, and superduperdb will automatically analyze the data type, and match the construction of the table and datatype."]}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["from superduperdb import Document\n", "\n", "table_or_collection = db['documents']\n", "\n", "ids = db.execute(table_or_collection.insert([Document(data) for data in datas]))\n", "select = table_or_collection.select()"]}, {"cell_type": "markdown", "id": "2a46a283-4fef-40d8-8df4-7023479ec2dd", "metadata": {}, "source": ["## Select a Model"]}, {"cell_type": "code", "execution_count": null, "id": "a2ef0e1e-22ea-4af3-b914-1a3eed23a754", "metadata": {}, "outputs": [], "source": ["model_name = \"facebook/opt-125m\"\n", "model_kwargs = dict()\n", "tokenizer_kwargs = dict()\n", "\n", "# or \n", "# model_name = \"mistralai/Mistral-7B-Instruct-v0.2\"\n", "# token = \"hf_xxxx\"\n", "# model_kwargs = dict(token=token)\n", "# tokenizer_kwargs = dict(token=token)"]}, {"cell_type": "markdown", "id": "fcf339d2-6eaa-4a90-8718-6d6e6120c400", "metadata": {}, "source": ["<!-- TABS -->\n", "## Build A Trainable LLM"]}, {"cell_type": "markdown", "id": "1f6c2662-ce55-4767-8e3d-ef3901fd31ee", "metadata": {}, "source": ["**Create an LLM Trainer for training**\n", "\n", "The parameters of this LLM Trainer are basically the same as `transformers.TrainingArguments`, but some additional parameters have been added for easier training setup."]}, {"cell_type": "code", "execution_count": null, "id": "5deed34b-189c-4662-8972-aed92718225d", "metadata": {}, "outputs": [], "source": ["from superduperdb.ext.transformers import LLM, LLMTrainer\n", "trainer = LLMTrainer(\n", "    identifier=\"llm-finetune-trainer\",\n", "    output_dir=\"output/finetune\",\n", "    overwrite_output_dir=True,\n", "    num_train_epochs=3,\n", "    save_total_limit=3,\n", "    logging_steps=10,\n", "    evaluation_strategy=\"steps\",\n", "    save_steps=100,\n", "    eval_steps=100,\n", "    per_device_train_batch_size=1,\n", "    per_device_eval_batch_size=1,\n", "    gradient_accumulation_steps=2,\n", "    max_seq_length=512,\n", "    key=key,\n", "    select=select,\n", "    transform=transform,\n", "    training_kwargs=training_kwargs,\n", ")"]}, {"cell_type": "code", "execution_count": null, "id": "089bc70f-00e0-4a13-a438-658146efd4a4", "metadata": {}, "outputs": [], "source": ["# <tab: Lora>\n", "trainer.use_lora = True"]}, {"cell_type": "code", "execution_count": null, "id": "30ab34bc-999c-4300-aa47-d40a78536d61", "metadata": {}, "outputs": [], "source": ["# <tab: QLora>\n", "trainer.use_lora = True\n", "trainer.bits = 4"]}, {"cell_type": "code", "execution_count": null, "id": "1392ffec-80aa-4be7-b40f-665c2803e980", "metadata": {}, "outputs": [], "source": ["# <tab: Deepspeed>\n", "!pip install deepspeed\n", "deepspeed = {\n", "    \"train_batch_size\": \"auto\",\n", "    \"train_micro_batch_size_per_gpu\": \"auto\",\n", "    \"gradient_accumulation_steps\": \"auto\",\n", "    \"zero_optimization\": {\n", "        \"stage\": 2,\n", "    },\n", "}\n", "trainer.use_lora = True\n", "trainer.bits = 4\n", "trainer.deepspeed = deepspeed"]}, {"cell_type": "code", "execution_count": null, "id": "f414fc61-2466-4bf6-8ea2-460524806880", "metadata": {}, "outputs": [], "source": ["# <tab: Multi-GPUS>\n", "trainer.use_lora = True\n", "trainer.bits = 4\n", "trainer.num_gpus = 2"]}, {"cell_type": "markdown", "id": "a370f6c8-ce2f-443e-ae89-a4e95c4375a8", "metadata": {}, "source": ["Create a trainable LLM model and add it to the database, then the training task will run automatically."]}, {"cell_type": "code", "execution_count": null, "id": "400c1ebb-345e-4030-9f9e-96099f53664c", "metadata": {}, "outputs": [], "source": ["llm = LLM(\n", "    identifier=\"llm\",\n", "    model_name_or_path=model_name,\n", "    trainer=trainer,\n", "    model_kwargs=model_kwargs,\n", "    tokenizer_kwargs=tokenizer_kwargs,\n", ")\n", "\n", "db.apply(llm)"]}, {"cell_type": "markdown", "id": "7edd846d-aa81-456f-b2ea-fc2d230a41a2", "metadata": {}, "source": ["## Load the trained model\n", "There are two methods to load a trained model:\n", "\n", "- **Load the model directly**: This will load the model with the best metrics (if the transformers' best model save strategy is set) or the last version of the model.\n", "- **Use a specified checkpoint**: This method downloads the specified checkpoint, then initializes the base model, and finally merges the checkpoint with the base model. This approach supports custom operations such as resetting flash_attentions, model quantization, etc., during initialization."]}, {"cell_type": "code", "execution_count": null, "id": "db2e1a0d-c760-4a01-b4bf-c6e83296ca8e", "metadata": {}, "outputs": [], "source": ["# <tab: Load Trained Model Directly>\n", "llm = db.load(\"model\", \"llm\")"]}, {"cell_type": "code", "execution_count": null, "id": "ea5fbf38-8f2b-4c3f-9ae8-c184001b4495", "metadata": {}, "outputs": [], "source": ["# <tab: Use a specified checkpoint>\n", "from superduperdb.ext.transformers import LLM, LLMTrainer\n", "experiment_id = db.show(\"checkpoint\")[-1]\n", "version = None # None means the last checkpoint\n", "checkpoint = db.load(\"checkpoint\", experiment_id, version=version)\n", "llm = LLM(\n", "    identifier=\"llm\",\n", "    model_name_or_path=model_name,\n", "    adapter_id=checkpoint,\n", "    model_kwargs=dict(load_in_4bit=True)\n", ")"]}, {"cell_type": "code", "execution_count": null, "id": "60b8933b-723d-481c-9d6e-dff14d256377", "metadata": {}, "outputs": [], "source": ["llm.predict(input_text, max_new_tokens=200)"]}]}