From 064720c89db66cadc277933a52c0dcdd343724a1 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Tue, 16 Sep 2025 10:19:38 -0400 Subject: [PATCH 01/22] Updated for AIM405 --- .../05.01_fine-tuning-pipeline.ipynb | 2233 +++++++++-------- .../task_05_fmops/scripts/requirements.txt | 3 +- .../task_05_fmops/steps/__init__.py | 0 .../task_05_fmops/steps/deploy_step.py | 114 + .../task_05_fmops/steps/evaluation_mlflow.py | 61 - .../steps/finetune_llama3b_hf.py | 96 - .../task_05_fmops/steps/finetune_step.py | 244 ++ .../steps/model_registration_step.py | 465 ++++ .../task_05_fmops/steps/pipeline_utils.py | 104 + .../task_05_fmops/steps/preprocess_llama3.py | 64 - .../task_05_fmops/steps/preprocess_step.py | 218 ++ .../steps/qualitative_eval_step.py | 486 ++++ .../steps/quantitative_eval_step.py | 410 +++ .../task_05_fmops/steps/utils.py | 20 - 14 files changed, 3210 insertions(+), 1308 deletions(-) create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/__init__.py create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py delete mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/evaluation_mlflow.py delete mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_llama3b_hf.py create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_step.py create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/model_registration_step.py create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py delete mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_llama3.py create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py delete mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/utils.py diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index da7c626..46172d2 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -47,18 +47,53 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 41, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T16:13:00.445227Z", + "iopub.status.busy": "2025-09-15T16:13:00.445009Z", + "iopub.status.idle": "2025-09-15T16:13:03.807709Z", + "shell.execute_reply": "2025-09-15T16:13:03.807149Z", + "shell.execute_reply.started": "2025-09-15T16:13:00.445211Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install -r ./scripts/requirements.txt --upgrade --quiet" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 42, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T16:13:03.808639Z", + "iopub.status.busy": "2025-09-15T16:13:03.808465Z", + "iopub.status.idle": "2025-09-15T16:13:03.812578Z", + "shell.execute_reply": "2025-09-15T16:13:03.812157Z", + "shell.execute_reply.started": "2025-09-15T16:13:03.808620Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'status': 'ok', 'restart': True}" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from IPython import get_ipython\n", "get_ipython().kernel.do_shutdown(True)" @@ -75,9 +110,26 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:00.218254Z", + "iopub.status.busy": "2025-09-15T19:40:00.218098Z", + "iopub.status.idle": "2025-09-15T19:40:01.616938Z", + "shell.execute_reply": "2025-09-15T19:40:01.616490Z", + "shell.execute_reply.started": "2025-09-15T19:40:00.218239Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n" + ] + } + ], "source": [ "import os\n", "import boto3\n", @@ -97,7 +149,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### 2. SageMaker Session and IAM Role" + "### 2. SageMaker Session and IAM Role" ] }, { @@ -109,14 +161,23 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:03.372461Z", + "iopub.status.busy": "2025-09-15T19:40:03.372249Z", + "iopub.status.idle": "2025-09-15T19:40:03.839460Z", + "shell.execute_reply": "2025-09-15T19:40:03.838980Z", + "shell.execute_reply.started": "2025-09-15T19:40:03.372444Z" + } + }, "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", "role = sagemaker.get_execution_role()\n", + "role = \"arn:aws:iam::329542461890:role/data-scientist-role\"\n", "instance_type = \"ml.m5.xlarge\"\n", - "pipeline_name = \"deepseek-finetune-pipeline\"\n", + "pipeline_name = \"AIM405-deepseek-finetune-pipeline\"\n", "bucket_name = sagemaker_session.default_bucket()\n", "default_prefix = sagemaker_session.default_bucket_prefix\n", "if default_prefix:\n", @@ -153,11 +214,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 3, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:04.974625Z", + "iopub.status.busy": "2025-09-15T19:40:04.974410Z", + "iopub.status.idle": "2025-09-15T19:40:04.977645Z", + "shell.execute_reply": "2025-09-15T19:40:04.977162Z", + "shell.execute_reply.started": "2025-09-15T19:40:04.974607Z" + } + }, "outputs": [], "source": [ - "mlflow_tracking_server_arn = None\n", + "mlflow_tracking_server_arn = \"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server/my-tracking-server\"\n", "\n", "if not mlflow_tracking_server_arn:\n", " try:\n", @@ -175,9 +244,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 4, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:05.424627Z", + "iopub.status.busy": "2025-09-15T19:40:05.424466Z", + "iopub.status.idle": "2025-09-15T19:40:05.428129Z", + "shell.execute_reply": "2025-09-15T19:40:05.427674Z", + "shell.execute_reply.started": "2025-09-15T19:40:05.424614Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting config.yaml\n" + ] + } + ], "source": [ "%%writefile config.yaml\n", "SchemaVersion: '1.0'\n", @@ -198,8 +283,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 5, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:05.653390Z", + "iopub.status.busy": "2025-09-15T19:40:05.653238Z", + "iopub.status.idle": "2025-09-15T19:40:05.655757Z", + "shell.execute_reply": "2025-09-15T19:40:05.655243Z", + "shell.execute_reply.started": "2025-09-15T19:40:05.653376Z" + } + }, "outputs": [], "source": [ "# Set path to config file\n", @@ -210,24 +303,98 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Download Model Data from Huggingface" + "### 4. Download Model Data from Huggingface" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:06.298968Z", + "iopub.status.busy": "2025-09-15T19:40:06.298787Z", + "iopub.status.idle": "2025-09-15T19:40:07.006290Z", + "shell.execute_reply": "2025-09-15T19:40:07.005739Z", + "shell.execute_reply.started": "2025-09-15T19:40:06.298955Z" + }, "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading model deepseek-ai/DeepSeek-R1-Distill-Llama-8B\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7fa82b75a9294a348461ae5fa30bca99", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 11 files: 0%| | 0/11 [00:00 tuple:\n", - " import boto3\n", - " import shutil\n", - " import sagemaker\n", - " import os\n", - " import pandas as pd\n", - " from sagemaker.config import load_sagemaker_config\n", - " import mlflow\n", - " import traceback\n", - " from datasets import load_dataset\n", - " from sklearn.model_selection import train_test_split\n", - " from datasets import Dataset, DatasetDict\n", - " from random import randint\n", - "\n", - " mlflow.set_tracking_uri(tracking_server_arn)\n", - " mlflow.set_experiment(experiment_name)\n", - "\n", - " # Preprocessing code - runs regardless of MLflow status\n", - " try:\n", - " with mlflow.start_run(run_name=run_name) as run:\n", - " run_id = run.info.run_id\n", - " with mlflow.start_run(run_name=\"Processing\", nested=True):\n", - " #mlflow.autolog()\n", - " # Initialize SageMaker and S3 clients\n", - " sagemaker_session = sagemaker.Session()\n", - " s3_client = boto3.client('s3')\n", - " \n", - " bucket_name = sagemaker_session.default_bucket()\n", - " default_prefix = sagemaker_session.default_bucket_prefix\n", - " configs = load_sagemaker_config()\n", - " \n", - " # Load dataset with proper error handling\n", - " sample_dataset_size = 100\n", - " try:\n", - " dataset = load_dataset(\"FreedomIntelligence/medical-o1-reasoning-SFT\", \"en\")\n", - " except Exception as e:\n", - " error_msg = f\"Error loading dataset: {str(e)}\\n{traceback.format_exc()}\"\n", - " print(error_msg)\n", - " raise RuntimeError(f\"Failed to load dataset: {str(e)}\")\n", - " \n", - " df = pd.DataFrame(dataset['train'])\n", - " df = df[:sample_dataset_size]\n", - " \n", - " # Split dataset\n", - " train, test = train_test_split(df, test_size=0.1, random_state=42, shuffle=True)\n", - " \n", - " print(\"Number of train elements: \", len(train))\n", - " print(\"Number of test elements: \", len(test))\n", - " \n", - " # Log dataset statistics if MLflow is enabled\n", - " mlflow.log_param(\"dataset_source\", \"FreedomIntelligence/medical-o1-reasoning-SFT\")\n", - " mlflow.log_param(\"train_size\", len(train))\n", - " mlflow.log_param(\"test_size\", len(test))\n", - " mlflow.log_param(\"dataset_sample_size\", sample_dataset_size) # Log that we're using a subset of 100 samples\n", - " \n", - " # Define prompt template\n", - " prompt_template = f\"\"\"\n", - " <|begin_of_text|>\n", - " <|start_header_id|>system<|end_header_id|>\n", - " You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", - " Below is an instruction that describes a task, paired with an input that provides further context. \n", - " Write a response that appropriately completes the request.\n", - " Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", - " <|eot_id|><|start_header_id|>user<|end_header_id|>\n", - " {{question}}<|eot_id|>\n", - " <|start_header_id|>assistant<|end_header_id|>\n", - " {{complex_cot}}\n", - " \n", - " {{answer}}\n", - " <|eot_id|>\n", - " \"\"\"\n", - " \n", - " # Template dataset to add prompt to each sample\n", - " def template_dataset(sample):\n", - " try:\n", - " sample[\"text\"] = prompt_template.format(question=sample[\"Question\"],\n", - " complex_cot=sample[\"Complex_CoT\"],\n", - " answer=sample[\"Response\"])\n", - " return sample\n", - " except KeyError as e:\n", - " print(f\"KeyError in template_dataset: {str(e)}\")\n", - " # Provide default values for missing fields\n", - " missing_key = str(e).strip(\"'\")\n", - " if missing_key == \"Question\":\n", - " sample[\"text\"] = prompt_template.format(\n", - " question=\"[Missing question]\",\n", - " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", - " answer=sample.get(\"Response\", \"[Missing response]\")\n", - " )\n", - " elif missing_key == \"Complex_CoT\":\n", - " sample[\"text\"] = prompt_template.format(\n", - " question=sample[\"Question\"],\n", - " complex_cot=\"[Missing CoT]\",\n", - " answer=sample.get(\"Response\", \"[Missing response]\")\n", - " )\n", - " elif missing_key == \"Response\":\n", - " sample[\"text\"] = prompt_template.format(\n", - " question=sample[\"Question\"],\n", - " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", - " answer=\"[Missing response]\"\n", - " )\n", - " return sample\n", - " \n", - " # Create datasets\n", - " train_dataset = Dataset.from_pandas(train)\n", - " test_dataset = Dataset.from_pandas(test)\n", - " \n", - " dataset = DatasetDict({\"train\": train_dataset, \"test\": test_dataset})\n", - " \n", - " train_dataset = dataset[\"train\"].map(template_dataset, remove_columns=list(dataset[\"train\"].features))\n", - " \n", - " # Safely get a sample text, handling potential index errors\n", - " try:\n", - " sample_index = randint(0, len(train_dataset) - 1)\n", - " sample_text = train_dataset[sample_index][\"text\"]\n", - " print(f\"Sample text from index {sample_index}:\")\n", - " print(sample_text)\n", - " except (IndexError, KeyError) as e:\n", - " sample_text = \"Error retrieving sample text: \" + str(e)\n", - " print(sample_text)\n", - " \n", - " test_dataset = dataset[\"test\"].map(template_dataset, remove_columns=list(dataset[\"test\"].features))\n", - " \n", - " # Set paths\n", - " if default_prefix:\n", - " input_path = f'{default_prefix}/datasets/llm-fine-tuning-modeltrainer-sft'\n", - " else:\n", - " input_path = f'datasets/llm-fine-tuning-modeltrainer-sft'\n", - " \n", - " # Create directories with error handling\n", - " try:\n", - " os.makedirs(\"./data/train\", exist_ok=True)\n", - " os.makedirs(\"./data/test\", exist_ok=True)\n", - " except OSError as e:\n", - " error_msg = f\"Error creating directories: {str(e)}\"\n", - " print(error_msg)\n", - " # Continue with execution as we'll try to save files anyway\n", - " \n", - " # Save datasets locally with error handling\n", - " try:\n", - " train_dataset.to_json(\"./data/train/dataset.json\", orient=\"records\")\n", - " test_dataset.to_json(\"./data/test/dataset.json\", orient=\"records\")\n", - " except Exception as e:\n", - " error_msg = f\"Error saving datasets locally: {str(e)}\\n{traceback.format_exc()}\"\n", - " print(error_msg)\n", - " raise RuntimeError(f\"Failed to save datasets locally: {str(e)}\")\n", - " \n", - " # Define S3 paths\n", - " train_data_path = f\"s3://{bucket_name}/{input_path}/train/dataset.json\"\n", - " test_dataset_path = f\"s3://{bucket_name}/{input_path}/test/dataset.json\"\n", - " \n", - " # Store results for return\n", - " result_train_data_path = train_data_path\n", - " result_test_dataset_path = test_dataset_path\n", - " \n", - " # Log dataset paths if MLflow is enabled\n", - " mlflow.log_param(\"train_data_path\", train_data_path)\n", - " mlflow.log_param(\"test_dataset_path\", test_dataset_path)\n", - " \n", - " # Upload files to S3 with retries\n", - " max_retries = 3\n", - " for attempt in range(max_retries):\n", - " try:\n", - " print(f\"Uploading train dataset to S3, attempt {attempt+1}/{max_retries}\")\n", - " s3_client.upload_file(\"./data/train/dataset.json\", bucket_name, f\"{input_path}/train/dataset.json\")\n", - " print(f\"Uploading test dataset to S3, attempt {attempt+1}/{max_retries}\")\n", - " s3_client.upload_file(\"./data/test/dataset.json\", bucket_name, f\"{input_path}/test/dataset.json\")\n", - " print(\"S3 upload successful\")\n", - " break\n", - " except Exception as e:\n", - " error_msg = f\"Error in S3 upload (attempt {attempt+1}/{max_retries}): {str(e)}\"\n", - " print(error_msg)\n", - " if attempt == max_retries - 1: # Last attempt failed\n", - " raise RuntimeError(f\"Failed to upload datasets to S3 after {max_retries} attempts: {str(e)}\")\n", - " \n", - " print(f\"Datasets uploaded to:\")\n", - " print(train_data_path)\n", - " print(test_dataset_path)\n", - " \n", - " # Log a sample of the dataset as an artifact if MLflow is enabled\n", - " try:\n", - " with open(\"./data/sample.txt\", \"w\") as f:\n", - " f.write(sample_text)\n", - " mlflow.log_artifact(\"./data/sample.txt\", \"dataset_samples\")\n", - " except Exception as e:\n", - " print(f\"Error logging sample as artifact: {str(e)}\")\n", - " \n", - " # Clean up\n", - " try:\n", - " if os.path.exists(\"./data\"):\n", - " shutil.rmtree(\"./data\")\n", - " except Exception as e:\n", - " print(f\"Warning: Error cleaning up temporary files: {str(e)}\")\n", - " \n", - " except Exception as e:\n", - " error_msg = f\"Critical error in preprocessing: {str(e)}\\n{traceback.format_exc()}\"\n", - " print(error_msg)\n", - " raise RuntimeError(f\"Preprocessing failed: {str(e)}\")\n", - " \n", - "\n", - " return run_id, result_train_data_path, result_test_dataset_path" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -593,8 +529,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 7, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:07.955020Z", + "iopub.status.busy": "2025-09-15T19:40:07.954829Z", + "iopub.status.idle": "2025-09-15T19:40:07.967364Z", + "shell.execute_reply": "2025-09-15T19:40:07.966956Z", + "shell.execute_reply.started": "2025-09-15T19:40:07.955004Z" + } + }, "outputs": [], "source": [ "%%bash\n", @@ -671,9 +615,27 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 8, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:08.997498Z", + "iopub.status.busy": "2025-09-15T19:40:08.997336Z", + "iopub.status.idle": "2025-09-15T19:40:09.565044Z", + "shell.execute_reply": "2025-09-15T19:40:09.564574Z", + "shell.execute_reply.started": "2025-09-15T19:40:08.997484Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Fetched defaults config from location: /home/sagemaker-user/generative-ai-on-amazon-sagemaker/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops\n", + "Training config uploaded to:\n", + "s3://sagemaker-us-east-1-329542461890/training_config/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/config/args.yaml\n" + ] + } + ], "source": [ "from sagemaker.s3 import S3Uploader\n", "\n", @@ -694,786 +656,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**Fine-tuning Step**\n", - "\n", - "This is where the actual model adaptation occurs. The step takes the preprocessed data and applies it to fine-tune the base LLM (in this case, a Deepseek model). It incorporates the LoRA technique for efficient adaptation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@step(\n", - " name=\"ModelFineTuning\",\n", - " instance_type=instance_type,\n", - " display_name=\"Model Fine Tuning\",\n", - " keep_alive_period_in_seconds=900,\n", - " dependencies=\"./scripts/requirements.txt\"\n", - ")\n", - "def train(\n", - " tracking_server_arn: str,\n", - " train_dataset_s3_path: str,\n", - " test_dataset_s3_path: str,\n", - " train_config_s3_path: str,\n", - " experiment_name: str,\n", - " model_id: str,\n", - " run_id: str,\n", - "):\n", - " import sagemaker\n", - " import boto3\n", - " import mlflow\n", - " import yaml\n", - " import json\n", - " import time\n", - " import datetime\n", - " import os\n", - " import traceback\n", - " import tempfile\n", - " from pathlib import Path\n", - " from sagemaker.pytorch import PyTorch\n", - " \n", - " # Initialize variables and tracking\n", - " start_time = time.time()\n", - " model_name = model_id.split(\"/\")[-1] if \"/\" in model_id else model_id\n", - " training_job_name = None\n", - " \n", - " mlflow.set_tracking_uri(tracking_server_arn)\n", - " mlflow.set_experiment(experiment_name)\n", - " \n", - " try:\n", - " with mlflow.start_run(run_id=run_id):\n", - " with mlflow.start_run(run_name=\"FinetuningStep\", nested=True) as training_run:\n", - " mlflow.autolog()\n", - " training_run_id = training_run.info.run_id\n", - " # Enable detailed tracking\n", - " mlflow.set_tag(\"component\", \"model_fine_tuning\")\n", - " mlflow.log_param(\"model_id\", model_id)\n", - " mlflow.log_param(\"train_dataset\", train_dataset_s3_path)\n", - " mlflow.log_param(\"test_dataset\", test_dataset_s3_path)\n", - " mlflow.log_param(\"training_start_time\", datetime.datetime.now().isoformat())\n", - " \n", - " # Download and parse the training config YAML to log hyperparameters\n", - " with tempfile.NamedTemporaryFile(delete=False) as tmp:\n", - " s3_client = boto3.client(\"s3\")\n", - " \n", - " # Parse S3 path\n", - " config_parts = train_config_s3_path.replace(\"s3://\", \"\").split(\"/\", 1)\n", - " bucket = config_parts[0]\n", - " key = config_parts[1]\n", - " \n", - " # Download config file\n", - " try:\n", - " s3_client.download_file(bucket, key, tmp.name)\n", - " # Parse the YAML config\n", - " with open(tmp.name, 'r') as f:\n", - " config = yaml.safe_load(f)\n", - " \n", - " # Log all hyperparameters from config\n", - " print(\"Logging hyperparameters to MLflow:\")\n", - " for param_name, param_value in config.items():\n", - " # Skip complex objects that can't be logged as parameters\n", - " if isinstance(param_value, (str, int, float, bool)):\n", - " print(f\" {param_name}: {param_value}\")\n", - " mlflow.log_param(param_name, param_value)\n", - " elif param_name == \"fsdp_config\" and isinstance(param_value, dict):\n", - " # Log nested config as JSON\n", - " mlflow.log_param(\"fsdp_config_json\", json.dumps(param_value))\n", - " \n", - " # Log file as artifact for reference\n", - " mlflow.log_artifact(tmp.name, \"training_config\")\n", - " \n", - " except Exception as e:\n", - " print(f\"Error parsing config file: {e}\")\n", - " \n", - " finally:\n", - " # Clean up temp file\n", - " if os.path.exists(tmp.name):\n", - " os.remove(tmp.name)\n", - " \n", - " # Launch the training job\n", - " job_name = f\"deepseek-finetune-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}\"\n", - " \n", - " sagemaker_session = sagemaker.Session()\n", - " \n", - " # Define metric definitions for more detailed CloudWatch metrics\n", - " metric_definitions = [\n", - " {'Name': 'loss', 'Regex': \"'loss':\\\\s*([0-9.]+)\"},\n", - " {'Name': 'epoch', 'Regex': \"'epoch':\\\\s*([0-9.]+)\"},\n", - " {'Name': 'train_loss', 'Regex': \"'train_loss':\\\\s*([0-9.]+)\"},\n", - " {'Name': 'lr', 'Regex': \"'learning_rate':\\\\s*([0-9.e-]+)\"},\n", - " {'Name': 'step', 'Regex': \"'step':\\\\s*([0-9.]+)\"},\n", - " {'Name': 'samples_per_second', 'Regex': \"'train_samples_per_second':\\\\s*([0-9.]+)\"},\n", - " ]\n", - " \n", - " # Log the metric definitions we're using\n", - " mlflow.log_param(\"tracked_metrics\", [m['Name'] for m in metric_definitions])\n", - " \n", - " pytorch_estimator = PyTorch(\n", - " entry_point='train.py',\n", - " source_dir=\"./scripts\",\n", - " job_name=job_name,\n", - " base_job_name=job_name,\n", - " max_run=50000,\n", - " role=role,\n", - " framework_version=\"2.2.0\",\n", - " py_version=\"py310\",\n", - " instance_count=1,\n", - " instance_type=\"ml.p3.2xlarge\",\n", - " sagemaker_session=sagemaker_session,\n", - " volume_size=50,\n", - " disable_output_compression=False,\n", - " keep_alive_period_in_seconds=1800,\n", - " distribution={\"torch_distributed\": {\"enabled\": True}},\n", - " hyperparameters={\n", - " \"config\": \"/opt/ml/input/data/config/args.yaml\"\n", - " },\n", - " metric_definitions=metric_definitions,\n", - " debugger_hook_config=False,\n", - " environment={\"MLFLOW_RUN_ID\": training_run_id}\n", - " )\n", - " \n", - " # Define a data input dictionary with our uploaded S3 URIs\n", - " data = {\n", - " 'train': train_dataset_s3_path,\n", - " 'test': test_dataset_s3_path,\n", - " 'config': train_config_s3_path\n", - " }\n", - " \n", - " print(f\"Data for Training Run: {data}\")\n", - " \n", - " # Log training job information\n", - " mlflow.log_param(\"job_name\", job_name)\n", - " mlflow.log_param(\"instance_type\", \"ml.p3.2xlarge\")\n", - " \n", - " # Start the training job\n", - " pytorch_estimator.fit(data, wait=True)\n", - " \n", - " # Get information about the completed training job\n", - " latest_run_job_name = pytorch_estimator.latest_training_job.job_name\n", - " print(f\"Latest Job Name: {latest_run_job_name}\")\n", - " \n", - " sagemaker_client = boto3.client('sagemaker')\n", - " \n", - " # Describe the training job\n", - " response = sagemaker_client.describe_training_job(TrainingJobName=latest_run_job_name)\n", - " \n", - " # Extract the model artifacts S3 path\n", - " model_artifacts_s3_path = response['ModelArtifacts']['S3ModelArtifacts']\n", - " \n", - " # Extract the output path (this is the general output location)\n", - " output_path = response['OutputDataConfig']['S3OutputPath']\n", - " \n", - " # Get training time metrics\n", - " training_start_time = response.get('TrainingStartTime')\n", - " training_end_time = response.get('TrainingEndTime')\n", - " billable_time = response.get('BillableTimeInSeconds', 0)\n", - " \n", - " # Calculate duration\n", - " total_training_time = 0\n", - " if training_start_time and training_end_time:\n", - " total_training_time = (training_end_time - training_start_time).total_seconds()\n", - " \n", - " # Log job results and metrics to MLflow\n", - " # Log basic job info\n", - " mlflow.log_param(\"training_job_name\", latest_run_job_name)\n", - " mlflow.log_param(\"model_artifacts_path\", model_artifacts_s3_path)\n", - " mlflow.log_param(\"output_path\", output_path)\n", - " \n", - " # Log performance metrics\n", - " mlflow.log_metric(\"billable_time_seconds\", billable_time)\n", - " mlflow.log_metric(\"total_training_time_seconds\", total_training_time)\n", - " \n", - " # Log training job status\n", - " mlflow.log_param(\"training_job_status\", response.get('TrainingJobStatus'))\n", - " \n", - " # Log any secondary status\n", - " if 'SecondaryStatus' in response:\n", - " mlflow.log_param(\"secondary_status\", response.get('SecondaryStatus'))\n", - " \n", - " # Log any failure reason\n", - " if 'FailureReason' in response:\n", - " mlflow.log_param(\"failure_reason\", response.get('FailureReason'))\n", - " \n", - " # Get CloudWatch logs for the training job\n", - " logs_client = boto3.client('logs')\n", - " log_group = \"/aws/sagemaker/TrainingJobs\"\n", - " log_stream = latest_run_job_name\n", - " \n", - " try:\n", - " # Get the last 1000 log events\n", - " log_events = logs_client.get_log_events(\n", - " logGroupName=log_group,\n", - " logStreamName=log_stream,\n", - " limit=1000\n", - " )\n", - " \n", - " # Extract and save logs\n", - " log_output = \"\\n\".join([event['message'] for event in log_events['events']])\n", - " \n", - " # Save logs to file and log as artifact\n", - " with tempfile.NamedTemporaryFile(delete=False, mode='w', suffix='.txt') as tmp:\n", - " tmp.write(log_output)\n", - " log_file_path = tmp.name\n", - " \n", - " mlflow.log_artifact(log_file_path, \"training_logs\")\n", - " os.remove(log_file_path)\n", - " \n", - " except Exception as e:\n", - " print(f\"Error fetching training logs: {e}\")\n", - " \n", - " # Log total execution time of this step\n", - " step_duration = time.time() - start_time\n", - " mlflow.log_metric(\"step_execution_time_seconds\", step_duration)\n", - " \n", - " # Log model metadata\n", - " mlflow.set_tag(\"model_path\", model_artifacts_s3_path)\n", - " mlflow.set_tag(\"training_completed_at\", datetime.datetime.now().isoformat())\n", - " \n", - " print(f\"Model artifacts S3 path: {model_artifacts_s3_path}\")\n", - "\n", - " except Exception as e:\n", - " error_msg = f\"Error in model fine-tuning: {str(e)}\\n{traceback.format_exc()}\"\n", - " print(error_msg)\n", - " \n", - " raise RuntimeError(f\"Fine-tuning failed: {str(e)}\")\n", - "\n", - " return run_id, training_run_id, model_artifacts_s3_path, output_path" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Deploy Step\n", - "This step deploys the model for evaluation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@step(\n", - " name=\"ModelDeploy\",\n", - " instance_type=instance_type,\n", - " display_name=\"Model Deploy\",\n", - " keep_alive_period_in_seconds=900\n", - ")\n", - "def deploy(\n", - " model_artifacts_s3_path: str,\n", - " output_path: str,\n", - " model_id: str,\n", - "):\n", - " import sagemaker\n", - " import boto3\n", - " from sagemaker import get_execution_role\n", - " from sagemaker import Model\n", - " import time\n", - " \n", - " sagemaker_session = sagemaker.Session()\n", - " instance_count = 1\n", - " instance_type = \"ml.g5.2xlarge\"\n", - " health_check_timeout = 700\n", - " \n", - " # Get the name for the endpoint\n", - " endpoint_name = f\"{model_id.split('/')[-1].replace('.', '-').replace('_','-')}-sft-djl\"\n", - " \n", - " # Delete existing endpoint if it exists\n", - " print(f\"Checking for existing endpoint: {endpoint_name}\")\n", - " sm_client = boto3.client('sagemaker')\n", - " try:\n", - " sm_client.describe_endpoint(EndpointName=endpoint_name)\n", - " print(f\"Endpoint {endpoint_name} exists, deleting it before deployment\")\n", - " sm_client.delete_endpoint(EndpointName=endpoint_name)\n", - "\n", - " print(f\"Deleting endpoint config {endpoint_name}\")\n", - " sm_client.delete_endpoint_config(EndpointConfigName=endpoint_name)\n", - " \n", - " # Wait for endpoint to be fully deleted\n", - " print(\"Waiting for endpoint to be fully deleted...\")\n", - " wait_seconds = 10\n", - " total_wait_time = 0\n", - " max_wait_time = 300 # 5 minutes maximum wait\n", - " endpoint_deleted = False\n", - " \n", - " while total_wait_time < max_wait_time and not endpoint_deleted:\n", - " try:\n", - " sm_client.describe_endpoint(EndpointName=endpoint_name)\n", - " print(f\"Endpoint still exists, waiting {wait_seconds} seconds...\")\n", - " time.sleep(wait_seconds)\n", - " total_wait_time += wait_seconds\n", - " except sm_client.exceptions.ClientError:\n", - " print(f\"Endpoint {endpoint_name} successfully deleted\")\n", - " endpoint_deleted = True\n", - " \n", - " if not endpoint_deleted:\n", - " print(f\"Warning: Endpoint still exists after {max_wait_time} seconds\")\n", - " \n", - " except sm_client.exceptions.ClientError:\n", - " print(f\"Endpoint {endpoint_name} does not exist, proceeding with deployment\")\n", - " \n", - " # Continue with model deployment\n", - " image_uri = sagemaker.image_uris.retrieve(\n", - " framework=\"djl-lmi\",\n", - " region=sagemaker_session.boto_session.region_name,\n", - " version=\"latest\"\n", - " )\n", - " \n", - " model_data = model_artifacts_s3_path\n", - " \n", - " # Create model only once\n", - " model = Model(\n", - " image_uri=image_uri,\n", - " model_data=model_data,\n", - " role=get_execution_role(),\n", - " env={\n", - " 'HF_MODEL_ID': \"/opt/ml/model\", # path to where sagemaker stores the model\n", - " 'OPTION_TRUST_REMOTE_CODE': 'true',\n", - " 'OPTION_ROLLING_BATCH': \"vllm\",\n", - " 'OPTION_DTYPE': 'bf16',\n", - " 'OPTION_QUANTIZE': 'fp8',\n", - " 'OPTION_TENSOR_PARALLEL_DEGREE': 'max',\n", - " 'OPTION_MAX_ROLLING_BATCH_SIZE': '32',\n", - " 'OPTION_MODEL_LOADING_TIMEOUT': '3600',\n", - " 'OPTION_MAX_MODEL_LEN': '4096'\n", - " }\n", - " )\n", - "\n", - " print(f\"deploying endpoint: {endpoint_name}\")\n", - " \n", - " predictor = model.deploy(\n", - " endpoint_name=endpoint_name,\n", - " initial_instance_count=instance_count,\n", - " instance_type=instance_type,\n", - " container_startup_health_check_timeout=health_check_timeout,\n", - " model_data_download_timeout=3600\n", - " )\n", - " \n", - " return endpoint_name" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Evaluation Step\n", - "\n", - "After fine-tuning, this step assesses the model's performance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@step(\n", - " name=\"ModelEvaluation\",\n", - " instance_type=instance_type,\n", - " display_name=\"Model Evaluation\",\n", - " keep_alive_period_in_seconds=900,\n", - " dependencies=\"./eval/requirements.txt\"\n", - ")\n", - "def evaluate(\n", - " tracking_server_arn: str,\n", - " experiment_name: str,\n", - " run_id: str,\n", - " endpoint_name: str,\n", - ")-> dict:\n", - " import os\n", - " import json\n", - " import time\n", - " import boto3\n", - " import pandas as pd\n", - " import numpy as np\n", - " import matplotlib.pyplot as plt\n", - " from tqdm.notebook import tqdm\n", - " from datasets import load_dataset\n", - " import mlflow\n", - " import uuid\n", - " import traceback\n", - " from datetime import datetime\n", - " \n", - " # Import LightEval metrics\n", - " from lighteval.metrics.metrics_sample import ROUGE, Doc\n", - " \n", - " # Initialize LightEval metrics calculators\n", - " rouge_metrics = ROUGE(\n", - " methods=[\"rouge1\", \"rouge2\", \"rougeL\"],\n", - " multiple_golds=False,\n", - " bootstrap=False,\n", - " normalize_gold=None,\n", - " normalize_pred=None\n", - " )\n", - " \n", - " # This function allows you to interact with a deployed SageMaker endpoint to get predictions from the DeepSeek model\n", - " def invoke_sagemaker_endpoint(payload, endpoint_name):\n", - " \"\"\"\n", - " Invoke a SageMaker endpoint with the given payload.\n", - " \n", - " Args:\n", - " payload (dict): The input data to send to the endpoint\n", - " endpoint_name (str): The name of the SageMaker endpoint\n", - " \n", - " Returns:\n", - " dict: The response from the endpoint\n", - " \"\"\"\n", - " try:\n", - " start_time = time.time()\n", - " response = sm_client.invoke_endpoint(\n", - " EndpointName=endpoint_name,\n", - " ContentType='application/json',\n", - " Body=json.dumps(payload)\n", - " )\n", - " inference_time = time.time() - start_time\n", - " \n", - " response_body = response['Body'].read().decode('utf-8')\n", - " return json.loads(response_body), inference_time\n", - " except Exception as e:\n", - " print(f\"Error invoking endpoint {endpoint_name}: {str(e)}\")\n", - " return None, -1\n", - " \n", - " def calculate_metrics(predictions, references):\n", - " \"\"\"\n", - " Calculate all evaluation metrics for summarization using LightEval.\n", - " \n", - " Args:\n", - " predictions (list): List of generated summaries\n", - " references (list): List of reference summaries\n", - " \n", - " Returns:\n", - " dict: Dictionary containing all metric scores\n", - " \"\"\"\n", - " metrics = {}\n", - " \n", - " # Create Doc objects for the Rouge and BertScore metrics\n", - " docs = []\n", - " for reference in references:\n", - " docs.append(Doc(\n", - " {\"target\": reference},\n", - " choices=[reference], # Dummy choices\n", - " gold_index=0 # Dummy gold_index\n", - " ))\n", - " \n", - " # Calculate ROUGE scores for each prediction-reference pair\n", - " rouge_scores = {\n", - " 'rouge1_f': [], \n", - " 'rouge2_f': [], \n", - " 'rougeL_f': [],\n", - " # Add precision and recall scores too\n", - " 'rouge1_precision': [],\n", - " 'rouge1_recall': [],\n", - " 'rouge2_precision': [],\n", - " 'rouge2_recall': [],\n", - " 'rougeL_precision': [],\n", - " 'rougeL_recall': []\n", - " }\n", - " \n", - " for pred, ref in zip(predictions, references):\n", - " # For ROUGE calculation\n", - " rouge_result = rouge_metrics.compute(golds=[ref], predictions=[pred])\n", - " rouge_scores['rouge1_f'].append(rouge_result['rouge1'])\n", - " rouge_scores['rouge2_f'].append(rouge_result['rouge2'])\n", - " rouge_scores['rougeL_f'].append(rouge_result['rougeL'])\n", - " \n", - " # For more detailed ROUGE metrics (we get precision and recall too)\n", - " detailed_rouge = rouge_metrics.compute_detailed(golds=[ref], predictions=[pred])\n", - " rouge_scores['rouge1_precision'].append(detailed_rouge[0]['rouge1_precision'])\n", - " rouge_scores['rouge1_recall'].append(detailed_rouge[0]['rouge1_recall'])\n", - " rouge_scores['rouge2_precision'].append(detailed_rouge[0]['rouge2_precision'])\n", - " rouge_scores['rouge2_recall'].append(detailed_rouge[0]['rouge2_recall'])\n", - " rouge_scores['rougeL_precision'].append(detailed_rouge[0]['rougeL_precision'])\n", - " rouge_scores['rougeL_recall'].append(detailed_rouge[0]['rougeL_recall'])\n", - " \n", - " # Average ROUGE scores\n", - " for key in rouge_scores:\n", - " metrics[key] = sum(rouge_scores[key]) / len(rouge_scores[key])\n", - " \n", - " # Calculate prediction statistics\n", - " metrics['avg_prediction_length'] = np.mean([len(pred.split()) for pred in predictions])\n", - " metrics['min_prediction_length'] = min([len(pred.split()) for pred in predictions])\n", - " metrics['max_prediction_length'] = max([len(pred.split()) for pred in predictions])\n", - " \n", - " # Calculate reference statistics\n", - " metrics['avg_reference_length'] = np.mean([len(ref.split()) for ref in references])\n", - " metrics['min_reference_length'] = min([len(ref.split()) for ref in references])\n", - " metrics['max_reference_length'] = max([len(ref.split()) for ref in references])\n", - " \n", - " # Calculate length ratio\n", - " metrics['avg_length_ratio'] = np.mean([len(pred.split()) / len(ref.split()) if len(ref.split()) > 0 else 0 \n", - " for pred, ref in zip(predictions, references)])\n", - " \n", - " print(f\"Metrics: {metrics}\")\n", - " \n", - " return metrics\n", - " \n", - " def generate_summaries_with_model(endpoint_name, dataset):\n", - " \"\"\"\n", - " Generate summaries using a model deployed on SageMaker.\n", - " \n", - " Args:\n", - " endpoint_name (str): SageMaker endpoint name\n", - " dataset: Dataset containing dialogues\n", - " \n", - " Returns:\n", - " list: Generated summaries\n", - " list: Inference times for each summary\n", - " \"\"\"\n", - " predictions = []\n", - " inference_times = []\n", - " failed_generations = 0\n", - " \n", - " for example in tqdm(dataset, desc=\"Generating Responses\"):\n", - " question = example[\"Question\"]\n", - " \n", - " # Prepare the prompt for the model\n", - " prompt = f\"\"\"\n", - " <|begin_of_text|>\n", - " <|start_header_id|>system<|end_header_id|>\n", - " You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", - " Below is an instruction that describes a task, paired with an input that provides further context. \n", - " Write a response that appropriately completes the request.\n", - " Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", - " <|eot_id|><|start_header_id|>user<|end_header_id|>\n", - " {question}<|eot_id|>\n", - " <|start_header_id|>assistant<|end_header_id|>\"\"\"\n", - " \n", - " # Payload for SageMaker endpoint\n", - " payload = {\n", - " \"inputs\": prompt,\n", - " \"parameters\": {\n", - " \"max_new_tokens\": 512,\n", - " \"top_p\": 0.9,\n", - " \"temperature\": 0.6,\n", - " \"return_full_text\": False\n", - " }\n", - " }\n", - " \n", - " # Call the model endpoint\n", - " try:\n", - " response, inference_time = invoke_sagemaker_endpoint(payload, endpoint_name)\n", - " \n", - " # Extract the generated text\n", - " if response is None:\n", - " prediction = \"Error generating response.\"\n", - " failed_generations += 1\n", - " elif isinstance(response, list):\n", - " prediction = response[0].get('generated_text', '').strip()\n", - " elif isinstance(response, dict):\n", - " prediction = response.get('generated_text', '').strip()\n", - " else:\n", - " prediction = str(response).strip()\n", - " \n", - " prediction = prediction.split(\"<|eot_id|>\")[0] if \"<|eot_id|>\" in prediction else prediction\n", - " \n", - " # Log individual inference metrics\n", - " mlflow.log_metric(f\"inference_time_sample_{len(predictions)}\", inference_time)\n", - " \n", - " inference_times.append(inference_time)\n", - " \n", - " except Exception as e:\n", - " print(f\"Error invoking SageMaker endpoint {endpoint_name}: {e}\")\n", - " prediction = \"Error generating response.\"\n", - " failed_generations += 1\n", - " inference_times.append(-1)\n", - " \n", - " predictions.append(prediction)\n", - " \n", - " # Log failure rate\n", - " mlflow.log_metric(\"failed_generations\", failed_generations)\n", - " mlflow.log_metric(\"failure_rate\", failed_generations / len(dataset) if len(dataset) > 0 else 0)\n", - " \n", - " return predictions, inference_times\n", - " \n", - " def evaluate_model_on_dataset(model_config, dataset):\n", - " \"\"\"\n", - " Evaluate a fine-tuned model on a dataset using both automated and human metrics.\n", - " \n", - " Args:\n", - " model_config (dict): Model configuration with name and endpoint\n", - " dataset: dataset for evaluation\n", - " \n", - " Returns:\n", - " dict: Evaluation results\n", - " \"\"\"\n", - " model_name = model_config[\"name\"]\n", - " endpoint_name = model_config[\"endpoint\"]\n", - " \n", - " print(f\"\\nEvaluating model: {model_name} on endpoint: {endpoint_name}\")\n", - " \n", - " # Get references\n", - " references = [\"\\n\".join([example[\"Complex_CoT\"], example[\"Response\"]]) for example in dataset]\n", - " \n", - " # Generate summaries\n", - " print(\"\\nGenerating Responses...\")\n", - " predictions, inference_times = generate_summaries_with_model(endpoint_name, dataset)\n", - " \n", - " # Log inference time metrics\n", - " valid_times = [t for t in inference_times if t > 0]\n", - " if valid_times:\n", - " mlflow.log_metric(\"avg_inference_time\", np.mean(valid_times))\n", - " mlflow.log_metric(\"min_inference_time\", min(valid_times))\n", - " mlflow.log_metric(\"max_inference_time\", max(valid_times))\n", - " mlflow.log_metric(\"p95_inference_time\", np.percentile(valid_times, 95))\n", - " \n", - " # Calculate automated metrics using LightEval\n", - " print(\"\\nCalculating evaluation metrics with LightEval...\")\n", - " metrics = calculate_metrics(predictions, references)\n", - " \n", - " # Log all calculated metrics to MLflow\n", - " for metric_name, metric_value in metrics.items():\n", - " mlflow.log_metric(metric_name, metric_value)\n", - " \n", - " # Create a comparison table of predictions vs references\n", - " comparison_data = []\n", - " for i, (pred, ref) in enumerate(zip(predictions[:5], references[:5])):\n", - " comparison_data.append({\n", - " \"example_id\": i,\n", - " \"prediction\": pred[:500] + (\"...\" if len(pred) > 500 else \"\"), # Truncate for readability\n", - " \"reference\": ref[:500] + (\"...\" if len(ref) > 500 else \"\"), # Truncate for readability\n", - " \"rouge1_f\": rouge_metrics.compute(golds=[ref], predictions=[pred])['rouge1']\n", - " })\n", - " \n", - " comparison_df = pd.DataFrame(comparison_data)\n", - " # Save comparison to a temporary CSV and log it as an artifact\n", - " temp_csv = f\"/tmp/predictions_comparison_{uuid.uuid4().hex[:8]}.csv\"\n", - " comparison_df.to_csv(temp_csv, index=False)\n", - " mlflow.log_artifact(temp_csv, \"model_predictions\")\n", - " \n", - " # Format results\n", - " results = {\n", - " \"model_name\": model_name,\n", - " \"endpoint_name\": endpoint_name,\n", - " \"num_samples\": len(dataset),\n", - " \"metrics\": metrics,\n", - " \"predictions\": predictions[:5], # First 5 predictions\n", - " \"references\": references[:5] # First 5 references\n", - " }\n", - " \n", - " # Print key results\n", - " print(f\"\\nResults for {model_name}:\")\n", - " print(f\"ROUGE-1 F1: {metrics['rouge1_f']:.4f}\")\n", - " print(f\"ROUGE-2 F1: {metrics['rouge2_f']:.4f}\")\n", - " print(f\"ROUGE-L F1: {metrics['rougeL_f']:.4f}\")\n", - " print(f\"Average Inference Time: {np.mean([t for t in inference_times if t > 0]):.3f} seconds\")\n", - " \n", - " return results, metrics['rouge1_f'], metrics['rouge2_f'], metrics['rougeL_f']\n", - " \n", - " mlflow.set_tracking_uri(tracking_server_arn)\n", - " mlflow.set_experiment(experiment_name)\n", - " \n", - " with mlflow.start_run(run_id=run_id):\n", - " with mlflow.start_run(run_name=\"ModelEvaluation\", nested=True):\n", - " mlflow.autolog()\n", - " \n", - " # Initialize the SageMaker client\n", - " sm_client = boto3.client('sagemaker-runtime')\n", - " \n", - " FINETUNED_MODEL_ENDPOINT = endpoint_name # Update with Fine-tuned model endpoint name\n", - " \n", - " # Define the model to evaluate\n", - " model_to_evaluate = {\n", - " \"name\": \"Fine-tuned DeepSeek-R1-Distill-Llama-8B\", \n", - " \"endpoint\": FINETUNED_MODEL_ENDPOINT\n", - " }\n", - " # Limit the number of samples to evaluate (for faster execution)\n", - " num_samples = 10\n", - " \n", - " # Log evaluation parameters to MLflow\n", - " mlflow.log_param(\"evaluation_endpoint\", FINETUNED_MODEL_ENDPOINT)\n", - " mlflow.log_param(\"evaluation_num_samples\", num_samples)\n", - " mlflow.log_param(\"evaluation_timestamp\", datetime.now().isoformat())\n", - " \n", - " # Load the test split of the medical-o1 dataset\n", - " try:\n", - " dataset = load_dataset(\"FreedomIntelligence/medical-o1-reasoning-SFT\", \"en\", split=\"train\")\n", - " \n", - " max_samples = len(dataset)\n", - " \n", - " dataset = dataset.shuffle().select(range(min(num_samples, max_samples)))\n", - " print(f\"Loaded medical-o1-reasoning dataset with {len(dataset)} samples out of {max_samples}\")\n", - " \n", - " mlflow.log_param(\"dataset_name\", \"FreedomIntelligence/medical-o1-reasoning-SFT\")\n", - " mlflow.log_param(\"dataset_actual_samples\", len(dataset))\n", - " except Exception as e:\n", - " error_msg = f\"Error loading dataset: {str(e)}\"\n", - " print(error_msg)\n", - " raise\n", - " \n", - " # Display a sample from the dataset\n", - " sample = dataset[0]\n", - " \n", - " print(\"\\nQuestion:\\n\", sample[\"Question\"], \"\\n\\n====\\n\")\n", - " print(\"Complex_CoT:\\n\", sample[\"Complex_CoT\"], \"\\n\\n====\\n\")\n", - " print(\"Response:\\n\", sample[\"Response\"], \"\\n\\n====\\n\")\n", - "\n", - " try:\n", - " finetuned_model_results, rouge1_f, rouge2_f, rougeL_f = evaluate_model_on_dataset(model_to_evaluate, dataset)\n", - " print(f\"ROUGE-1 F1: {rouge1_f}\")\n", - " print(f\"ROUGE-2 F1: {rouge2_f}\")\n", - " print(f\"ROUGE-L F1: {rougeL_f}\")\n", - " \n", - " # Create and log visualizations if MLflow is enabled\n", - " # Log model card with performance summary\n", - " model_card = f\"\"\"\n", - " # Model Evaluation Report\n", - " \n", - " ## Model Information\n", - " - **Model Name**: {model_to_evaluate[\"name\"]}\n", - " - **Endpoint**: {model_to_evaluate[\"endpoint\"]}\n", - " - **Evaluation Date**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", - " - **Dataset**: FreedomIntelligence/medical-o1-reasoning-SFT\n", - " - **Samples Evaluated**: {len(dataset)}\n", - " \n", - " ## Performance Metrics\n", - " - **ROUGE-1 F1**: {rouge1_f:.4f}\n", - " - **ROUGE-2 F1**: {rouge2_f:.4f}\n", - " - **ROUGE-L F1**: {rougeL_f:.4f}\n", - " - **Average Inference Time**: {np.mean([t for t in finetuned_model_results[0][\"inference_times\"] if t > 0]):.3f} seconds\n", - " \n", - " ## Detailed Metrics\n", - " {json.dumps(finetuned_model_results[0][\"metrics\"], indent=2)}\n", - " \"\"\"\n", - "\n", - " with open(\"/tmp/model_card.md\", \"w\") as f:\n", - " f.write(model_card)\n", - " \n", - " mlflow.log_artifact(\"/tmp/model_card.md\", \"evaluation_summary\")\n", - " \n", - " # Create a simple bar chart for ROUGE metrics\n", - " plt.figure(figsize=(10, 6))\n", - " metrics = finetuned_model_results[0][\"metrics\"]\n", - " rouge_metrics = {\n", - " 'ROUGE-1 F1': metrics['rouge1_f'], \n", - " 'ROUGE-2 F1': metrics['rouge2_f'], \n", - " 'ROUGE-L F1': metrics['rougeL_f']\n", - " }\n", - " plt.bar(rouge_metrics.keys(), rouge_metrics.values())\n", - " plt.title('ROUGE Metrics')\n", - " plt.ylabel('Score')\n", - " plt.ylim(0, 1)\n", - " plt.grid(axis='y', linestyle='--', alpha=0.7)\n", - " plt.savefig('/tmp/rouge_metrics.png')\n", - " mlflow.log_artifact('/tmp/rouge_metrics.png', \"evaluation_plots\")\n", - " \n", - " except Exception as e:\n", - " error_msg = f\"Error in model evaluation: {str(e)}\\n{traceback.format_exc()}\"\n", - " print(error_msg)\n", - " \n", - " # Return at least something even if evaluation fails\n", - " return {\"error\": str(e), \"rougeL_f\": 0.0}\n", - "\n", - " return {\"rougeL_f\": rougeL_f}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 7. Pipeline Creation and Execution\n", + "### 6. Pipeline Creation and Execution\n", "\n", "This final section brings all the components together into an executable pipeline.\n", "\n", @@ -1484,39 +667,112 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 9, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:15.462330Z", + "iopub.status.busy": "2025-09-15T19:40:15.462113Z", + "iopub.status.idle": "2025-09-15T19:40:16.947294Z", + "shell.execute_reply": "2025-09-15T19:40:16.946826Z", + "shell.execute_reply.started": "2025-09-15T19:40:15.462314Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:datasets:PyTorch version 2.6.0 available.\n", + "INFO:datasets:TensorFlow version 2.18.0 available.\n" + ] + } + ], "source": [ - "# Defining the steps of the pipeline\n", - "preprocessing_step = preprocess(\n", + "from steps import (\n", + " preprocess_step,\n", + " finetune_step,\n", + " deploy_step,\n", + " quantitative_eval_step,\n", + " qualitative_eval_step,\n", + " model_registration_step\n", + ")\n", + "from sagemaker.workflow.step_collections import StepCollection\n", + "\n", + "preprocessing_step = preprocess_step.preprocess(\n", " tracking_server_arn=mlflow_tracking_server_arn,\n", " experiment_name=pipeline_name,\n", " run_name=ExecutionVariables.PIPELINE_EXECUTION_ID,\n", - " input_path=input_path,\n", + " input_path=input_path\n", ")\n", "\n", - "training_step = train(\n", + "training_step = finetune_step.train(\n", " tracking_server_arn=mlflow_tracking_server_arn,\n", " experiment_name=pipeline_name,\n", " run_id=preprocessing_step[0],\n", " train_dataset_s3_path=preprocessing_step[1],\n", " test_dataset_s3_path=preprocessing_step[2],\n", " train_config_s3_path=train_config_s3_path,\n", + " role=role,\n", " model_id=model_s3_destination,\n", ")\n", + "run_id=training_step[0]\n", + "model_artifacts_s3_path=training_step[2]\n", + "output_path=training_step[3]\n", "\n", - "deploy_step = deploy(\n", - " model_artifacts_s3_path=training_step[2],\n", - " output_path=training_step[3],\n", + "deploy_step = deploy_step.deploy(\n", + " model_artifacts_s3_path=model_artifacts_s3_path,\n", + " output_path=output_path,\n", " model_id=model_s3_destination,\n", ")\n", + "endpoint_name=deploy_step\n", "\n", - "evaluate_step = evaluate(\n", + "quantitative_eval_step = quantitative_eval_step.quantitative_evaluate(\n", + " tracking_server_arn=mlflow_tracking_server_arn,\n", " experiment_name=pipeline_name,\n", + " run_id=run_id,\n", + " endpoint_name=endpoint_name\n", + ")\n", + "\n", + "qualitative_eval_step = qualitative_eval_step.qualitative_evaluate(\n", " tracking_server_arn=mlflow_tracking_server_arn,\n", - " run_id=training_step[0],\n", - " endpoint_name=deploy_step[0],\n", + " experiment_name=pipeline_name,\n", + " run_id=run_id,\n", + " endpoint_name=endpoint_name\n", + ")\n", + "\n", + "evaluation_gate = ConditionStep(\n", + " name=\"EvaluationGate\",\n", + " depends_on=[qualitative_eval_step],\n", + " conditions=[\n", + " ConditionGreaterThanOrEqualTo(\n", + " left=quantitative_eval_step[\"rougeL_f\"],\n", + " right=0.2\n", + " ),\n", + " ConditionGreaterThanOrEqualTo(\n", + " left=qualitative_eval_step[\"avg_medical_accuracy\"],\n", + " right=3.0\n", + " )\n", + " ],\n", + " if_steps=[\n", + " model_registration_step.register_model(\n", + " tracking_server_arn=mlflow_tracking_server_arn,\n", + " experiment_name=pipeline_name,\n", + " run_id=run_id, # Assuming training_step returns run_id as first output\n", + " model_artifacts_s3_path=model_artifacts_s3_path, # Assuming training_step returns artifacts path as second output\n", + " model_id=model_id,\n", + " model_name=f\"Fine-Tuned-Medical-DeepSeek\",\n", + " endpoint_name=endpoint_name,\n", + " evaluation_score=quantitative_eval_step[\"rougeL_f\"], # Get the evaluation score\n", + " pipeline_name=pipeline_name,\n", + " model_description=\"Fine-tuned medical LLM for clinical reasoning and diagnostics\"\n", + " )\n", + " ],\n", + " else_steps=[\n", + " FailStep(\n", + " name=\"EvaluationFailed\",\n", + " error_message=\"Model evaluation failed to meet quality thresholds.\"\n", + " )\n", + " ]\n", ")\n", "\n", "# Combining the steps into the pipeline definition\n", @@ -1525,7 +781,13 @@ " parameters=[\n", " instance_type,\n", " ],\n", - " steps=[preprocessing_step, training_step, deploy_step, evaluate_step],\n", + " steps=[\n", + " preprocessing_step,\n", + " training_step,\n", + " deploy_step,\n", + " quantitative_eval_step,\n", + " evaluation_gate\n", + " ],\n", ")" ] }, @@ -1540,9 +802,190 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 10, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:17.597437Z", + "iopub.status.busy": "2025-09-15T19:40:17.597048Z", + "iopub.status.idle": "2025-09-15T19:40:29.608656Z", + "shell.execute_reply": "2025-09-15T19:40:29.608049Z", + "shell.execute_reply.started": "2025-09-15T19:40:17.597418Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.Dependencies\n", + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.IncludeLocalWorkDir\n", + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.CustomFileFilter.IgnoreNamePatterns\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-09-15 19:40:19,366 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-17-829/function\n", + "2025-09-15 19:40:19,431 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-17-829/arguments\n", + "2025-09-15 19:40:19,620 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpfb7vb7bb/requirements.txt'\n", + "2025-09-15 19:40:19,655 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "2025-09-15 19:40:19,663 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmprcsjitzu/temp_workspace/sagemaker_remote_function_workspace'\n", + "2025-09-15 19:40:19,683 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmprcsjitzu/workspace.zip'\n", + "2025-09-15 19:40:19,721 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-15-19-40-17-829/workspace.zip'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.IncludeLocalWorkDir\n", + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.CustomFileFilter.IgnoreNamePatterns\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-09-15 19:40:21,013 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-17-829/function\n", + "2025-09-15 19:40:21,107 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-17-829/arguments\n", + "2025-09-15 19:40:21,179 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmppb5hw8pw/requirements.txt'\n", + "2025-09-15 19:40:21,207 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.Dependencies\n", + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.IncludeLocalWorkDir\n", + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.CustomFileFilter.IgnoreNamePatterns\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-09-15 19:40:22,505 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-17-829/function\n", + "2025-09-15 19:40:22,582 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-17-829/arguments\n", + "2025-09-15 19:40:22,671 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmprgxfuk_g/requirements.txt'\n", + "2025-09-15 19:40:22,700 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.IncludeLocalWorkDir\n", + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.CustomFileFilter.IgnoreNamePatterns\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-09-15 19:40:24,015 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-17-829/function\n", + "2025-09-15 19:40:24,143 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-17-829/arguments\n", + "2025-09-15 19:40:24,228 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpmkhl1x38/requirements.txt'\n", + "2025-09-15 19:40:24,253 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.Dependencies\n", + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.IncludeLocalWorkDir\n", + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.CustomFileFilter.IgnoreNamePatterns\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-09-15 19:40:25,566 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-17-829/function\n", + "2025-09-15 19:40:25,628 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-17-829/arguments\n", + "2025-09-15 19:40:25,682 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpjbq8x0jf/requirements.txt'\n", + "2025-09-15 19:40:25,715 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.IncludeLocalWorkDir\n", + "sagemaker.config INFO - Applied value from config key = SageMaker.PythonSDK.Modules.RemoteFunction.CustomFileFilter.IgnoreNamePatterns\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-09-15 19:40:27,000 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-17-829/function\n", + "2025-09-15 19:40:27,068 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-17-829/arguments\n", + "2025-09-15 19:40:27,156 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmprwbp1rt0/requirements.txt'\n", + "2025-09-15 19:40:27,184 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", + "2025-09-15 19:40:27,613 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-27-612/function\n", + "2025-09-15 19:40:27,695 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-27-612/arguments\n", + "2025-09-15 19:40:28,100 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp74qxvfzn/requirements.txt'\n", + "2025-09-15 19:40:28,140 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "2025-09-15 19:40:28,147 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmplvdgxn1c/temp_workspace/sagemaker_remote_function_workspace'\n", + "2025-09-15 19:40:28,167 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmplvdgxn1c/workspace.zip'\n", + "2025-09-15 19:40:28,261 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-15-19-40-27-612/workspace.zip'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", + "2025-09-15 19:40:28,264 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-27-612/function\n", + "2025-09-15 19:40:28,326 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-27-612/arguments\n", + "2025-09-15 19:40:28,435 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpsf8y777x/requirements.txt'\n", + "2025-09-15 19:40:28,492 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", + "2025-09-15 19:40:28,493 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-27-612/function\n", + "2025-09-15 19:40:28,610 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-27-612/arguments\n", + "2025-09-15 19:40:28,675 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp0zi14pe5/requirements.txt'\n", + "2025-09-15 19:40:28,708 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", + "2025-09-15 19:40:28,710 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-27-612/function\n", + "2025-09-15 19:40:28,781 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-27-612/arguments\n", + "2025-09-15 19:40:28,897 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmp5g8dwq8c/requirements.txt'\n", + "2025-09-15 19:40:28,922 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", + "2025-09-15 19:40:28,924 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-27-612/function\n", + "2025-09-15 19:40:29,078 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-27-612/arguments\n", + "2025-09-15 19:40:29,145 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp9c5wx9mq/requirements.txt'\n", + "2025-09-15 19:40:29,175 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", + "2025-09-15 19:40:29,176 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-27-612/function\n", + "2025-09-15 19:40:29,239 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-27-612/arguments\n", + "2025-09-15 19:40:29,302 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpz_c1935q/requirements.txt'\n", + "2025-09-15 19:40:29,326 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:329542461890:pipeline/AIM405-deepseek-finetune-pipeline',\n", + " 'PipelineVersionId': 38,\n", + " 'ResponseMetadata': {'RequestId': '4edfb27f-a58f-4b54-b689-80c5af37636f',\n", + " 'HTTPStatusCode': 200,\n", + " 'HTTPHeaders': {'x-amzn-requestid': '4edfb27f-a58f-4b54-b689-80c5af37636f',\n", + " 'content-type': 'application/x-amz-json-1.1',\n", + " 'content-length': '124',\n", + " 'date': 'Mon, 15 Sep 2025 19:40:29 GMT'},\n", + " 'RetryAttempts': 0}}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pipeline.upsert(role)" ] @@ -1558,13 +1001,671 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 11, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:40:29.609713Z", + "iopub.status.busy": "2025-09-15T19:40:29.609414Z", + "iopub.status.idle": "2025-09-15T19:40:29.847339Z", + "shell.execute_reply": "2025-09-15T19:40:29.846879Z", + "shell.execute_reply.started": "2025-09-15T19:40:29.609690Z" + } + }, "outputs": [], "source": [ "execution = pipeline.start()" ] }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:38:10.407187Z", + "iopub.status.busy": "2025-09-15T19:38:10.406934Z", + "iopub.status.idle": "2025-09-15T19:38:11.082172Z", + "shell.execute_reply": "2025-09-15T19:38:11.081383Z", + "shell.execute_reply.started": "2025-09-15T19:38:10.407167Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Registering model: Fine-Tuned-Medical-DeepSeek\n", + "🏃 View run pewfpvcn2tde at: https://us-east-1.experiments.sagemaker.aws/#/experiments/1/runs/ac48b182ac8746178c9dc26c42176a2c\n", + "🧪 View experiment at: https://us-east-1.experiments.sagemaker.aws/#/experiments/1\n" + ] + }, + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ /opt/conda/lib/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:401 in       │\n",
+       "│ log_param                                                                                        │\n",
+       "│                                                                                                  │\n",
+       "│   398 │   │   param = Param(key, str(value))                                                     │\n",
+       "│   399 │   │   try:                                                                               │\n",
+       "│   400 │   │   │   if synchronous:                                                                │\n",
+       "│ ❱ 401 │   │   │   │   self.store.log_param(run_id, param)                                        │\n",
+       "│   402 │   │   │   │   return value                                                               │\n",
+       "│   403 │   │   │   else:                                                                          │\n",
+       "│   404 │   │   │   │   return self.store.log_param_async(run_id, param)                           │\n",
+       "│                                                                                                  │\n",
+       "│ /opt/conda/lib/python3.12/site-packages/mlflow/store/tracking/rest_store.py:644 in log_param     │\n",
+       "│                                                                                                  │\n",
+       "│    641 │   │   req_body = message_to_json(                                                       │\n",
+       "│    642 │   │   │   LogParam(run_uuid=run_id, run_id=run_id, key=param.key, value=param.value)    │\n",
+       "│    643 │   │   )                                                                                 │\n",
+       "│ ❱  644 │   │   self._call_endpoint(LogParam, req_body)                                           │\n",
+       "│    645 │                                                                                         │\n",
+       "│    646 │   def set_experiment_tag(self, experiment_id, tag):                                     │\n",
+       "│    647 │   │   \"\"\"                                                                               │\n",
+       "│                                                                                                  │\n",
+       "│ /opt/conda/lib/python3.12/site-packages/mlflow/store/tracking/rest_store.py:134 in               │\n",
+       "│ _call_endpoint                                                                                   │\n",
+       "│                                                                                                  │\n",
+       "│    131 │   │   else:                                                                             │\n",
+       "│    132 │   │   │   endpoint, method = _METHOD_TO_INFO[api]                                       │\n",
+       "│    133 │   │   response_proto = api.Response()                                                   │\n",
+       "│ ❱  134 │   │   return call_endpoint(                                                             │\n",
+       "│    135 │   │   │   self.get_host_creds(),                                                        │\n",
+       "│    136 │   │   │   endpoint,                                                                     │\n",
+       "│    137 │   │   │   method,                                                                       │\n",
+       "│                                                                                                  │\n",
+       "│ /opt/conda/lib/python3.12/site-packages/mlflow/utils/rest_utils.py:554 in call_endpoint          │\n",
+       "│                                                                                                  │\n",
+       "│   551 │   │   call_kwargs[\"json\"] = json_body                                                    │\n",
+       "│   552 │   │   response = http_request(**call_kwargs)                                             │\n",
+       "│   553 │                                                                                          │\n",
+       "│ ❱ 554 │   response = verify_rest_response(response, endpoint)                                    │\n",
+       "│   555 │   response_to_parse = response.text                                                      │\n",
+       "│   556 │   try:                                                                                   │\n",
+       "│   557 │   │   js_dict = json.loads(response_to_parse)                                            │\n",
+       "│                                                                                                  │\n",
+       "│ /opt/conda/lib/python3.12/site-packages/mlflow/utils/rest_utils.py:308 in verify_rest_response   │\n",
+       "│                                                                                                  │\n",
+       "│   305 │   # Handle non-200 status codes                                                          │\n",
+       "│   306 │   if response.status_code != 200:                                                        │\n",
+       "│   307 │   │   if _can_parse_as_json_object(response.text):                                       │\n",
+       "│ ❱ 308 │   │   │   raise RestException(json.loads(response.text))                                 │\n",
+       "│   309 │   │   else:                                                                              │\n",
+       "│   310 │   │   │   base_msg = (                                                                   │\n",
+       "│   311 │   │   │   │   f\"API request to endpoint {endpoint} \"                                     │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "RestException: INVALID_PARAMETER_VALUE: Changing param values is not allowed. Param with \n",
+       "key='registered_model_name' was already logged with \n",
+       "value='Fine-Tuned-Medical-DeepSeek-deepseek-ai/DeepSeek-R1-Distill-Llama-8B' for run \n",
+       "ID='ac48b182ac8746178c9dc26c42176a2c'. Attempted logging new value 'Fine-Tuned-Medical-DeepSeek'.\n",
+       "\n",
+       "During handling of the above exception, another exception occurred:\n",
+       "\n",
+       "╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱  1 register_model(                                                                             │\n",
+       "│    2 │   tracking_server_arn=\"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server    │\n",
+       "│    3 │   experiment_name=\"AIM405-deepseek-finetune-pipeline\",                                    │\n",
+       "│    4 │   run_id=\"ac48b182ac8746178c9dc26c42176a2c\",  # Assuming training_step returns run_id     │\n",
+       "│                                                                                                  │\n",
+       "│ in register_model:43                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│    40 │   │   }                                                                                  │\n",
+       "│    41 │   │                                                                                      │\n",
+       "│    42 │   │   # Log model info as parameters                                                     │\n",
+       "│ ❱  43 │   │   mlflow.log_param(\"registered_model_name\", model_name)                              │\n",
+       "│    44 │   │   mlflow.log_param(\"model_artifacts_path\", model_artifacts_s3_path)                  │\n",
+       "│    45 │   │   mlflow.log_param(\"evaluation_score\", evaluation_score)                             │\n",
+       "│    46 │   │   mlflow.log_param(\"endpoint_name\", endpoint_name)                                   │\n",
+       "│                                                                                                  │\n",
+       "│ /opt/conda/lib/python3.12/site-packages/mlflow/tracking/fluent.py:775 in log_param               │\n",
+       "│                                                                                                  │\n",
+       "│    772 │   \"\"\"                                                                                   │\n",
+       "│    773 │   run_id = _get_or_start_run().info.run_id                                              │\n",
+       "│    774 │   synchronous = synchronous if synchronous is not None else not MLFLOW_ENABLE_ASYNC_LO  │\n",
+       "│ ❱  775 │   return MlflowClient().log_param(run_id, key, value, synchronous=synchronous)          │\n",
+       "│    776                                                                                           │\n",
+       "│    777                                                                                           │\n",
+       "│    778 def flush_async_logging() -> None:                                                        │\n",
+       "│                                                                                                  │\n",
+       "│ /opt/conda/lib/python3.12/site-packages/mlflow/tracking/client.py:2098 in log_param              │\n",
+       "│                                                                                                  │\n",
+       "│   2095 │   │   │   synchronous if synchronous is not None else not MLFLOW_ENABLE_ASYNC_LOGGING.  │\n",
+       "│   2096 │   │   )                                                                                 │\n",
+       "│   2097 │   │   if synchronous:                                                                   │\n",
+       "│ ❱ 2098 │   │   │   self._tracking_client.log_param(run_id, key, value, synchronous=True)         │\n",
+       "│   2099 │   │   │   return value                                                                  │\n",
+       "│   2100 │   │   else:                                                                             │\n",
+       "│   2101 │   │   │   return self._tracking_client.log_param(run_id, key, value, synchronous=False  │\n",
+       "│                                                                                                  │\n",
+       "│ /opt/conda/lib/python3.12/site-packages/mlflow/telemetry/track.py:29 in wrapper                  │\n",
+       "│                                                                                                  │\n",
+       "│    26 │   │   │   result = None                                                                  │\n",
+       "│    27 │   │   │   start_time = time.time()                                                       │\n",
+       "│    28 │   │   │   try:                                                                           │\n",
+       "│ ❱  29 │   │   │   │   result = func(*args, **kwargs)                                             │\n",
+       "│    30 │   │   │   │   return result  # noqa: RET504                                              │\n",
+       "│    31 │   │   │   except Exception:                                                              │\n",
+       "│    32 │   │   │   │   success = False                                                            │\n",
+       "│                                                                                                  │\n",
+       "│ /opt/conda/lib/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:408 in       │\n",
+       "│ log_param                                                                                        │\n",
+       "│                                                                                                  │\n",
+       "│   405 │   │   except MlflowException as e:                                                       │\n",
+       "│   406 │   │   │   if e.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE):                    │\n",
+       "│   407 │   │   │   │   msg = f\"{e.message}{PARAM_VALIDATION_MSG}\"                                 │\n",
+       "│ ❱ 408 │   │   │   │   raise MlflowException(msg, INVALID_PARAMETER_VALUE)                        │\n",
+       "│   409 │   │   │   else:                                                                          │\n",
+       "│   410 │   │   │   │   raise e                                                                    │\n",
+       "│   411                                                                                            │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "MlflowException: INVALID_PARAMETER_VALUE: Changing param values is not allowed. Param with \n",
+       "key='registered_model_name' was already logged with \n",
+       "value='Fine-Tuned-Medical-DeepSeek-deepseek-ai/DeepSeek-R1-Distill-Llama-8B' for run \n",
+       "ID='ac48b182ac8746178c9dc26c42176a2c'. Attempted logging new value 'Fine-Tuned-Medical-DeepSeek'.\n",
+       "\n",
+       "The cause of this error is typically due to repeated calls\n",
+       "to an individual run_id event logging.\n",
+       "\n",
+       "Incorrect Example:\n",
+       "---------------------------------------\n",
+       "with mlflow.start_run():\n",
+       "    mlflow.log_param(\"depth\", 3)\n",
+       "    mlflow.log_param(\"depth\", 5)\n",
+       "---------------------------------------\n",
+       "\n",
+       "Which will throw an MlflowException for overwriting a\n",
+       "logged parameter.\n",
+       "\n",
+       "Correct Example:\n",
+       "---------------------------------------\n",
+       "with mlflow.start_run():\n",
+       "    with mlflow.start_run(nested=True):\n",
+       "        mlflow.log_param(\"depth\", 3)\n",
+       "    with mlflow.start_run(nested=True):\n",
+       "        mlflow.log_param(\"depth\", 5)\n",
+       "---------------------------------------\n",
+       "\n",
+       "Which will create a new nested run for each individual\n",
+       "model and prevent parameter key collisions within the\n",
+       "tracking store.\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[38;2;255;0;0m╭─\u001b[0m\u001b[38;2;255;0;0m──────────────────────────────\u001b[0m\u001b[38;2;255;0;0m \u001b[0m\u001b[1;38;2;255;0;0mTraceback \u001b[0m\u001b[1;2;38;2;255;0;0m(most recent call last)\u001b[0m\u001b[38;2;255;0;0m \u001b[0m\u001b[38;2;255;0;0m───────────────────────────────\u001b[0m\u001b[38;2;255;0;0m─╮\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/tracking/_tracking_service/\u001b[0m\u001b[1;33mclient.py\u001b[0m:\u001b[94m401\u001b[0m in \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m398 \u001b[0m\u001b[2m│ │ \u001b[0mparam = Param(key, \u001b[96mstr\u001b[0m(value)) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m399 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m400 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m synchronous: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m401 \u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m.store.log_param(run_id, param) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m402 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mreturn\u001b[0m value \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m403 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m404 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m.store.log_param_async(run_id, param) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/store/tracking/\u001b[0m\u001b[1;33mrest_store.py\u001b[0m:\u001b[94m644\u001b[0m in \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 641 \u001b[0m\u001b[2m│ │ \u001b[0mreq_body = message_to_json( \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 642 \u001b[0m\u001b[2m│ │ │ \u001b[0mLogParam(run_uuid=run_id, run_id=run_id, key=param.key, value=param.value) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 643 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 644 \u001b[2m│ │ \u001b[0m\u001b[1;4;96mself\u001b[0m\u001b[1;4m._call_endpoint(LogParam, req_body)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 645 \u001b[0m\u001b[2m│ \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 646 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m\u001b[90m \u001b[0m\u001b[92mset_experiment_tag\u001b[0m(\u001b[96mself\u001b[0m, experiment_id, tag): \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 647 \u001b[0m\u001b[2;90m│ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/store/tracking/\u001b[0m\u001b[1;33mrest_store.py\u001b[0m:\u001b[94m134\u001b[0m in \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[92m_call_endpoint\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 131 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 132 \u001b[0m\u001b[2m│ │ │ \u001b[0mendpoint, method = _METHOD_TO_INFO[api] \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 133 \u001b[0m\u001b[2m│ │ \u001b[0mresponse_proto = api.Response() \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 134 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m call_endpoint( \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 135 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.get_host_creds(), \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 136 \u001b[0m\u001b[2m│ │ │ \u001b[0mendpoint, \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 137 \u001b[0m\u001b[2m│ │ │ \u001b[0mmethod, \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/utils/\u001b[0m\u001b[1;33mrest_utils.py\u001b[0m:\u001b[94m554\u001b[0m in \u001b[92mcall_endpoint\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m551 \u001b[0m\u001b[2m│ │ \u001b[0mcall_kwargs[\u001b[33m\"\u001b[0m\u001b[33mjson\u001b[0m\u001b[33m\"\u001b[0m] = json_body \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m552 \u001b[0m\u001b[2m│ │ \u001b[0mresponse = http_request(**call_kwargs) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m553 \u001b[0m\u001b[2m│ \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m554 \u001b[2m│ \u001b[0mresponse = \u001b[1;4mverify_rest_response(response, endpoint)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m555 \u001b[0m\u001b[2m│ \u001b[0mresponse_to_parse = response.text \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m556 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m557 \u001b[0m\u001b[2m│ │ \u001b[0mjs_dict = json.loads(response_to_parse) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/utils/\u001b[0m\u001b[1;33mrest_utils.py\u001b[0m:\u001b[94m308\u001b[0m in \u001b[92mverify_rest_response\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m305 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Handle non-200 status codes\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m306 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m response.status_code != \u001b[94m200\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m307 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m _can_parse_as_json_object(response.text): \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m308 \u001b[2m│ │ │ \u001b[0m\u001b[1;4;94mraise\u001b[0m\u001b[1;4m RestException(json.loads(response.text))\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m309 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m310 \u001b[0m\u001b[2m│ │ │ \u001b[0mbase_msg = ( \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m311 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mAPI request to endpoint \u001b[0m\u001b[33m{\u001b[0mendpoint\u001b[33m}\u001b[0m\u001b[33m \u001b[0m\u001b[33m\"\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mRestException: \u001b[0mINVALID_PARAMETER_VALUE: Changing param values is not allowed. Param with \n", + "\u001b[38;2;215;175;0mkey\u001b[0m=\u001b[38;2;0;135;0m'registered_model_name'\u001b[0m was already logged with \n", + "\u001b[38;2;215;175;0mvalue\u001b[0m=\u001b[38;2;0;135;0m'Fine-Tuned-Medical-DeepSeek-deepseek-ai/DeepSeek-R1-Distill-Llama-8B'\u001b[0m for run \n", + "\u001b[38;2;215;175;0mID\u001b[0m=\u001b[38;2;0;135;0m'ac48b182ac8746178c9dc26c42176a2c'\u001b[0m. Attempted logging new value \u001b[38;2;0;135;0m'Fine-Tuned-Medical-DeepSeek'\u001b[0m.\n", + "\n", + "\u001b[3mDuring handling of the above exception, another exception occurred:\u001b[0m\n", + "\n", + "\u001b[38;2;255;0;0m╭─\u001b[0m\u001b[38;2;255;0;0m──────────────────────────────\u001b[0m\u001b[38;2;255;0;0m \u001b[0m\u001b[1;38;2;255;0;0mTraceback \u001b[0m\u001b[1;2;38;2;255;0;0m(most recent call last)\u001b[0m\u001b[38;2;255;0;0m \u001b[0m\u001b[38;2;255;0;0m───────────────────────────────\u001b[0m\u001b[38;2;255;0;0m─╮\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 1 register_model( \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 2 \u001b[0m\u001b[2m│ \u001b[0mtracking_server_arn=\u001b[33m\"\u001b[0m\u001b[33marn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 3 \u001b[0m\u001b[2m│ \u001b[0mexperiment_name=\u001b[33m\"\u001b[0m\u001b[33mAIM405-deepseek-finetune-pipeline\u001b[0m\u001b[33m\"\u001b[0m, \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 4 \u001b[0m\u001b[2m│ \u001b[0mrun_id=\u001b[33m\"\u001b[0m\u001b[33mac48b182ac8746178c9dc26c42176a2c\u001b[0m\u001b[33m\"\u001b[0m, \u001b[2m# Assuming training_step returns run_id \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m in \u001b[92mregister_model\u001b[0m:\u001b[94m43\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 40 \u001b[0m\u001b[2m│ │ \u001b[0m} \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 41 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 42 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Log model info as parameters\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 43 \u001b[2m│ │ \u001b[0m\u001b[1;4mmlflow.log_param(\u001b[0m\u001b[1;4;33m\"\u001b[0m\u001b[1;4;33mregistered_model_name\u001b[0m\u001b[1;4;33m\"\u001b[0m\u001b[1;4m, model_name)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 44 \u001b[0m\u001b[2m│ │ \u001b[0mmlflow.log_param(\u001b[33m\"\u001b[0m\u001b[33mmodel_artifacts_path\u001b[0m\u001b[33m\"\u001b[0m, model_artifacts_s3_path) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 45 \u001b[0m\u001b[2m│ │ \u001b[0mmlflow.log_param(\u001b[33m\"\u001b[0m\u001b[33mevaluation_score\u001b[0m\u001b[33m\"\u001b[0m, evaluation_score) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 46 \u001b[0m\u001b[2m│ │ \u001b[0mmlflow.log_param(\u001b[33m\"\u001b[0m\u001b[33mendpoint_name\u001b[0m\u001b[33m\"\u001b[0m, endpoint_name) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/tracking/\u001b[0m\u001b[1;33mfluent.py\u001b[0m:\u001b[94m775\u001b[0m in \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 772 \u001b[0m\u001b[2;33m│ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 773 \u001b[0m\u001b[2m│ \u001b[0mrun_id = _get_or_start_run().info.run_id \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 774 \u001b[0m\u001b[2m│ \u001b[0msynchronous = synchronous \u001b[94mif\u001b[0m synchronous \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m \u001b[95mnot\u001b[0m MLFLOW_ENABLE_ASYNC_LO \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 775 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[1;4mMlflowClient().log_param(run_id, key, value, synchronous=synchronous)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 776 \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 777 \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 778 \u001b[0m\u001b[94mdef\u001b[0m\u001b[90m \u001b[0m\u001b[92mflush_async_logging\u001b[0m() -> \u001b[94mNone\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/tracking/\u001b[0m\u001b[1;33mclient.py\u001b[0m:\u001b[94m2098\u001b[0m in \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2095 \u001b[0m\u001b[2m│ │ │ \u001b[0msynchronous \u001b[94mif\u001b[0m synchronous \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m \u001b[95mnot\u001b[0m MLFLOW_ENABLE_ASYNC_LOGGING. \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2096 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2097 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m synchronous: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m2098 \u001b[2m│ │ │ \u001b[0m\u001b[1;4;96mself\u001b[0m\u001b[1;4m._tracking_client.log_param(run_id, key, value, synchronous=\u001b[0m\u001b[1;4;94mTrue\u001b[0m\u001b[1;4m)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2099 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m value \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2100 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2101 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._tracking_client.log_param(run_id, key, value, synchronous=\u001b[94mFalse\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/telemetry/\u001b[0m\u001b[1;33mtrack.py\u001b[0m:\u001b[94m29\u001b[0m in \u001b[92mwrapper\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 26 \u001b[0m\u001b[2m│ │ │ \u001b[0mresult = \u001b[94mNone\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 27 \u001b[0m\u001b[2m│ │ │ \u001b[0mstart_time = time.time() \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 28 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 29 \u001b[2m│ │ │ │ \u001b[0mresult = func(*args, **kwargs) \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 30 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mreturn\u001b[0m result \u001b[2m# noqa: RET504\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 31 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 32 \u001b[0m\u001b[2m│ │ │ │ \u001b[0msuccess = \u001b[94mFalse\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/tracking/_tracking_service/\u001b[0m\u001b[1;33mclient.py\u001b[0m:\u001b[94m408\u001b[0m in \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m405 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mexcept\u001b[0m MlflowException \u001b[94mas\u001b[0m e: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m406 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m e.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE): \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m407 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mmsg = \u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m{\u001b[0me.message\u001b[33m}\u001b[0m\u001b[33m{\u001b[0mPARAM_VALIDATION_MSG\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m408 \u001b[2m│ │ │ │ \u001b[0m\u001b[1;4;94mraise\u001b[0m\u001b[1;4m MlflowException(msg, INVALID_PARAMETER_VALUE)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m409 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m410 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m411 \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", + "\u001b[38;2;255;0;0m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mMlflowException: \u001b[0mINVALID_PARAMETER_VALUE: Changing param values is not allowed. Param with \n", + "\u001b[38;2;215;175;0mkey\u001b[0m=\u001b[38;2;0;135;0m'registered_model_name'\u001b[0m was already logged with \n", + "\u001b[38;2;215;175;0mvalue\u001b[0m=\u001b[38;2;0;135;0m'Fine-Tuned-Medical-DeepSeek-deepseek-ai/DeepSeek-R1-Distill-Llama-8B'\u001b[0m for run \n", + "\u001b[38;2;215;175;0mID\u001b[0m=\u001b[38;2;0;135;0m'ac48b182ac8746178c9dc26c42176a2c'\u001b[0m. Attempted logging new value \u001b[38;2;0;135;0m'Fine-Tuned-Medical-DeepSeek'\u001b[0m.\n", + "\n", + "The cause of this error is typically due to repeated calls\n", + "to an individual run_id event logging.\n", + "\n", + "Incorrect Example:\n", + "---------------------------------------\n", + "with \u001b[1;38;2;225;0;225mmlflow.start_run\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m:\n", + " \u001b[1;38;2;225;0;225mmlflow.log_param\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;0;135;0m\"depth\"\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m\n", + " \u001b[1;38;2;225;0;225mmlflow.log_param\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;0;135;0m\"depth\"\u001b[0m, \u001b[1;36m5\u001b[0m\u001b[1m)\u001b[0m\n", + "---------------------------------------\n", + "\n", + "Which will throw an MlflowException for overwriting a\n", + "logged parameter.\n", + "\n", + "Correct Example:\n", + "---------------------------------------\n", + "with \u001b[1;38;2;225;0;225mmlflow.start_run\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m:\n", + " with \u001b[1;38;2;225;0;225mmlflow.start_run\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;215;175;0mnested\u001b[0m=\u001b[3;38;2;0;135;0mTrue\u001b[0m\u001b[1m)\u001b[0m:\n", + " \u001b[1;38;2;225;0;225mmlflow.log_param\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;0;135;0m\"depth\"\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m\n", + " with \u001b[1;38;2;225;0;225mmlflow.start_run\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;215;175;0mnested\u001b[0m=\u001b[3;38;2;0;135;0mTrue\u001b[0m\u001b[1m)\u001b[0m:\n", + " \u001b[1;38;2;225;0;225mmlflow.log_param\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;0;135;0m\"depth\"\u001b[0m, \u001b[1;36m5\u001b[0m\u001b[1m)\u001b[0m\n", + "---------------------------------------\n", + "\n", + "Which will create a new nested run for each individual\n", + "model and prevent parameter key collisions within the\n", + "tracking store.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "register_model(\n", + " tracking_server_arn=\"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server/my-tracking-server\",\n", + " experiment_name=\"AIM405-deepseek-finetune-pipeline\",\n", + " run_id=\"ac48b182ac8746178c9dc26c42176a2c\", # Assuming training_step returns run_id as first output\n", + " model_artifacts_s3_path=\"s3://sagemaker-us-east-1-329542461890/deepseek-finetune-2025-09-15-18-39-44-2025-09-15-18-39-44-661/output/model.tar.gz\", # Assuming training_step returns artifacts path as second output\n", + " model_id=\"s3://sagemaker-us-east-1-329542461890/models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B\",\n", + " model_name=f\"Fine-Tuned-Medical-DeepSeek\",\n", + " endpoint_name=\"deepseek-ai-DeepSeek-R1-Distill-Llama-8B-sft-djl\",\n", + " evaluation_score=0.22410036842493342, # Get the evaluation score\n", + " pipeline_name=\"AIM405-deepseek-finetune-pipeline\",\n", + " model_description=\"Fine-tuned medical LLM for clinical reasoning and diagnostics\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-15T19:37:33.572826Z", + "iopub.status.busy": "2025-09-15T19:37:33.572611Z", + "iopub.status.idle": "2025-09-15T19:37:33.584816Z", + "shell.execute_reply": "2025-09-15T19:37:33.584262Z", + "shell.execute_reply.started": "2025-09-15T19:37:33.572810Z" + } + }, + "outputs": [], + "source": [ + "def register_model(\n", + " tracking_server_arn: str,\n", + " experiment_name: str,\n", + " run_id: str,\n", + " model_artifacts_s3_path: str,\n", + " model_id: str,\n", + " model_name: str,\n", + " endpoint_name: str,\n", + " evaluation_score: float,\n", + " pipeline_name: str,\n", + " model_description: str\n", + "):\n", + " import json\n", + " import mlflow\n", + " import boto3\n", + " import os\n", + " import tempfile\n", + " import time\n", + " from datetime import datetime\n", + " \n", + " print(f\"Registering model: {model_name}\")\n", + " \n", + " # Set up MLflow tracking\n", + " mlflow.set_tracking_uri(tracking_server_arn)\n", + " mlflow.set_experiment(experiment_name)\n", + " \n", + " # Connect to MLflow with the specific run\n", + " with mlflow.start_run(run_id=run_id):\n", + " # Create model metadata\n", + " tags = {\n", + " \"model_id\": model_id,\n", + " \"base_model\": model_id.split('/')[-1],\n", + " \"task\": \"medical_qa\",\n", + " \"framework\": \"pytorch\",\n", + " \"endpoint_name\": endpoint_name,\n", + " \"model_artifacts_s3_path\": model_artifacts_s3_path,\n", + " \"deployment_timestamp\": datetime.now().isoformat(),\n", + " \"description\": model_description,\n", + " \"registered_by\": pipeline_name\n", + " }\n", + " \n", + " # Log model info as parameters\n", + " mlflow.log_param(\"registered_model_name\", model_name)\n", + " mlflow.log_param(\"model_artifacts_path\", model_artifacts_s3_path)\n", + " mlflow.log_param(\"evaluation_score\", evaluation_score)\n", + " mlflow.log_param(\"endpoint_name\", endpoint_name)\n", + " # mlflow.log_param(\"registration_timestamp\", datetime.now().isoformat())\n", + " mlflow.log_param(\"registration_timestamp\", \"2025-09-15T19:28:36.049313\")\n", + " \n", + " # Log endpoint information as an artifact\n", + " model_info = {\n", + " \"model_name\": model_name,\n", + " \"model_id\": model_id,\n", + " \"endpoint_name\": endpoint_name,\n", + " \"model_artifacts_s3_path\": model_artifacts_s3_path,\n", + " \"evaluation_score\": float(evaluation_score),\n", + " # \"registration_timestamp\": datetime.now().isoformat()\n", + " \"registration_timestamp\": \"2025-09-15T19:28:36.049313\"\n", + " }\n", + " \n", + " with open(\"/tmp/model_info.json\", \"w\") as f:\n", + " json.dump(model_info, f, indent=2)\n", + " mlflow.log_artifact(\"/tmp/model_info.json\")\n", + " \n", + " # Create model card\n", + " model_card = f\"\"\"\n", + " # {model_name}\n", + " \n", + " ## Model Information\n", + " - **Base Model**: {model_id}\n", + " - **Task**: Medical Question Answering\n", + " - **Evaluation Score**: {evaluation_score:.4f}\n", + " - **Endpoint**: {endpoint_name}\n", + " \n", + " ## Description\n", + " {model_description}\n", + " \n", + " ## Registration Details\n", + " - Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + " - Pipeline: {pipeline_name}\n", + " \"\"\"\n", + " \n", + " with open(\"/tmp/model_card.md\", \"w\") as f:\n", + " f.write(model_card)\n", + " mlflow.log_artifact(\"/tmp/model_card.md\")\n", + " \n", + " # PART 1: REGISTER WITH MLFLOW MODEL REGISTRY\n", + " mlflow_version = None\n", + " try:\n", + " client = mlflow.tracking.MlflowClient()\n", + " \n", + " # Check if model exists and create if it doesn't\n", + " try:\n", + " client.get_registered_model(model_name)\n", + " print(f\"Model {model_name} already exists in MLflow registry\")\n", + " except mlflow.exceptions.MlflowException:\n", + " client.create_registered_model(\n", + " name=model_name,\n", + " description=f\"Fine-tuned medical LLM based on {model_id}\"\n", + " )\n", + " print(f\"Created new registered model: {model_name}\")\n", + " \n", + " # Create empty model directory with artifacts\n", + " with tempfile.TemporaryDirectory() as tmp_dir:\n", + " # Create a minimal model file to log\n", + " os.makedirs(os.path.join(tmp_dir, \"model\"), exist_ok=True)\n", + " \n", + " # Copy model info and card to directory\n", + " with open(os.path.join(tmp_dir, \"model\", \"model_info.json\"), \"w\") as f:\n", + " json.dump(model_info, f, indent=2)\n", + " \n", + " with open(os.path.join(tmp_dir, \"model\", \"model_card.md\"), \"w\") as f:\n", + " f.write(model_card)\n", + " \n", + " # Create a model reference file pointing to the S3 artifacts\n", + " model_ref = {\n", + " \"artifact_path\": model_artifacts_s3_path,\n", + " \"flavors\": {\n", + " \"pytorch\": {\n", + " \"model_data\": model_artifacts_s3_path,\n", + " \"pytorch_version\": \"2.0+\"\n", + " }\n", + " },\n", + " \"run_id\": run_id,\n", + " \"model_class\": \"LLM\",\n", + " \"model_format\": \"PyTorch\"\n", + " }\n", + " \n", + " with open(os.path.join(tmp_dir, \"model\", \"MLmodel\"), \"w\") as f:\n", + " json.dump(model_ref, f, indent=2)\n", + " \n", + " # Log artifacts directory as model\n", + " mlflow.log_artifacts(tmp_dir, artifact_path=\"\")\n", + " \n", + " # Now register the model - try both methods\n", + " try:\n", + " # Method 1: Use direct registration with source as run URI\n", + " model_uri = f\"runs:/{run_id}/model\"\n", + " model_details = mlflow.register_model(\n", + " model_uri=model_uri,\n", + " name=model_name,\n", + " tags=tags\n", + " )\n", + " mlflow_version = model_details.version\n", + " \n", + " except Exception as e1:\n", + " print(f\"Method 1 registration failed: {str(e1)}\")\n", + " \n", + " try:\n", + " # Method 2: Create version with client API\n", + " model_version = client.create_model_version(\n", + " name=model_name,\n", + " source=f\"runs:/{run_id}/model\", # Use run URI instead of direct S3\n", + " run_id=run_id,\n", + " description=f\"Fine-tuned LLM deployed at endpoint: {endpoint_name}\"\n", + " )\n", + " mlflow_version = model_version.version\n", + " \n", + " # Wait for model registration to complete\n", + " for _ in range(10): # Try for up to ~50 seconds\n", + " version_details = client.get_model_version(model_name, model_version.version)\n", + " if version_details.status == \"READY\":\n", + " break\n", + " time.sleep(5)\n", + " \n", + " # Add tags to the registered model version\n", + " for key, value in tags.items():\n", + " client.set_model_version_tag(model_name, model_version.version, key, value)\n", + " except Exception as e2:\n", + " print(f\"Method 2 registration failed: {str(e2)}\")\n", + " mlflow_version = \"unknown\"\n", + " \n", + " if mlflow_version and mlflow_version != \"unknown\":\n", + " # Transition model to Production/Staging based on evaluation score\n", + " if evaluation_score >= 0.3: # Example threshold\n", + " client.transition_model_version_stage(\n", + " name=model_name,\n", + " version=mlflow_version,\n", + " stage=\"Production\",\n", + " archive_existing_versions=True\n", + " )\n", + " print(f\"Model {model_name} version {mlflow_version} promoted to Production\")\n", + " else:\n", + " client.transition_model_version_stage(\n", + " name=model_name,\n", + " version=mlflow_version,\n", + " stage=\"Staging\",\n", + " archive_existing_versions=False\n", + " )\n", + " print(f\"Model {model_name} version {mlflow_version} added to Staging due to lower evaluation score\")\n", + " \n", + " print(f\"Successfully registered model in MLflow: {model_name}, version: {mlflow_version}\")\n", + " \n", + " except Exception as e:\n", + " print(f\"Error registering model in MLflow: {str(e)}\")\n", + " mlflow_version = \"unknown\"\n", + " \n", + " # PART 2: REGISTER WITH SAGEMAKER MODEL REGISTRY\n", + " sm_model_version = \"unknown\"\n", + " try:\n", + " sm_client = boto3.client('sagemaker')\n", + " \n", + " # Create a normalized name for SageMaker resources\n", + " sm_model_name = model_name.replace(\".\", \"-\").replace(\"_\", \"-\")\n", + " \n", + " # Create or update model package group\n", + " try:\n", + " sm_client.describe_model_package_group(ModelPackageGroupName=sm_model_name)\n", + " print(f\"SageMaker model package group {sm_model_name} already exists\")\n", + " except sm_client.exceptions.ClientError:\n", + " sm_client.create_model_package_group(\n", + " ModelPackageGroupName=sm_model_name,\n", + " ModelPackageGroupDescription=f\"Fine-tuned LLM model: {model_name}\"\n", + " )\n", + " print(f\"Created SageMaker model package group: {sm_model_name}\")\n", + " \n", + " # Create a model package and register it\n", + " try:\n", + " # Create model package\n", + " response = sm_client.create_model_package(\n", + " ModelPackageGroupName=sm_model_name,\n", + " ModelPackageDescription=model_description,\n", + " SourceAlgorithmSpecification={\n", + " 'SourceAlgorithms': [\n", + " {\n", + " 'AlgorithmName': 'pytorch-llm',\n", + " 'ModelDataUrl': model_artifacts_s3_path\n", + " }\n", + " ]\n", + " },\n", + " ValidationSpecification={\n", + " 'ValidationRole': 'dummy-role', # Required but not used\n", + " 'ValidationProfiles': [\n", + " {\n", + " 'ProfileName': 'ValidationProfile1',\n", + " 'TransformJobDefinition': {\n", + " 'TransformInput': {\n", + " 'DataSource': {\n", + " 'S3DataSource': {\n", + " 'S3DataType': 'S3Prefix',\n", + " 'S3Uri': 's3://dummy-bucket/dummy-prefix' # Required but not used\n", + " }\n", + " }\n", + " },\n", + " 'TransformOutput': {\n", + " 'S3OutputPath': 's3://dummy-bucket/dummy-output' # Required but not used\n", + " },\n", + " 'TransformResources': {\n", + " 'InstanceType': 'ml.m5.large', # Required but not used\n", + " 'InstanceCount': 1\n", + " }\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " ModelApprovalStatus='Approved',\n", + " MetadataProperties={\n", + " 'GeneratedBy': pipeline_name,\n", + " 'Repository': model_id,\n", + " 'EvaluationScore': str(evaluation_score)\n", + " },\n", + " ModelMetrics={\n", + " 'ModelQuality': {\n", + " 'Statistics': {\n", + " 'ContentType': 'application/json',\n", + " 'S3Uri': f\"s3://{model_artifacts_s3_path.split('/', 3)[2]}/{run_id}/artifacts/model_info.json\"\n", + " }\n", + " }\n", + " }\n", + " )\n", + " \n", + " sm_model_version = response['ModelPackageArn'].split('/')[-1]\n", + " print(f\"Created SageMaker model package: {sm_model_version}\")\n", + " \n", + " except Exception as e_package:\n", + " print(f\"Error creating model package: {str(e_package)}\")\n", + " \n", + " # Log SageMaker details\n", + " mlflow.log_param(\"sagemaker_model_group\", sm_model_name)\n", + " mlflow.log_param(\"sagemaker_model_version\", sm_model_version)\n", + " \n", + " print(f\"Successfully integrated with SageMaker model registry\")\n", + " \n", + " except Exception as e:\n", + " print(f\"Warning: Error in SageMaker model registry integration: {str(e)}\")\n", + " \n", + " return model_name, str(mlflow_version)" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/requirements.txt b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/requirements.txt index 06e0d14..3b58af5 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/requirements.txt +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/requirements.txt @@ -1,8 +1,9 @@ +awscli==1.42.25 transformers==4.50.2 peft==0.14.0 accelerate==1.3.0 bitsandbytes==0.45.1 -datasets==3.2.0 +datasets==3.5.0 evaluate==0.4.3 huggingface_hub[hf_transfer]==0.33.4 mlflow diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/__init__.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py new file mode 100644 index 0000000..325847b --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py @@ -0,0 +1,114 @@ +# ### 6. Deploy Step +# This step deploys the model for evaluation + +import sagemaker +import boto3 +from sagemaker import get_execution_role +from sagemaker import Model +from sagemaker.model_monitor import DataCaptureConfig +import time +from sagemaker.workflow.function_step import step +from .pipeline_utils import PIPELINE_INSTANCE_TYPE + + +@step( + name="ModelDeploy", + instance_type=PIPELINE_INSTANCE_TYPE, + display_name="Model Deploy", + keep_alive_period_in_seconds=900 +) +def deploy( + model_artifacts_s3_path: str, + output_path: str, + model_id: str, +): + sagemaker_session = sagemaker.Session() + instance_count = 1 + instance_type = "ml.g5.2xlarge" + health_check_timeout = 700 + + # Get the name for the endpoint + endpoint_name = f"{model_id.split('/')[-1].replace('.', '-').replace('_','-')}-sft-djl" + + # Delete existing endpoint if it exists + print(f"Checking for existing endpoint: {endpoint_name}") + sm_client = boto3.client('sagemaker') + try: + sm_client.describe_endpoint(EndpointName=endpoint_name) + print(f"Endpoint {endpoint_name} exists, deleting it before deployment") + sm_client.delete_endpoint(EndpointName=endpoint_name) + + print(f"Deleting endpoint config {endpoint_name}") + sm_client.delete_endpoint_config(EndpointConfigName=endpoint_name) + + # Wait for endpoint to be fully deleted + print("Waiting for endpoint to be fully deleted...") + wait_seconds = 10 + total_wait_time = 0 + max_wait_time = 300 # 5 minutes maximum wait + endpoint_deleted = False + + while total_wait_time < max_wait_time and not endpoint_deleted: + try: + sm_client.describe_endpoint(EndpointName=endpoint_name) + print(f"Endpoint still exists, waiting {wait_seconds} seconds...") + time.sleep(wait_seconds) + total_wait_time += wait_seconds + except sm_client.exceptions.ClientError: + print(f"Endpoint {endpoint_name} successfully deleted") + endpoint_deleted = True + + if not endpoint_deleted: + print(f"Warning: Endpoint still exists after {max_wait_time} seconds") + + except sm_client.exceptions.ClientError: + print(f"Endpoint {endpoint_name} does not exist, proceeding with deployment") + + # Continue with model deployment + image_uri = sagemaker.image_uris.retrieve( + framework="djl-lmi", + region=sagemaker_session.boto_session.region_name, + version="latest" + ) + + model_data = model_artifacts_s3_path + + # Create model only once + model = Model( + image_uri=image_uri, + model_data=model_data, + role=get_execution_role(), + env={ + 'HF_MODEL_ID': "/opt/ml/model", # path to where sagemaker stores the model + 'OPTION_TRUST_REMOTE_CODE': 'true', + 'OPTION_ROLLING_BATCH': "vllm", + 'OPTION_DTYPE': 'bf16', + 'OPTION_QUANTIZE': 'fp8', + 'OPTION_TENSOR_PARALLEL_DEGREE': 'max', + 'OPTION_MAX_ROLLING_BATCH_SIZE': '32', + 'OPTION_MODEL_LOADING_TIMEOUT': '3600', + 'OPTION_MAX_MODEL_LEN': '4096' + } + ) + + print(f"deploying endpoint: {endpoint_name}") + + data_capture_config = DataCaptureConfig( + enable_capture=True, + sampling_percentage=100, + destination_s3_uri='s3://sagemaker-us-east-1-329542461890/data-capture/', + capture_options=["REQUEST", "RESPONSE"], + csv_content_types=["text/csv"], + json_content_types=["application/json"] + ) + + predictor = model.deploy( + endpoint_name=endpoint_name, + initial_instance_count=instance_count, + instance_type=instance_type, + container_startup_health_check_timeout=health_check_timeout, + model_data_download_timeout=3600, + data_capture_config=data_capture_config + ) + + return endpoint_name \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/evaluation_mlflow.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/evaluation_mlflow.py deleted file mode 100644 index bcb70a7..0000000 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/evaluation_mlflow.py +++ /dev/null @@ -1,61 +0,0 @@ -import boto3 -import sagemaker -from sagemaker.s3_utils import parse_s3_url -import mlflow -import tempfile -from pathlib import Path -import pandas as pd -import json -from dataclasses import dataclass -from typing import Tuple, Optional -import json - - -def evaluation(model, preprocess_step_ret, finetune_ret, mlflow_arn, experiment_name, run_id): - mlflow.set_tracking_uri(mlflow_arn) - mlflow.set_experiment(experiment_name) - - print(preprocess_step_ret['run_id']) - - with mlflow.start_run(run_id=preprocess_step_ret['run_id']) as run: - s3 = boto3.client("s3") - sess = sagemaker.Session() - - dataset_info = mlflow.get_run(preprocess_step_ret['run_id']).inputs.dataset_inputs[1].dataset - - print(dataset_info) - print(f"Dataset name: {dataset_info.name}") - print(f"Dataset digest: {dataset_info.digest}") - print(f"Dataset profile: {dataset_info.profile}") - print(f"Dataset schema: {dataset_info.schema}") - - dataset_source = mlflow.data.get_source(dataset_info) - ds = dataset_source.load() - # get the bucket name using full s3 poth - - eval_data=pd.read_json(ds, orient='records', lines=True) - - data = [] - for index, row in eval_data.iterrows(): - for message in row['messages']: - if message["role"] == "user": - question = message["content"] - elif message["role"] == "assistant": - answer = message["content"] - data.append({"question": question, "answer": answer}) - - df = pd.DataFrame(data, columns=["question", "answer"]) - print(df.head()) - - - logged_model = f"runs:/{preprocess_step_ret['run_id']}/model" - loaded_model = mlflow.pyfunc.load_model(model_uri=logged_model) - results = mlflow.evaluate( - model=loaded_model, - data=df, - targets="answer", - model_type="question-answering", - evaluator_config={"col_mapping": {"inputs": "question"}}, - ) - print(results.metrics) - return "done" \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_llama3b_hf.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_llama3b_hf.py deleted file mode 100644 index 9a7b394..0000000 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_llama3b_hf.py +++ /dev/null @@ -1,96 +0,0 @@ -from steps.utils import endpoint_exists -from sagemaker.jumpstart.estimator import JumpStartEstimator -from sagemaker.huggingface import HuggingFace -from huggingface_hub import HfFolder -import mlflow -import time -import json -import boto3 - -def finetune_llama3b(preprocess_step_ret, train_config, lora_config, role, mlflow_arn, experiment_name,run_name, *args): - - mlflow.set_tracking_uri(mlflow_arn) - mlflow.set_experiment(experiment_name) - - with mlflow.start_run(run_id=preprocess_step_ret['run_id']) as run: - - model_id = train_config["model_id"] - endpoint_name = train_config["endpoint_name"] - instance_type = train_config["finetune_instance_type"] - num_instances = train_config["finetune_num_instances"] - epoch = train_config["epoch"] - per_device_train_batch_size = train_config["per_device_train_batch_size"] - - lora_config = json.loads(lora_config) - - lora_r = lora_config["lora_r"] - lora_alpha = lora_config["lora_alpha"] - lora_dropout = lora_config["lora_dropout"] - - train_data_path = preprocess_step_ret["training_input_path"] - - training_job_name = f'huggingface-qlora-{train_config["epoch"]}-{lora_config["lora_r"]}-{time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())}' - - hyperparameters = { - 'model_id': model_id, # pre-trained model - 'dataset_path': '/opt/ml/input/data/training', # path where sagemaker will save training dataset - 'epochs': epoch, # number of training epochs - 'per_device_train_batch_size': per_device_train_batch_size, # batch size for training - 'lr': 2e-4, # learning rate used during training - 'hf_token': "", # huggingface token to access llama 2 - 'merge_weights': True, # whether to merge LoRA into the model - 'lora_r': lora_r, - 'lora_alpha': lora_alpha, - 'lora_dropout': lora_dropout, - 'mlflow_arn': mlflow_arn, - 'experiment_name': experiment_name, - 'run_id': preprocess_step_ret['run_id'] - } - - # Add SageMaker environment variables to help with debugging - environment = { - "HUGGINGFACE_HUB_CACHE": "/tmp/.cache", - "NCCL_DEBUG": "INFO", # Helps debug NCCL issues - "NCCL_P2P_DISABLE": "1", # Can help with some networking issues - "PYTORCH_CUDA_ALLOC_CONF": "max_split_size_mb:512" # Helps with memory management - } - - if endpoint_exists(endpoint_name): - print("Endpoint already exists") - training_job_name = None - else: - # Define distributed training configuration - distribution = { - 'torch_distributed': { - 'enabled': True - } - } - - huggingface_estimator = HuggingFace( - entry_point='llama3_fine_tuning.py', # train script - source_dir='scripts', # directory which includes all the files needed for training - instance_type=instance_type, # instances type used for the training job - instance_count=num_instances, # the number of instances used for training - base_job_name=training_job_name, # the name of the training job - role=role, # IAM role used in training job to access AWS resources - volume_size=300, # the size of the EBS volume in GB - py_version='py311', # the python version used in the training job - hyperparameters=hyperparameters, # the hyperparameters passed to the training job - environment=environment, - distribution=distribution, # Added distributed training config - image_uri=f'763104351884.dkr.ecr.{boto3.session.Session().region_name}.amazonaws.com/pytorch-training:2.5.1-gpu-py311-cu124-ubuntu22.04-sagemaker', - metric_definitions=[ - {'Name': 'huggingface-textgeneration:loss', 'Regex': "'loss':\s*([0-9.]+)"}, - {'Name': 'huggingface-textgeneration:epoch', 'Regex': "'epoch':\s*([0-9.]+)"}, - {'Name': 'huggingface-textgeneration:train_loss', 'Regex': "'train_loss':\s*([0-9.]+)"}, - ] - ) - - data = {'training': train_data_path} - - # starting the train job with our uploaded datasets as input - huggingface_estimator.fit(data, wait=True) - - training_job_name = huggingface_estimator.latest_training_job.name - - return {"training_job_name": training_job_name, "run_id": preprocess_step_ret['run_id']} \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_step.py new file mode 100644 index 0000000..42786fd --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_step.py @@ -0,0 +1,244 @@ +# **Fine-tuning Step** + +# This is where the actual model adaptation occurs. The step takes the preprocessed data and applies it to fine-tune the base LLM (in this case, a Deepseek model). It incorporates the LoRA technique for efficient adaptation. + +import sagemaker +import boto3 +import mlflow +import yaml +import json +import time +import datetime +import os +import traceback +import tempfile +from pathlib import Path +from sagemaker.pytorch import PyTorch +from sagemaker.workflow.function_step import step +from .pipeline_utils import PIPELINE_INSTANCE_TYPE + +@step( + name="ModelFineTuning", + instance_type=PIPELINE_INSTANCE_TYPE, + display_name="Model Fine Tuning", + keep_alive_period_in_seconds=900, + dependencies="./scripts/requirements.txt" +) +def train( + tracking_server_arn: str, + train_dataset_s3_path: str, + test_dataset_s3_path: str, + train_config_s3_path: str, + role: str, + experiment_name: str, + model_id: str, + run_id: str, +): + + + # Initialize variables and tracking + start_time = time.time() + model_name = model_id.split("/")[-1] if "/" in model_id else model_id + training_job_name = None + + mlflow.set_tracking_uri(tracking_server_arn) + mlflow.set_experiment(experiment_name) + + try: + with mlflow.start_run(run_id=run_id): + with mlflow.start_run(run_name="FinetuningStep", nested=True) as training_run: + mlflow.autolog() + training_run_id = training_run.info.run_id + # Enable detailed tracking + mlflow.set_tag("component", "model_fine_tuning") + mlflow.log_param("model_id", model_id) + mlflow.log_param("train_dataset", train_dataset_s3_path) + mlflow.log_param("test_dataset", test_dataset_s3_path) + mlflow.log_param("training_start_time", datetime.datetime.now().isoformat()) + + # Download and parse the training config YAML to log hyperparameters + with tempfile.NamedTemporaryFile(delete=False) as tmp: + s3_client = boto3.client("s3") + + # Parse S3 path + config_parts = train_config_s3_path.replace("s3://", "").split("/", 1) + bucket = config_parts[0] + key = config_parts[1] + + # Download config file + try: + s3_client.download_file(bucket, key, tmp.name) + # Parse the YAML config + with open(tmp.name, 'r') as f: + config = yaml.safe_load(f) + + # Log all hyperparameters from config + print("Logging hyperparameters to MLflow:") + for param_name, param_value in config.items(): + # Skip complex objects that can't be logged as parameters + if isinstance(param_value, (str, int, float, bool)): + print(f" {param_name}: {param_value}") + mlflow.log_param(param_name, param_value) + elif param_name == "fsdp_config" and isinstance(param_value, dict): + # Log nested config as JSON + mlflow.log_param("fsdp_config_json", json.dumps(param_value)) + + # Log file as artifact for reference + mlflow.log_artifact(tmp.name, "training_config") + + except Exception as e: + print(f"Error parsing config file: {e}") + + finally: + # Clean up temp file + if os.path.exists(tmp.name): + os.remove(tmp.name) + + # Launch the training job + job_name = f"deepseek-finetune-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}" + + sagemaker_session = sagemaker.Session() + + # Define metric definitions for more detailed CloudWatch metrics + metric_definitions = [ + {'Name': 'loss', 'Regex': "'loss':\\s*([0-9.]+)"}, + {'Name': 'epoch', 'Regex': "'epoch':\\s*([0-9.]+)"}, + {'Name': 'train_loss', 'Regex': "'train_loss':\\s*([0-9.]+)"}, + {'Name': 'lr', 'Regex': "'learning_rate':\\s*([0-9.e-]+)"}, + {'Name': 'step', 'Regex': "'step':\\s*([0-9.]+)"}, + {'Name': 'samples_per_second', 'Regex': "'train_samples_per_second':\\s*([0-9.]+)"}, + ] + + # Log the metric definitions we're using + mlflow.log_param("tracked_metrics", [m['Name'] for m in metric_definitions]) + + pytorch_estimator = PyTorch( + entry_point='train.py', + source_dir="./scripts", + job_name=job_name, + base_job_name=job_name, + max_run=50000, + role=role, + framework_version="2.2.0", + py_version="py310", + instance_count=1, + instance_type="ml.p3.2xlarge", + sagemaker_session=sagemaker_session, + volume_size=50, + disable_output_compression=False, + keep_alive_period_in_seconds=1800, + distribution={"torch_distributed": {"enabled": True}}, + hyperparameters={ + "config": "/opt/ml/input/data/config/args.yaml" + }, + metric_definitions=metric_definitions, + debugger_hook_config=False, + environment={"MLFLOW_RUN_ID": training_run_id} + ) + + # Define a data input dictionary with our uploaded S3 URIs + data = { + 'train': train_dataset_s3_path, + 'test': test_dataset_s3_path, + 'config': train_config_s3_path + } + + print(f"Data for Training Run: {data}") + + # Log training job information + mlflow.log_param("job_name", job_name) + mlflow.log_param("instance_type", "ml.p3.2xlarge") + + # Start the training job + pytorch_estimator.fit(data, wait=True) + + # Get information about the completed training job + latest_run_job_name = pytorch_estimator.latest_training_job.job_name + print(f"Latest Job Name: {latest_run_job_name}") + + sagemaker_client = boto3.client('sagemaker') + + # Describe the training job + response = sagemaker_client.describe_training_job(TrainingJobName=latest_run_job_name) + + # Extract the model artifacts S3 path + model_artifacts_s3_path = response['ModelArtifacts']['S3ModelArtifacts'] + + # Extract the output path (this is the general output location) + output_path = response['OutputDataConfig']['S3OutputPath'] + + # Get training time metrics + training_start_time = response.get('TrainingStartTime') + training_end_time = response.get('TrainingEndTime') + billable_time = response.get('BillableTimeInSeconds', 0) + + # Calculate duration + total_training_time = 0 + if training_start_time and training_end_time: + total_training_time = (training_end_time - training_start_time).total_seconds() + + # Log job results and metrics to MLflow + # Log basic job info + mlflow.log_param("training_job_name", latest_run_job_name) + mlflow.log_param("model_artifacts_path", model_artifacts_s3_path) + mlflow.log_param("output_path", output_path) + + # Log performance metrics + mlflow.log_metric("billable_time_seconds", billable_time) + mlflow.log_metric("total_training_time_seconds", total_training_time) + + # Log training job status + mlflow.log_param("training_job_status", response.get('TrainingJobStatus')) + + # Log any secondary status + if 'SecondaryStatus' in response: + mlflow.log_param("secondary_status", response.get('SecondaryStatus')) + + # Log any failure reason + if 'FailureReason' in response: + mlflow.log_param("failure_reason", response.get('FailureReason')) + + # Get CloudWatch logs for the training job + logs_client = boto3.client('logs') + log_group = "/aws/sagemaker/TrainingJobs" + log_stream = latest_run_job_name + + try: + # Get the last 1000 log events + log_events = logs_client.get_log_events( + logGroupName=log_group, + logStreamName=log_stream, + limit=1000 + ) + + # Extract and save logs + log_output = "\n".join([event['message'] for event in log_events['events']]) + + # Save logs to file and log as artifact + with tempfile.NamedTemporaryFile(delete=False, mode='w', suffix='.txt') as tmp: + tmp.write(log_output) + log_file_path = tmp.name + + mlflow.log_artifact(log_file_path, "training_logs") + os.remove(log_file_path) + + except Exception as e: + print(f"Error fetching training logs: {e}") + + # Log total execution time of this step + step_duration = time.time() - start_time + mlflow.log_metric("step_execution_time_seconds", step_duration) + + # Log model metadata + mlflow.set_tag("model_path", model_artifacts_s3_path) + mlflow.set_tag("training_completed_at", datetime.datetime.now().isoformat()) + + print(f"Model artifacts S3 path: {model_artifacts_s3_path}") + + except Exception as e: + error_msg = f"Error in model fine-tuning: {str(e)}\n{traceback.format_exc()}" + print(error_msg) + + raise RuntimeError(f"Fine-tuning failed: {str(e)}") + + return run_id, training_run_id, model_artifacts_s3_path, output_path \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/model_registration_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/model_registration_step.py new file mode 100644 index 0000000..d4e819e --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/model_registration_step.py @@ -0,0 +1,465 @@ +# ### 7. Model Registration Step +# This step registers the fine-tuned model in MLflow model registry and SageMaker AI model registry + +from sagemaker.workflow.function_step import step +from .pipeline_utils import PIPELINE_INSTANCE_TYPE + + +# @step( +# name="ModelRegistration", +# instance_type=PIPELINE_INSTANCE_TYPE, +# display_name="Model Registration", +# keep_alive_period_in_seconds=900 +# ) +# def register_model( +# tracking_server_arn: str, +# experiment_name: str, +# run_id: str, +# model_artifacts_s3_path: str, +# model_id: str, +# model_name: str, +# endpoint_name: str, +# evaluation_score: float, +# pipeline_name: str, +# model_description: str +# ): +# import json +# import mlflow +# import boto3 +# from datetime import datetime + +# print(f"Registering model: {model_name}") + +# # Set up MLflow tracking +# mlflow.set_tracking_uri(tracking_server_arn) +# mlflow.set_experiment(experiment_name) + +# # Connect to MLflow with the specific run +# with mlflow.start_run(run_id=run_id): +# # Create model metadata +# tags = { +# "model_id": model_id, +# "base_model": model_id.split('/')[-1], +# "task": "medical_qa", +# "framework": "pytorch", +# "endpoint_name": endpoint_name, +# "model_artifacts_s3_path": model_artifacts_s3_path, +# "deployment_timestamp": datetime.now().isoformat(), +# "description": model_description, +# "registered_by": pipeline_name +# } + +# # Log model info as parameters +# mlflow.log_param("registered_model_name", model_name) +# mlflow.log_param("model_artifacts_path", model_artifacts_s3_path) +# mlflow.log_param("evaluation_score", evaluation_score) +# mlflow.log_param("endpoint_name", endpoint_name) +# mlflow.log_param("registration_timestamp", datetime.now().isoformat()) + +# # Log endpoint information as an artifact +# with open("/tmp/model_info.json", "w") as f: +# json.dump({ +# "model_name": model_name, +# "model_id": model_id, +# "endpoint_name": endpoint_name, +# "model_artifacts_s3_path": model_artifacts_s3_path, +# "evaluation_score": float(evaluation_score), +# "registration_timestamp": datetime.now().isoformat() +# }, f, indent=2) +# mlflow.log_artifact("/tmp/model_info.json") + +# # Register the model +# try: +# client = mlflow.tracking.MlflowClient() + +# # Check if model exists and create if it doesn't +# try: +# client.get_registered_model(model_name) +# print(f"Model {model_name} already exists in registry") +# except mlflow.exceptions.MlflowException: +# client.create_registered_model( +# name=model_name, +# description=f"Fine-tuned medical LLM based on {model_id}" +# ) +# print(f"Created new registered model: {model_name}") + +# # Create a new model version +# model_version = client.create_model_version( +# name=model_name, +# source=model_artifacts_s3_path, # Direct S3 path to model artifacts +# run_id=run_id, +# description=f"Fine-tuned LLM deployed at endpoint: {endpoint_name}" +# ) + +# # Wait for model registration to complete +# import time +# for _ in range(10): # Try for up to ~50 seconds +# version_details = client.get_model_version(model_name, model_version.version) +# if version_details.status == "READY": +# break +# time.sleep(5) + +# # Add tags to the registered model version +# for key, value in tags.items(): +# client.set_model_version_tag(model_name, model_version.version, key, value) + +# # Transition model to Production/Staging based on evaluation score +# if evaluation_score >= 0.3: # Example threshold +# client.transition_model_version_stage( +# name=model_name, +# version=model_version.version, +# stage="Production", +# archive_existing_versions=True +# ) +# print(f"Model {model_name} version {model_version.version} promoted to Production") +# else: +# client.transition_model_version_stage( +# name=model_name, +# version=model_version.version, +# stage="Staging", +# archive_existing_versions=False +# ) +# print(f"Model {model_name} version {model_version.version} added to Staging due to lower evaluation score") + +# print(f"Successfully registered model: {model_name}, version: {model_version.version}") +# latest_version = model_version.version + +# except Exception as e: +# print(f"Error registering model: {str(e)}") +# # Try alternative approach +# try: +# # Register model using mlflow API (simpler approach) +# model_details = mlflow.register_model( +# model_uri=f"runs:/{run_id}/model", +# name=model_name, +# tags=tags +# ) +# latest_version = model_details.version +# print(f"Alternative registration successful: {model_name} version {latest_version}") +# except Exception as e2: +# print(f"Alternative registration failed: {str(e2)}") +# latest_version = "unknown" + +# # Create SageMaker integration for the model +# try: +# sm_client = boto3.client('sagemaker') + +# # Create a normalized name for SageMaker resources +# sm_model_name = model_name.replace(".", "-").replace("_", "-") + +# # Create or update model package group +# try: +# sm_client.describe_model_package_group(ModelPackageGroupName=sm_model_name) +# print(f"SageMaker model package group {sm_model_name} already exists") +# except sm_client.exceptions.ClientError: +# sm_client.create_model_package_group( +# ModelPackageGroupName=sm_model_name, +# ModelPackageGroupDescription=f"Fine-tuned LLM model: {model_name}" +# ) +# print(f"Created SageMaker model package group: {sm_model_name}") + +# # Log SageMaker details +# mlflow.log_param("sagemaker_model_group", sm_model_name) + +# print(f"Successfully integrated with SageMaker model registry") + +# except Exception as e: +# print(f"Warning: Error in SageMaker model registry integration: {str(e)}") + +# return model_name, latest_version + + + + + +@step( + name="ModelRegistration", + instance_type=PIPELINE_INSTANCE_TYPE, + display_name="Model Registration", + keep_alive_period_in_seconds=900 +) +def register_model( + tracking_server_arn: str, + experiment_name: str, + run_id: str, + model_artifacts_s3_path: str, + model_id: str, + model_name: str, + endpoint_name: str, + evaluation_score: float, + pipeline_name: str, + model_description: str +): + import json + import mlflow + import boto3 + import os + import tempfile + import time + from datetime import datetime + + print(f"Registering model: {model_name}") + + # Set up MLflow tracking + mlflow.set_tracking_uri(tracking_server_arn) + mlflow.set_experiment(experiment_name) + + # Connect to MLflow with the specific run + with mlflow.start_run(run_id=run_id): + # Create model metadata + tags = { + "model_id": model_id, + "base_model": model_id.split('/')[-1], + "task": "medical_qa", + "framework": "pytorch", + "endpoint_name": endpoint_name, + "model_artifacts_s3_path": model_artifacts_s3_path, + "deployment_timestamp": datetime.now().isoformat(), + "description": model_description, + "registered_by": pipeline_name + } + + # Log model info as parameters + mlflow.log_param("registered_model_name", model_name) + mlflow.log_param("model_artifacts_path", model_artifacts_s3_path) + mlflow.log_param("evaluation_score", evaluation_score) + mlflow.log_param("endpoint_name", endpoint_name) + mlflow.log_param("registration_timestamp", datetime.now().isoformat()) + + # Log endpoint information as an artifact + model_info = { + "model_name": model_name, + "model_id": model_id, + "endpoint_name": endpoint_name, + "model_artifacts_s3_path": model_artifacts_s3_path, + "evaluation_score": float(evaluation_score), + "registration_timestamp": datetime.now().isoformat() + } + + with open("/tmp/model_info.json", "w") as f: + json.dump(model_info, f, indent=2) + mlflow.log_artifact("/tmp/model_info.json") + + # Create model card + model_card = f""" + # {model_name} + + ## Model Information + - **Base Model**: {model_id} + - **Task**: Medical Question Answering + - **Evaluation Score**: {evaluation_score:.4f} + - **Endpoint**: {endpoint_name} + + ## Description + {model_description} + + ## Registration Details + - Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} + - Pipeline: {pipeline_name} + """ + + with open("/tmp/model_card.md", "w") as f: + f.write(model_card) + mlflow.log_artifact("/tmp/model_card.md") + + # PART 1: REGISTER WITH MLFLOW MODEL REGISTRY + mlflow_version = None + try: + client = mlflow.tracking.MlflowClient() + + # Check if model exists and create if it doesn't + try: + client.get_registered_model(model_name) + print(f"Model {model_name} already exists in MLflow registry") + except mlflow.exceptions.MlflowException: + client.create_registered_model( + name=model_name, + description=f"Fine-tuned medical LLM based on {model_id}" + ) + print(f"Created new registered model: {model_name}") + + # Create empty model directory with artifacts + with tempfile.TemporaryDirectory() as tmp_dir: + # Create a minimal model file to log + os.makedirs(os.path.join(tmp_dir, "model"), exist_ok=True) + + # Copy model info and card to directory + with open(os.path.join(tmp_dir, "model", "model_info.json"), "w") as f: + json.dump(model_info, f, indent=2) + + with open(os.path.join(tmp_dir, "model", "model_card.md"), "w") as f: + f.write(model_card) + + # Create a model reference file pointing to the S3 artifacts + model_ref = { + "artifact_path": model_artifacts_s3_path, + "flavors": { + "pytorch": { + "model_data": model_artifacts_s3_path, + "pytorch_version": "2.0+" + } + }, + "run_id": run_id, + "model_class": "LLM", + "model_format": "PyTorch" + } + + with open(os.path.join(tmp_dir, "model", "MLmodel"), "w") as f: + json.dump(model_ref, f, indent=2) + + # Log artifacts directory as model + mlflow.log_artifacts(tmp_dir, artifact_path="") + + # Now register the model - try both methods + try: + # Method 1: Use direct registration with source as run URI + model_uri = f"runs:/{run_id}/model" + model_details = mlflow.register_model( + model_uri=model_uri, + name=model_name, + tags=tags + ) + mlflow_version = model_details.version + + except Exception as e1: + print(f"Method 1 registration failed: {str(e1)}") + + try: + # Method 2: Create version with client API + model_version = client.create_model_version( + name=model_name, + source=f"runs:/{run_id}/model", # Use run URI instead of direct S3 + run_id=run_id, + description=f"Fine-tuned LLM deployed at endpoint: {endpoint_name}" + ) + mlflow_version = model_version.version + + # Wait for model registration to complete + for _ in range(10): # Try for up to ~50 seconds + version_details = client.get_model_version(model_name, model_version.version) + if version_details.status == "READY": + break + time.sleep(5) + + # Add tags to the registered model version + for key, value in tags.items(): + client.set_model_version_tag(model_name, model_version.version, key, value) + except Exception as e2: + print(f"Method 2 registration failed: {str(e2)}") + mlflow_version = "unknown" + + if mlflow_version and mlflow_version != "unknown": + # Transition model to Production/Staging based on evaluation score + if evaluation_score >= 0.3: # Example threshold + client.transition_model_version_stage( + name=model_name, + version=mlflow_version, + stage="Production", + archive_existing_versions=True + ) + print(f"Model {model_name} version {mlflow_version} promoted to Production") + else: + client.transition_model_version_stage( + name=model_name, + version=mlflow_version, + stage="Staging", + archive_existing_versions=False + ) + print(f"Model {model_name} version {mlflow_version} added to Staging due to lower evaluation score") + + print(f"Successfully registered model in MLflow: {model_name}, version: {mlflow_version}") + + except Exception as e: + print(f"Error registering model in MLflow: {str(e)}") + mlflow_version = "unknown" + + # PART 2: REGISTER WITH SAGEMAKER MODEL REGISTRY + sm_model_version = "unknown" + try: + sm_client = boto3.client('sagemaker') + + # Create a normalized name for SageMaker resources + sm_model_name = model_name.replace(".", "-").replace("_", "-") + + # Create or update model package group + try: + sm_client.describe_model_package_group(ModelPackageGroupName=sm_model_name) + print(f"SageMaker model package group {sm_model_name} already exists") + except sm_client.exceptions.ClientError: + sm_client.create_model_package_group( + ModelPackageGroupName=sm_model_name, + ModelPackageGroupDescription=f"Fine-tuned LLM model: {model_name}" + ) + print(f"Created SageMaker model package group: {sm_model_name}") + + # Create a model package and register it + try: + # Create model package + response = sm_client.create_model_package( + ModelPackageGroupName=sm_model_name, + ModelPackageDescription=model_description, + SourceAlgorithmSpecification={ + 'SourceAlgorithms': [ + { + 'AlgorithmName': 'pytorch-llm', + 'ModelDataUrl': model_artifacts_s3_path + } + ] + }, + ValidationSpecification={ + 'ValidationRole': 'dummy-role', # Required but not used + 'ValidationProfiles': [ + { + 'ProfileName': 'ValidationProfile1', + 'TransformJobDefinition': { + 'TransformInput': { + 'DataSource': { + 'S3DataSource': { + 'S3DataType': 'S3Prefix', + 'S3Uri': 's3://dummy-bucket/dummy-prefix' # Required but not used + } + } + }, + 'TransformOutput': { + 'S3OutputPath': 's3://dummy-bucket/dummy-output' # Required but not used + }, + 'TransformResources': { + 'InstanceType': 'ml.m5.large', # Required but not used + 'InstanceCount': 1 + } + } + } + ] + }, + ModelApprovalStatus='Approved', + MetadataProperties={ + 'GeneratedBy': pipeline_name, + 'Repository': model_id, + 'EvaluationScore': str(evaluation_score) + }, + ModelMetrics={ + 'ModelQuality': { + 'Statistics': { + 'ContentType': 'application/json', + 'S3Uri': f"s3://{model_artifacts_s3_path.split('/', 3)[2]}/{run_id}/artifacts/model_info.json" + } + } + } + ) + + sm_model_version = response['ModelPackageArn'].split('/')[-1] + print(f"Created SageMaker model package: {sm_model_version}") + + except Exception as e_package: + print(f"Error creating model package: {str(e_package)}") + + # Log SageMaker details + mlflow.log_param("sagemaker_model_group", sm_model_name) + mlflow.log_param("sagemaker_model_version", sm_model_version) + + print(f"Successfully integrated with SageMaker model registry") + + except Exception as e: + print(f"Warning: Error in SageMaker model registry integration: {str(e)}") + + return model_name, str(mlflow_version) \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py new file mode 100644 index 0000000..f51dd1d --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py @@ -0,0 +1,104 @@ +import boto3 +import botocore +import json +import time +from datetime import datetime + + +PIPELINE_INSTANCE_TYPE = "ml.m5.xlarge" + + +PROMPT_TEMPLATE = f""" +<|begin_of_text|> +<|start_header_id|>system<|end_header_id|> +You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. +Below is an instruction that describes a task, paired with an input that provides further context. +Write a response that appropriately completes the request. +Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. +<|eot_id|><|start_header_id|>user<|end_header_id|> +{{question}}<|eot_id|> +<|start_header_id|>assistant<|end_header_id|> +{{complex_cot}} + +{{answer}} +<|eot_id|> +""" + + +def endpoint_exists(endpoint_name): + endpoint_exist = False + + client = boto3.client('sagemaker') + response = client.list_endpoints() + endpoints = response["Endpoints"] + + for endpoint in endpoints: + if endpoint_name == endpoint["EndpointName"]: + endpoint_exist = True + break + + return endpoint_exist + + +def create_training_job_name(model_id): + return f"{model_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]}" + + +# Template dataset to add prompt to each sample +def template_dataset(sample): + try: + sample["text"] = PROMPT_TEMPLATE.format(question=sample["Question"], + complex_cot=sample["Complex_CoT"], + answer=sample["Response"]) + return sample + except KeyError as e: + print(f"KeyError in template_dataset: {str(e)}") + # Provide default values for missing fields + missing_key = str(e).strip("'") + if missing_key == "Question": + sample["text"] = PROMPT_TEMPLATE.format( + question="[Missing question]", + complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), + answer=sample.get("Response", "[Missing response]") + ) + elif missing_key == "Complex_CoT": + sample["text"] = PROMPT_TEMPLATE.format( + question=sample["Question"], + complex_cot="[Missing CoT]", + answer=sample.get("Response", "[Missing response]") + ) + elif missing_key == "Response": + sample["text"] = PROMPT_TEMPLATE.format( + question=sample["Question"], + complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), + answer="[Missing response]" + ) + return sample + + +def invoke_sagemaker_endpoint(payload, endpoint_name): + """ + Invoke a SageMaker endpoint with the given payload. + + Args: + payload (dict): The input data to send to the endpoint + endpoint_name (str): The name of the SageMaker endpoint + + Returns: + dict: The response from the endpoint + """ + sm_client = boto3.client('sagemaker-runtime') + try: + start_time = time.time() + response = sm_client.invoke_endpoint( + EndpointName=endpoint_name, + ContentType='application/json', + Body=json.dumps(payload) + ) + inference_time = time.time() - start_time + + response_body = response['Body'].read().decode('utf-8') + return json.loads(response_body), inference_time + except Exception as e: + print(f"Error invoking endpoint {endpoint_name}: {str(e)}") + return None, -1 \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_llama3.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_llama3.py deleted file mode 100644 index d6ae3a1..0000000 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_llama3.py +++ /dev/null @@ -1,64 +0,0 @@ -# Temporary preprocess step (to be changed with new dataset) -import boto3 -import pandas as pd -from datasets import load_dataset -from datasets import Dataset -from random import randint -import mlflow -import json - - -system_message = """You are Llama, an AI assistant. Your knowledge spans a wide range of topics, allowing you to anser the questions with honesty and truthfulness.""" - -def create_conversation(sample): - if sample["messages"][0]["role"] == "system": - return sample - else: - sample["messages"] = [{"role": "system", "content": system_message}] + sample["messages"] - return sample - -def preprocess(s3_bucket, dataset_name, train_sample, eval_sample, mlflow_arn, experiment_name, run_name): - - mlflow.set_tracking_uri(mlflow_arn) - mlflow.set_experiment(experiment_name) - - - # This is a very simple example, you can add your own data processing code here - dataset = load_dataset(dataset_name) - dataset = dataset.filter(lambda x: x['category'] == 'Open QA') - - columns_to_remove = list(dataset["train"].features) - columns_to_remove.remove("messages") - dataset = dataset.map(create_conversation, remove_columns=columns_to_remove,batched=False) - - dataset["train"] = dataset["train"].filter(lambda x: len(x["messages"][1:]) % 2 == 0) - dataset["test"] = dataset["test"].filter(lambda x: len(x["messages"][1:]) % 2 == 0) - - dataset["train"].to_json("train_dataset.json", orient="records", force_ascii=False) - dataset["test"].to_json("test_dataset.json", orient="records", force_ascii=False) - - # save training and test data to s3 - s3 = boto3.client("s3") - s3.upload_file("train_dataset.json", s3_bucket, f"dataset/{dataset_name}/{train_sample}/train/train_dataset.json") - s3.upload_file("test_dataset.json", s3_bucket, f"dataset/{dataset_name}/{eval_sample}/eval/eval_dataset.json") - - - training_input_path = f's3://{s3_bucket}/dataset/{dataset_name}/{train_sample}/train/train_dataset.json' - eval_input_path = f's3://{s3_bucket}/dataset/{dataset_name}/{eval_sample}/eval/eval_dataset.json' - - with mlflow.start_run(run_name=run_name) as run: - - run_id = run.info.run_id - print(run_id) - - # create pandas dataframe from train json - df_train = pd.read_json("train_dataset.json", orient="records", lines=True) - df_evaluate = pd.read_json("test_dataset.json", orient="records", lines=True) - - training_data = mlflow.data.from_pandas(df_train, source=training_input_path) - mlflow.log_input(training_data, context="training") - - evaluation_data = mlflow.data.from_pandas(df_evaluate, source=eval_input_path) - mlflow.log_input(evaluation_data, context="evaluation") - - return {"training_input_path": training_input_path, "eval_input_path": eval_input_path, "run_id": run_id} diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py new file mode 100644 index 0000000..bd85fa8 --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py @@ -0,0 +1,218 @@ +# **Preprocessing Step** + +# This step handles data preparation. We are going to prepare data for training and evaluation. We will log this data in MLflow +import boto3 +import shutil +import sagemaker +import os +import pandas as pd +from sagemaker.config import load_sagemaker_config +import mlflow +import traceback +from datasets import load_dataset +from sklearn.model_selection import train_test_split +from datasets import Dataset, DatasetDict +from random import randint +from sagemaker.workflow.function_step import step +from .pipeline_utils import ( + PIPELINE_INSTANCE_TYPE, + template_dataset +) + + +@step( + name="DataPreprocessing", + instance_type=PIPELINE_INSTANCE_TYPE, + display_name="Data Preprocessing", + keep_alive_period_in_seconds=900 +) +def preprocess( + tracking_server_arn: str, + input_path: str, + experiment_name: str, + run_name: str, +) -> tuple: + + + # prompt_template = f""" + # <|begin_of_text|> + # <|start_header_id|>system<|end_header_id|> + # You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. + # Below is an instruction that describes a task, paired with an input that provides further context. + # Write a response that appropriately completes the request. + # Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. + # <|eot_id|><|start_header_id|>user<|end_header_id|> + # {{question}}<|eot_id|> + # <|start_header_id|>assistant<|end_header_id|> + # {{complex_cot}} + + # {{answer}} + # <|eot_id|> + # """ + + # # Include template_dataset function directly here + # def template_dataset(sample): + # try: + # sample["text"] = prompt_template.format( + # question=sample["Question"], + # complex_cot=sample["Complex_CoT"], + # answer=sample["Response"] + # ) + # return sample + # except KeyError as e: + # print(f"KeyError in template_dataset: {str(e)}") + # # Provide default values for missing fields + # missing_key = str(e).strip("'") + # if missing_key == "Question": + # sample["text"] = prompt_template.format( + # question="[Missing question]", + # complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), + # answer=sample.get("Response", "[Missing response]") + # ) + # elif missing_key == "Complex_CoT": + # sample["text"] = prompt_template.format( + # question=sample["Question"], + # complex_cot="[Missing CoT]", + # answer=sample.get("Response", "[Missing response]") + # ) + # elif missing_key == "Response": + # sample["text"] = prompt_template.format( + # question=sample["Question"], + # complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), + # answer="[Missing response]" + # ) + # return sample + + mlflow.set_tracking_uri(tracking_server_arn) + mlflow.set_experiment(experiment_name) + + # Preprocessing code + try: + with mlflow.start_run(run_name=run_name) as run: + run_id = run.info.run_id + with mlflow.start_run(run_name="Processing", nested=True): + # Initialize SageMaker and S3 clients + sagemaker_session = sagemaker.Session() + s3_client = boto3.client('s3') + + bucket_name = sagemaker_session.default_bucket() + default_prefix = sagemaker_session.default_bucket_prefix + configs = load_sagemaker_config() + + # Set paths + if default_prefix: + input_path = f'{default_prefix}/datasets/llm-fine-tuning-modeltrainer-sft' + else: + input_path = f'datasets/llm-fine-tuning-modeltrainer-sft' + + # Load dataset with proper error handling + sample_dataset_size = 100 + try: + dataset = load_dataset("FreedomIntelligence/medical-o1-reasoning-SFT", "en") + except Exception as e: + error_msg = f"Error loading dataset: {str(e)}\n{traceback.format_exc()}" + print(error_msg) + raise RuntimeError(f"Failed to load dataset: {str(e)}") + + df = pd.DataFrame(dataset['train']) + df = df[:sample_dataset_size] + + # Split dataset + train, test = train_test_split(df, test_size=0.1, random_state=42, shuffle=True) + + print("Number of train elements: ", len(train)) + print("Number of test elements: ", len(test)) + + # Log dataset statistics if MLflow is enabled + mlflow.log_param("dataset_source", "FreedomIntelligence/medical-o1-reasoning-SFT") + mlflow.log_param("train_size", len(train)) + mlflow.log_param("test_size", len(test)) + mlflow.log_param("dataset_sample_size", sample_dataset_size) # Log that we're using a subset of 100 samples + + # Create datasets + train_dataset = Dataset.from_pandas(train) + test_dataset = Dataset.from_pandas(test) + dataset = DatasetDict({"train": train_dataset, "test": test_dataset}) + train_dataset = dataset["train"].map(template_dataset, remove_columns=list(dataset["train"].features)) + test_dataset = dataset["test"].map(template_dataset, remove_columns=list(dataset["test"].features)) + + # Safely get a sample text, handling potential index errors + try: + sample_index = randint(0, len(train_dataset) - 1) + sample_text = train_dataset[sample_index]["text"] + print(f"Sample text from index {sample_index}:") + print(sample_text) + except (IndexError, KeyError) as e: + sample_text = "Error retrieving sample text: " + str(e) + print(sample_text) + + # Create directories with error handling + try: + os.makedirs("./data/train", exist_ok=True) + os.makedirs("./data/test", exist_ok=True) + except OSError as e: + error_msg = f"Error creating directories: {str(e)}" + print(error_msg) + + # Save datasets locally with error handling + try: + train_dataset.to_json("./data/train/dataset.json", orient="records") + test_dataset.to_json("./data/test/dataset.json", orient="records") + except Exception as e: + error_msg = f"Error saving datasets locally: {str(e)}\n{traceback.format_exc()}" + print(error_msg) + raise RuntimeError(f"Failed to save datasets locally: {str(e)}") + + # Define S3 paths + train_data_path = f"s3://{bucket_name}/{input_path}/train/dataset.json" + test_dataset_path = f"s3://{bucket_name}/{input_path}/test/dataset.json" + + # Store results for return + result_train_data_path = train_data_path + result_test_dataset_path = test_dataset_path + + # Log dataset paths if MLflow is enabled + mlflow.log_param("train_data_path", train_data_path) + mlflow.log_param("test_dataset_path", test_dataset_path) + + # Upload files to S3 with retries + max_retries = 3 + for attempt in range(max_retries): + try: + print(f"Uploading train dataset to S3, attempt {attempt+1}/{max_retries}") + s3_client.upload_file("./data/train/dataset.json", bucket_name, f"{input_path}/train/dataset.json") + print(f"Uploading test dataset to S3, attempt {attempt+1}/{max_retries}") + s3_client.upload_file("./data/test/dataset.json", bucket_name, f"{input_path}/test/dataset.json") + print("S3 upload successful") + break + except Exception as e: + error_msg = f"Error in S3 upload (attempt {attempt+1}/{max_retries}): {str(e)}" + print(error_msg) + if attempt == max_retries - 1: # Last attempt failed + raise RuntimeError(f"Failed to upload datasets to S3 after {max_retries} attempts: {str(e)}") + + print(f"Datasets uploaded to:") + print(train_data_path) + print(test_dataset_path) + + # Log a sample of the dataset as an artifact if MLflow is enabled + try: + with open("./data/sample.txt", "w") as f: + f.write(sample_text) + mlflow.log_artifact("./data/sample.txt", "dataset_samples") + except Exception as e: + print(f"Error logging sample as artifact: {str(e)}") + + # Clean up + try: + if os.path.exists("./data"): + shutil.rmtree("./data") + except Exception as e: + print(f"Warning: Error cleaning up temporary files: {str(e)}") + + except Exception as e: + error_msg = f"Critical error in preprocessing: {str(e)}\n{traceback.format_exc()}" + print(error_msg) + raise RuntimeError(f"Preprocessing failed: {str(e)}") + + return run_id, result_train_data_path, result_test_dataset_path \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py new file mode 100644 index 0000000..dc785a6 --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py @@ -0,0 +1,486 @@ +# ### 8. Qualitative Evaluation Step + +# After fine-tuning, this step assesses the model's qualitative performance. + +from sagemaker.workflow.function_step import step +from .pipeline_utils import PIPELINE_INSTANCE_TYPE + + +@step( + name="QualitativeModelEvaluation", + instance_type=PIPELINE_INSTANCE_TYPE, + display_name="Qualitative Model Evaluation", + keep_alive_period_in_seconds=900, + dependencies="./eval/requirements.txt" +) +def qualitative_evaluate( + tracking_server_arn: str, + experiment_name: str, + run_id: str, + endpoint_name: str, +) -> dict: + import os + import json + import time + import boto3 + import pandas as pd + import numpy as np + import matplotlib.pyplot as plt + from tqdm.notebook import tqdm + from datasets import load_dataset + import mlflow + import uuid + import traceback + from datetime import datetime + + # MLflow LLM-as-a-judge imports (compatible with MLflow 2.x) + from mlflow.metrics.genai import EvaluationExample, make_genai_metric + + def invoke_sagemaker_endpoint(payload, endpoint_name): + """ + Invoke a SageMaker endpoint with the given payload. + """ + try: + start_time = time.time() + response = sm_client.invoke_endpoint( + EndpointName=endpoint_name, + ContentType='application/json', + Body=json.dumps(payload) + ) + inference_time = time.time() - start_time + + response_body = response['Body'].read().decode('utf-8') + return json.loads(response_body), inference_time + except Exception as e: + print(f"Error invoking endpoint {endpoint_name}: {str(e)}") + return None, -1 + + def create_bedrock_judge_metrics(): + """ + Create custom LLM-as-a-judge metrics using AWS Bedrock Claude as the judge. + + Returns: + list: List of custom metrics for medical evaluation + """ + + # Medical Accuracy Metric using Bedrock Claude + medical_accuracy_examples = [ + EvaluationExample( + input="What is the first-line treatment for hypertension?", + output="ACE inhibitors or thiazide diuretics are typically first-line treatments for hypertension.", + score=4, + justification="The response correctly identifies evidence-based first-line treatments for hypertension." + ), + EvaluationExample( + input="What causes Type 1 diabetes?", + output="Type 1 diabetes is caused by autoimmune destruction of pancreatic beta cells.", + score=5, + justification="Accurate and concise explanation of Type 1 diabetes pathophysiology." + ), + EvaluationExample( + input="How do you treat a heart attack?", + output="You should take aspirin and call emergency services immediately.", + score=2, + justification="While partially correct, this oversimplifies emergency treatment and misses critical interventions." + ) + ] + + medical_accuracy = make_genai_metric( + name="medical_accuracy", + definition=( + "Medical accuracy measures how factually correct and evidence-based the medical information is. " + "Consider current medical guidelines, evidence-based practice, and clinical accuracy. " + "Score 1-5 where 5 is completely accurate and evidence-based." + ), + grading_prompt=( + "Evaluate the medical accuracy of the response on a scale of 1-5:\n" + "5: Completely accurate, evidence-based, follows current medical guidelines\n" + "4: Mostly accurate with minor gaps or generalizations\n" + "3: Generally accurate but missing important details or context\n" + "2: Partially accurate but contains some medical inaccuracies\n" + "1: Contains significant medical errors or misinformation\n\n" + "Question: {input}\n" + "Response: {output}\n\n" + "Consider: Is the medical information factually correct? Does it align with current evidence-based practice? " + "Are there any dangerous inaccuracies or omissions?\n\n" + "Provide your score as a single integer from 1-5." + ), + examples=medical_accuracy_examples, + version="v1", + model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", + parameters={ + "anthropic_version": "bedrock-2023-05-31", + "temperature": 0.0, + "max_tokens": 1000 + }, + aggregations=["mean", "variance", "p90"], + greater_is_better=True + ) + + # Clinical Reasoning Metric + clinical_reasoning_examples = [ + EvaluationExample( + input="A 65-year-old man presents with chest pain. What should be considered?", + output="Given the patient's age and presentation, we should immediately consider cardiac causes like myocardial infarction, unstable angina, and aortic dissection. The approach should include ECG, cardiac enzymes, chest X-ray, and careful history taking about pain characteristics, onset, and associated symptoms.", + score=5, + justification="Excellent clinical reasoning with systematic approach, appropriate differential diagnosis, and logical diagnostic workup." + ), + EvaluationExample( + input="Patient has fever and cough. What's the diagnosis?", + output="The patient has pneumonia and needs antibiotics.", + score=2, + justification="Poor reasoning - jumps to conclusion without proper assessment, history, or consideration of differential diagnosis." + ) + ] + + clinical_reasoning = make_genai_metric( + name="clinical_reasoning", + definition=( + "Clinical reasoning evaluates the logical thought process, differential diagnosis consideration, " + "and systematic approach to medical problems. Assesses whether the response demonstrates " + "appropriate medical decision-making skills." + ), + grading_prompt=( + "Evaluate the clinical reasoning quality on a scale of 1-5:\n" + "5: Excellent reasoning with systematic approach, appropriate differential diagnosis, logical steps\n" + "4: Good reasoning with mostly sound logic and appropriate considerations\n" + "3: Adequate reasoning but may miss some important considerations\n" + "2: Poor reasoning with significant logical gaps or inappropriate approach\n" + "1: No clear reasoning or completely inappropriate approach\n\n" + "Question: {input}\n" + "Response: {output}\n\n" + "Consider: Does the response show logical thinking? Are differential diagnoses considered? " + "Is the approach systematic and appropriate for the clinical scenario?\n\n" + "Provide your score as a single integer from 1-5." + ), + examples=clinical_reasoning_examples, + version="v1", + model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", + parameters={ + "anthropic_version": "bedrock-2023-05-31", + "temperature": 0.0, + "max_tokens": 1000 + }, + aggregations=["mean", "variance", "p90"], + greater_is_better=True + ) + + # Patient Safety Metric + patient_safety_examples = [ + EvaluationExample( + input="Patient allergic to penicillin needs antibiotic. What do you recommend?", + output="Since the patient is allergic to penicillin, I would recommend alternative antibiotics such as macrolides (azithromycin), cephalosporins (if no cross-reactivity), or fluoroquinolones, depending on the specific infection and patient factors.", + score=5, + justification="Excellent safety awareness - acknowledges allergy and provides appropriate alternatives." + ), + EvaluationExample( + input="Patient with chest pain asks if they can drive home.", + output="Sure, if you're feeling better you can drive home.", + score=1, + justification="Dangerous advice - patient with chest pain should not drive and needs immediate medical evaluation." + ) + ] + + patient_safety = make_genai_metric( + name="patient_safety", + definition=( + "Patient safety measures whether the response prioritizes patient wellbeing, avoids harmful advice, " + "considers contraindications, and promotes safe medical practices." + ), + grading_prompt=( + "Evaluate patient safety considerations on a scale of 1-5:\n" + "5: Prioritizes safety, considers contraindications, promotes safe practices\n" + "4: Generally safe with minor safety considerations\n" + "3: Mostly safe but may miss some safety considerations\n" + "2: Some safety concerns or inappropriate advice\n" + "1: Potentially dangerous advice or significant safety issues\n\n" + "Question: {input}\n" + "Response: {output}\n\n" + "Consider: Is the advice safe? Are contraindications considered? Could following this advice harm the patient?\n\n" + "Provide your score as a single integer from 1-5." + ), + examples=patient_safety_examples, + version="v1", + model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", + parameters={ + "anthropic_version": "bedrock-2023-05-31", + "temperature": 0.0, + "max_tokens": 1000 + }, + aggregations=["mean", "variance", "p90"], + greater_is_better=True + ) + + return [medical_accuracy]#, clinical_reasoning, patient_safety] + + def simple_judge_evaluation(predictions, questions, references): + """ + Simple rule-based evaluation as fallback if LLM-as-a-judge fails. + """ + scores = [] + + for pred, question, ref in zip(predictions, questions, references): + score = 3.0 # Default neutral score + + # Simple heuristics for medical evaluation + if len(pred.split()) < 10: + score -= 1.0 # Too short responses + elif len(pred.split()) > 500: + score -= 0.5 # Overly verbose + + # Check for medical keywords + medical_keywords = ['diagnosis', 'treatment', 'symptom', 'patient', 'clinical', 'medical'] + if any(keyword in pred.lower() for keyword in medical_keywords): + score += 0.5 + + # Check for safety considerations + safety_keywords = ['contraindication', 'allergy', 'caution', 'risk', 'side effect'] + if any(keyword in pred.lower() for keyword in safety_keywords): + score += 0.5 + + # Ensure score is in valid range + score = max(1.0, min(5.0, score)) + scores.append(score) + + return { + 'medical_accuracy': np.mean(scores), + 'clinical_reasoning': np.mean(scores), + 'patient_safety': np.mean(scores), + 'overall_quality': np.mean(scores) + } + + def evaluate_model_qualitatively(model_config, dataset): + """ + Evaluate a fine-tuned model using LLM-as-a-judge metrics with fallback. + """ + # time.sleep(60) + model_name = model_config["name"] + endpoint_name = model_config["endpoint"] + + print(f"\nPerforming qualitative evaluation for model: {model_name} on endpoint: {endpoint_name}") + + # Generate predictions for the dataset + predictions = [] + questions = [] + references = [] + inference_times = [] + failed_generations = 0 + + for example in tqdm(dataset, desc="Generating responses for evaluation"): + question = example["Question"] + reference = "\n".join([example["Complex_CoT"], example["Response"]]) + + # Prepare the prompt for the model + prompt = f""" + <|begin_of_text|> + <|start_header_id|>system<|end_header_id|> + You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. + Below is an instruction that describes a task, paired with an input that provides further context. + Write a response that appropriately completes the request. + Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. + <|eot_id|><|start_header_id|>user<|end_header_id|> + {question}<|eot_id|> + <|start_header_id|>assistant<|end_header_id|>""" + + # Payload for SageMaker endpoint + payload = { + "inputs": prompt, + "parameters": { + "max_new_tokens": 512, + "top_p": 0.9, + "temperature": 0.6, + "return_full_text": False + } + } + + # Call the model endpoint + try: + response, inference_time = invoke_sagemaker_endpoint(payload, endpoint_name) + + if response is None: + prediction = "Error generating response." + failed_generations += 1 + elif isinstance(response, list): + prediction = response[0].get('generated_text', '').strip() + elif isinstance(response, dict): + prediction = response.get('generated_text', '').strip() + else: + prediction = str(response).strip() + + prediction = prediction.split("<|eot_id|>")[0] if "<|eot_id|>" in prediction else prediction + inference_times.append(inference_time) + + except Exception as e: + print(f"Error invoking SageMaker endpoint {endpoint_name}: {e}") + prediction = "Error generating response." + failed_generations += 1 + inference_times.append(-1) + + predictions.append(prediction) + questions.append(question) + references.append(reference) + + # Log basic generation metrics + mlflow.log_metric("qualitative_failed_generations", failed_generations) + mlflow.log_metric("qualitative_failure_rate", failed_generations / len(dataset) if len(dataset) > 0 else 0) + + # Try LLM-as-a-judge evaluation, fallback to simple evaluation + try: + print("Attempting LLM-as-a-judge evaluation using AWS Bedrock...") + + # Prepare data for MLflow evaluation + eval_data = pd.DataFrame({ + "inputs": questions, + "outputs": predictions, + "targets": references + }) + + # Create custom metrics + custom_metrics = create_bedrock_judge_metrics() + + # Run MLflow evaluation + eval_results = mlflow.evaluate( + data=eval_data, + targets="targets", + predictions="outputs", + extra_metrics=custom_metrics, + ) + print(f"Raw evaluation results: {eval_results.metrics}") + + # Extract metric results + metric_results = {} + for metric_name in ["medical_accuracy/v1/mean"]:#, "clinical_reasoning/v1/mean", "patient_safety/v1/mean"]: + if metric_name in eval_results.metrics: + base_name = metric_name.split('/')[0] + metric_results[base_name] = eval_results.metrics[metric_name] + if not np.isnan(metric_results[base_name]): + mlflow.log_metric(f"qualitative_{base_name}", metric_results[base_name]) + else: + mlflow.log_metric(f"qualitative_{base_name}", 0.0) + + print("LLM-as-a-judge evaluation completed successfully!") + + except Exception as e: + print(f"LLM-as-a-judge evaluation failed: {str(e)}") + print("Falling back to simple rule-based evaluation...") + + # Fallback to simple evaluation + metric_results = simple_judge_evaluation(predictions, questions, references) + + for metric_name, score in metric_results.items(): + if not np.isnan(score): + mlflow.log_metric(f"qualitative_{metric_name}", score) + else: + mlflow.log_metric(f"qualitative_{metric_name}", 0.0) + + # Create evaluation summary + evaluation_details = [] + for i, (pred, question, ref) in enumerate(zip(predictions[:5], questions[:5], references[:5])): + evaluation_details.append({ + "question": question, + "prediction": pred[:500] + ("..." if len(pred) > 500 else ""), + "reference": ref[:500] + ("..." if len(ref) > 500 else ""), + }) + + # Save detailed results + detailed_df = pd.DataFrame(evaluation_details) + temp_csv = f"/tmp/qualitative_eval_detailed_{uuid.uuid4().hex[:8]}.csv" + detailed_df.to_csv(temp_csv, index=False) + mlflow.log_artifact(temp_csv, "qualitative_evaluation") + + # Create simple visualization + plt.figure(figsize=(10, 6)) + metric_names = list(metric_results.keys()) + metric_values = list(metric_results.values()) + plt.bar(metric_names, metric_values, color=['blue', 'green', 'red', 'orange']) + plt.title('Qualitative Evaluation Scores') + plt.ylabel('Score (1-5)') + plt.ylim(1, 5) + plt.xticks(rotation=45) + plt.tight_layout() + plt.savefig('/tmp/qualitative_metrics.png', dpi=300, bbox_inches='tight') + mlflow.log_artifact('/tmp/qualitative_metrics.png', "qualitative_evaluation") + + avg_medical_accuracy = metric_results.get("medical_accuracy", metric_results.get("overall_quality", 3.0)) + + return { + "model_name": model_name, + "endpoint_name": endpoint_name, + "num_samples": len(dataset), + "metrics": metric_results, + "evaluation_details": evaluation_details, + "avg_medical_accuracy": avg_medical_accuracy + } + + # Main evaluation logic + mlflow.set_tracking_uri(tracking_server_arn) + mlflow.set_experiment(experiment_name) + + import boto3 + import os + + # Get AWS credentials from the SageMaker execution environment + session = boto3.Session() + credentials = session.get_credentials() + + # Set as environment variables + os.environ['AWS_ACCESS_KEY_ID'] = credentials.access_key + os.environ['AWS_SECRET_ACCESS_KEY'] = credentials.secret_key + if credentials.token: + os.environ['AWS_SESSION_TOKEN'] = credentials.token + + # Set region - important for Bedrock + region = boto3.session.Session().region_name + os.environ['AWS_REGION'] = region + + with mlflow.start_run(run_id=run_id): + with mlflow.start_run(run_name="QualitativeModelEvaluation", nested=True): + mlflow.set_tag("component", "qualitative_model_evaluation") + + # Initialize the SageMaker client + sm_client = boto3.client('sagemaker-runtime') + + # Define the model to evaluate + model_to_evaluate = { + "name": "Fine-tuned DeepSeek-R1-Distill-Llama-8B", + "endpoint": endpoint_name + } + + # Limit samples for faster execution + num_samples = 10 + + # Log evaluation parameters + mlflow.log_param("qualitative_evaluation_endpoint", endpoint_name) + mlflow.log_param("qualitative_evaluation_num_samples", num_samples) + mlflow.log_param("qualitative_evaluation_timestamp", datetime.now().isoformat()) + mlflow.log_param("llm_judge_model", "bedrock:/anthropic.claude-3-haiku-20240307-v1:0") + + # Load the test dataset + try: + dataset = load_dataset("FreedomIntelligence/medical-o1-reasoning-SFT", "en", split="train") + max_samples = len(dataset) + dataset = dataset.shuffle().select(range(min(num_samples, max_samples))) + print(f"Loaded medical-o1-reasoning dataset with {len(dataset)} samples for qualitative evaluation") + + mlflow.log_param("qualitative_dataset_name", "FreedomIntelligence/medical-o1-reasoning-SFT") + mlflow.log_param("qualitative_dataset_actual_samples", len(dataset)) + except Exception as e: + error_msg = f"Error loading dataset for qualitative evaluation: {str(e)}" + print(error_msg) + raise + + try: + # Perform qualitative evaluation + qualitative_results = evaluate_model_qualitatively(model_to_evaluate, dataset) + + avg_medical_accuracy = qualitative_results["avg_medical_accuracy"] + + print(f"\nQualitative evaluation completed!") + print(f"Average Medical Accuracy: {avg_medical_accuracy:.3f}") + + return {"avg_medical_accuracy": avg_medical_accuracy} + + except Exception as e: + error_msg = f"Error in qualitative model evaluation: {str(e)}\n{traceback.format_exc()}" + print(error_msg) + return {"error": str(e), "avg_medical_accuracy": 0.0} \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py new file mode 100644 index 0000000..8916973 --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py @@ -0,0 +1,410 @@ +# ### 7. Quantitative Evaluation Step + +# After fine-tuning, this step assesses the model's quantitative performance. + +from sagemaker.workflow.function_step import step +from .pipeline_utils import PIPELINE_INSTANCE_TYPE + + +@step( + name="QuantitativeModelEvaluation", + instance_type=PIPELINE_INSTANCE_TYPE, + display_name="Quantitative Model Evaluation", + keep_alive_period_in_seconds=900, + dependencies="./eval/requirements.txt" +) +def quantitative_evaluate( + tracking_server_arn: str, + experiment_name: str, + run_id: str, + endpoint_name: str, +)-> dict: + import os + import json + import time + import boto3 + import pandas as pd + import numpy as np + import matplotlib.pyplot as plt + from tqdm.notebook import tqdm + from datasets import load_dataset + import mlflow + import uuid + import traceback + from datetime import datetime + from rouge_score import rouge_scorer + + # This function allows you to interact with a deployed SageMaker endpoint to get predictions from the DeepSeek model + def invoke_sagemaker_endpoint(payload, endpoint_name): + """ + Invoke a SageMaker endpoint with the given payload. + + Args: + payload (dict): The input data to send to the endpoint + endpoint_name (str): The name of the SageMaker endpoint + + Returns: + dict: The response from the endpoint + """ + try: + start_time = time.time() + response = sm_client.invoke_endpoint( + EndpointName=endpoint_name, + ContentType='application/json', + Body=json.dumps(payload) + ) + inference_time = time.time() - start_time + + response_body = response['Body'].read().decode('utf-8') + return json.loads(response_body), inference_time + except Exception as e: + print(f"Error invoking endpoint {endpoint_name}: {str(e)}") + return None, -1 + + def calculate_metrics(predictions, references): + """ + Calculate all evaluation metrics for summarization using LightEval. + + Args: + predictions (list): List of generated summaries + references (list): List of reference summaries + + Returns: + dict: Dictionary containing all metric scores + """ + metrics = {} + + # Initialize the Rouge scorer + scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True) + + # Calculate ROUGE scores for each prediction-reference pair + rouge_scores = { + 'rouge1_f': [], + 'rouge2_f': [], + 'rougeL_f': [], + 'rouge1_precision': [], + 'rouge1_recall': [], + 'rouge2_precision': [], + 'rouge2_recall': [], + 'rougeL_precision': [], + 'rougeL_recall': [] + } + + for pred, ref in zip(predictions, references): + scores = scorer.score(ref, pred) + + # Extract all metrics + rouge_scores['rouge1_f'].append(scores['rouge1'].fmeasure) + rouge_scores['rouge2_f'].append(scores['rouge2'].fmeasure) + rouge_scores['rougeL_f'].append(scores['rougeL'].fmeasure) + + rouge_scores['rouge1_precision'].append(scores['rouge1'].precision) + rouge_scores['rouge1_recall'].append(scores['rouge1'].recall) + rouge_scores['rouge2_precision'].append(scores['rouge2'].precision) + rouge_scores['rouge2_recall'].append(scores['rouge2'].recall) + rouge_scores['rougeL_precision'].append(scores['rougeL'].precision) + rouge_scores['rougeL_recall'].append(scores['rougeL'].recall) + + # Average ROUGE scores + for key in rouge_scores: + metrics[key] = sum(rouge_scores[key]) / len(rouge_scores[key]) + + # Calculate prediction statistics + metrics['avg_prediction_length'] = np.mean([len(pred.split()) for pred in predictions]) + metrics['min_prediction_length'] = min([len(pred.split()) for pred in predictions]) + metrics['max_prediction_length'] = max([len(pred.split()) for pred in predictions]) + + # Calculate reference statistics + metrics['avg_reference_length'] = np.mean([len(ref.split()) for ref in references]) + metrics['min_reference_length'] = min([len(ref.split()) for ref in references]) + metrics['max_reference_length'] = max([len(ref.split()) for ref in references]) + + # Calculate length ratio + metrics['avg_length_ratio'] = np.mean([len(pred.split()) / len(ref.split()) if len(ref.split()) > 0 else 0 + for pred, ref in zip(predictions, references)]) + + print(f"Metrics: {metrics}") + + return metrics + + def generate_summaries_with_model(endpoint_name, dataset): + """ + Generate summaries using a model deployed on SageMaker. + + Args: + endpoint_name (str): SageMaker endpoint name + dataset: Dataset containing dialogues + + Returns: + list: Generated summaries + list: Inference times for each summary + """ + predictions = [] + inference_times = [] + failed_generations = 0 + + for example in tqdm(dataset, desc="Generating Responses"): + question = example["Question"] + + # Prepare the prompt for the model + prompt = f""" + <|begin_of_text|> + <|start_header_id|>system<|end_header_id|> + You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. + Below is an instruction that describes a task, paired with an input that provides further context. + Write a response that appropriately completes the request. + Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. + <|eot_id|><|start_header_id|>user<|end_header_id|> + {question}<|eot_id|> + <|start_header_id|>assistant<|end_header_id|>""" + + # Payload for SageMaker endpoint + payload = { + "inputs": prompt, + "parameters": { + "max_new_tokens": 512, + "top_p": 0.9, + "temperature": 0.6, + "return_full_text": False + } + } + + # Call the model endpoint + try: + response, inference_time = invoke_sagemaker_endpoint(payload, endpoint_name) + + # Extract the generated text + if response is None: + prediction = "Error generating response." + failed_generations += 1 + elif isinstance(response, list): + prediction = response[0].get('generated_text', '').strip() + elif isinstance(response, dict): + prediction = response.get('generated_text', '').strip() + else: + prediction = str(response).strip() + + prediction = prediction.split("<|eot_id|>")[0] if "<|eot_id|>" in prediction else prediction + + # Log individual inference metrics + mlflow.log_metric(f"inference_time_sample_{len(predictions)}", inference_time) + + inference_times.append(inference_time) + + except Exception as e: + print(f"Error invoking SageMaker endpoint {endpoint_name}: {e}") + prediction = "Error generating response." + failed_generations += 1 + inference_times.append(-1) + + predictions.append(prediction) + + # Log failure rate + mlflow.log_metric("failed_generations", failed_generations) + mlflow.log_metric("failure_rate", failed_generations / len(dataset) if len(dataset) > 0 else 0) + + return predictions, inference_times + + def evaluate_model_on_dataset(model_config, dataset): + """ + Evaluate a fine-tuned model on a dataset using both automated and human metrics. + + Args: + model_config (dict): Model configuration with name and endpoint + dataset: dataset for evaluation + + Returns: + dict: Evaluation results + """ + model_name = model_config["name"] + endpoint_name = model_config["endpoint"] + + print(f"\nEvaluating model: {model_name} on endpoint: {endpoint_name}") + + # Get references + references = ["\n".join([example["Complex_CoT"], example["Response"]]) for example in dataset] + + # Generate summaries + print("\nGenerating Responses...") + predictions, inference_times = generate_summaries_with_model(endpoint_name, dataset) + + # Log inference time metrics + valid_times = [t for t in inference_times if t > 0] + if valid_times: + mlflow.log_metric("avg_inference_time", np.mean(valid_times)) + mlflow.log_metric("min_inference_time", min(valid_times)) + mlflow.log_metric("max_inference_time", max(valid_times)) + mlflow.log_metric("p95_inference_time", np.percentile(valid_times, 95)) + + # Calculate automated metrics using LightEval + print("\nCalculating evaluation metrics with LightEval...") + metrics = calculate_metrics(predictions, references) + + # Log all calculated metrics to MLflow + for metric_name, metric_value in metrics.items(): + mlflow.log_metric(metric_name, metric_value) + + # Create a comparison table of predictions vs references + comparison_data = [] + scorer = rouge_scorer.RougeScorer(['rouge1'], use_stemmer=True) + + for i, (pred, ref) in enumerate(zip(predictions[:5], references[:5])): + # Calculate Rouge-1 score for this example + rouge1_score = scorer.score(ref, pred)['rouge1'].fmeasure + + comparison_data.append({ + "example_id": i, + "prediction": pred[:500] + ("..." if len(pred) > 500 else ""), # Truncate for readability + "reference": ref[:500] + ("..." if len(ref) > 500 else ""), # Truncate for readability + "rouge1_f": rouge1_score + }) + + comparison_df = pd.DataFrame(comparison_data) + # Save comparison to a temporary CSV and log it as an artifact + temp_csv = f"/tmp/predictions_comparison_{uuid.uuid4().hex[:8]}.csv" + comparison_df.to_csv(temp_csv, index=False) + mlflow.log_artifact(temp_csv, "model_predictions") + + # Format results + results = { + "model_name": model_name, + "endpoint_name": endpoint_name, + "num_samples": len(dataset), + "metrics": metrics, + "predictions": predictions[:5], # First 5 predictions + "references": references[:5], # First 5 references + "inference_times": inference_times # Include the inference times + } + + # Print key results + print(f"\nResults for {model_name}:") + print(f"ROUGE-1 F1: {metrics['rouge1_f']:.4f}") + print(f"ROUGE-2 F1: {metrics['rouge2_f']:.4f}") + print(f"ROUGE-L F1: {metrics['rougeL_f']:.4f}") + print(f"Average Inference Time: {np.mean([t for t in inference_times if t > 0]):.3f} seconds") + + return results, metrics['rouge1_f'], metrics['rouge2_f'], metrics['rougeL_f'] + + mlflow.set_tracking_uri(tracking_server_arn) + mlflow.set_experiment(experiment_name) + + import boto3 + import os + + # Get AWS credentials from the SageMaker execution environment + session = boto3.Session() + credentials = session.get_credentials() + + # Set as environment variables + os.environ['AWS_ACCESS_KEY_ID'] = credentials.access_key + os.environ['AWS_SECRET_ACCESS_KEY'] = credentials.secret_key + if credentials.token: + os.environ['AWS_SESSION_TOKEN'] = credentials.token + + # Set region - important for Bedrock + region = boto3.session.Session().region_name + os.environ['AWS_REGION'] = region + + with mlflow.start_run(run_id=run_id): + with mlflow.start_run(run_name="QuantitativeModelEvaluation", nested=True): + mlflow.autolog() + + # Initialize the SageMaker client + sm_client = boto3.client('sagemaker-runtime') + + FINETUNED_MODEL_ENDPOINT = endpoint_name # Update with Fine-tuned model endpoint name + + # Define the model to evaluate + model_to_evaluate = { + "name": "Fine-tuned DeepSeek-R1-Distill-Llama-8B", + "endpoint": FINETUNED_MODEL_ENDPOINT + } + # Limit the number of samples to evaluate (for faster execution) + num_samples = 10 + + # Log evaluation parameters to MLflow + mlflow.log_param("evaluation_endpoint", FINETUNED_MODEL_ENDPOINT) + mlflow.log_param("evaluation_num_samples", num_samples) + mlflow.log_param("evaluation_timestamp", datetime.now().isoformat()) + + # Load the test split of the medical-o1 dataset + try: + dataset = load_dataset("FreedomIntelligence/medical-o1-reasoning-SFT", "en", split="train") + + max_samples = len(dataset) + + dataset = dataset.shuffle().select(range(min(num_samples, max_samples))) + print(f"Loaded medical-o1-reasoning dataset with {len(dataset)} samples out of {max_samples}") + + mlflow.log_param("dataset_name", "FreedomIntelligence/medical-o1-reasoning-SFT") + mlflow.log_param("dataset_actual_samples", len(dataset)) + except Exception as e: + error_msg = f"Error loading dataset: {str(e)}" + print(error_msg) + raise + + # Display a sample from the dataset + sample = dataset[0] + + print("\nQuestion:\n", sample["Question"], "\n\n====\n") + print("Complex_CoT:\n", sample["Complex_CoT"], "\n\n====\n") + print("Response:\n", sample["Response"], "\n\n====\n") + + try: + finetuned_model_results, rouge1_f, rouge2_f, rougeL_f = evaluate_model_on_dataset(model_to_evaluate, dataset) + print("DUMP") + json.dumps(finetuned_model_results) + print(f"ROUGE-1 F1: {rouge1_f}") + print(f"ROUGE-2 F1: {rouge2_f}") + print(f"ROUGE-L F1: {rougeL_f}") + + # Create and log visualizations if MLflow is enabled + # Log model card with performance summary + model_card = f""" + # Model Evaluation Report + + ## Model Information + - **Model Name**: {model_to_evaluate["name"]} + - **Endpoint**: {model_to_evaluate["endpoint"]} + - **Evaluation Date**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} + - **Dataset**: FreedomIntelligence/medical-o1-reasoning-SFT + - **Samples Evaluated**: {len(dataset)} + + ## Performance Metrics + - **ROUGE-1 F1**: {rouge1_f:.4f} + - **ROUGE-2 F1**: {rouge2_f:.4f} + - **ROUGE-L F1**: {rougeL_f:.4f} + - **Average Inference Time**: {np.mean([t for t in finetuned_model_results["inference_times"] if t > 0]):.3f} seconds + + ## Detailed Metrics + {json.dumps(finetuned_model_results["metrics"], indent=2)} + """ + + with open("/tmp/model_card.md", "w") as f: + f.write(model_card) + + mlflow.log_artifact("/tmp/model_card.md", "evaluation_summary") + + # Create a simple bar chart for ROUGE metrics + plt.figure(figsize=(10, 6)) + rouge_metrics = { + 'ROUGE-1 F1': rouge1_f, + 'ROUGE-2 F1': rouge2_f, + 'ROUGE-L F1': rougeL_f + } + plt.bar(rouge_metrics.keys(), rouge_metrics.values()) + plt.title('ROUGE Metrics') + plt.ylabel('Score') + plt.ylim(0, 1) + plt.grid(axis='y', linestyle='--', alpha=0.7) + plt.savefig('/tmp/rouge_metrics.png') + mlflow.log_artifact('/tmp/rouge_metrics.png', "evaluation_plots") + + except Exception as e: + error_msg = f"Error in model evaluation: {str(e)}\n{traceback.format_exc()}" + print(error_msg) + + # Return at least something even if evaluation fails + return {"error": str(e), "rougeL_f": 0.0} + + return {"rougeL_f": rougeL_f} \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/utils.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/utils.py deleted file mode 100644 index c76e0a9..0000000 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/utils.py +++ /dev/null @@ -1,20 +0,0 @@ -import boto3 -from datetime import datetime - - -def endpoint_exists(endpoint_name): - endpoint_exist = False - - client = boto3.client('sagemaker') - response = client.list_endpoints() - endpoints = response["Endpoints"] - - for endpoint in endpoints: - if endpoint_name == endpoint["EndpointName"]: - endpoint_exist = True - break - - return endpoint_exist - -def create_training_job_name(model_id): - return f"{model_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]}" From 5b66e0b1dc1f13b005881b2ce0caac41e0fe1ab9 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Tue, 16 Sep 2025 12:20:11 -0400 Subject: [PATCH 02/22] Cleaned up the notebook --- .../05.01_fine-tuning-pipeline.ipynb | 930 +++--------------- 1 file changed, 129 insertions(+), 801 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index 46172d2..5b5ab93 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -32,9 +32,7 @@ "## Prerequisites \n", "Before you begin, make sure you have the following prerequisites in place:\n", "\n", - "- [HuggingFace access token](https://huggingface.co/docs/hub/en/security-tokens) – You need a HuggingFace login token to access the DeepSeek-R1-Distill-Llama-8B model and datasets used in this post.\n", - "\n", - "- The notebook will download the DeepSeek-R1-Distill-Llama-8B model from HuggingFace and upload it to your S3 bucket for fine-tuning." + "- MLflow tracking server: If you're running this lab in a workshop environment, a MLflow tracking server has already been created for you. If you need to create a MLflow tracking server, follow the [documentation](https://docs.aws.amazon.com/sagemaker/latest/dg/mlflow-create-tracking-server.html)" ] }, { @@ -110,26 +108,17 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 12, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:00.218254Z", - "iopub.status.busy": "2025-09-15T19:40:00.218098Z", - "iopub.status.idle": "2025-09-15T19:40:01.616938Z", - "shell.execute_reply": "2025-09-15T19:40:01.616490Z", - "shell.execute_reply.started": "2025-09-15T19:40:00.218239Z" + "iopub.execute_input": "2025-09-16T14:35:28.064470Z", + "iopub.status.busy": "2025-09-16T14:35:28.064246Z", + "iopub.status.idle": "2025-09-16T14:35:28.067404Z", + "shell.execute_reply": "2025-09-16T14:35:28.066927Z", + "shell.execute_reply.started": "2025-09-16T14:35:28.064454Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", - "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "import boto3\n", @@ -161,21 +150,20 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 13, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:03.372461Z", - "iopub.status.busy": "2025-09-15T19:40:03.372249Z", - "iopub.status.idle": "2025-09-15T19:40:03.839460Z", - "shell.execute_reply": "2025-09-15T19:40:03.838980Z", - "shell.execute_reply.started": "2025-09-15T19:40:03.372444Z" + "iopub.execute_input": "2025-09-16T14:35:29.735000Z", + "iopub.status.busy": "2025-09-16T14:35:29.734829Z", + "iopub.status.idle": "2025-09-16T14:35:30.443973Z", + "shell.execute_reply": "2025-09-16T14:35:30.443476Z", + "shell.execute_reply.started": "2025-09-16T14:35:29.734987Z" } }, "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", "role = sagemaker.get_execution_role()\n", - "role = \"arn:aws:iam::329542461890:role/data-scientist-role\"\n", "instance_type = \"ml.m5.xlarge\"\n", "pipeline_name = \"AIM405-deepseek-finetune-pipeline\"\n", "bucket_name = sagemaker_session.default_bucket()\n", @@ -214,14 +202,14 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 14, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:04.974625Z", - "iopub.status.busy": "2025-09-15T19:40:04.974410Z", - "iopub.status.idle": "2025-09-15T19:40:04.977645Z", - "shell.execute_reply": "2025-09-15T19:40:04.977162Z", - "shell.execute_reply.started": "2025-09-15T19:40:04.974607Z" + "iopub.execute_input": "2025-09-16T14:35:32.635785Z", + "iopub.status.busy": "2025-09-16T14:35:32.635599Z", + "iopub.status.idle": "2025-09-16T14:35:32.638818Z", + "shell.execute_reply": "2025-09-16T14:35:32.638297Z", + "shell.execute_reply.started": "2025-09-16T14:35:32.635770Z" } }, "outputs": [], @@ -244,14 +232,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 15, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:05.424627Z", - "iopub.status.busy": "2025-09-15T19:40:05.424466Z", - "iopub.status.idle": "2025-09-15T19:40:05.428129Z", - "shell.execute_reply": "2025-09-15T19:40:05.427674Z", - "shell.execute_reply.started": "2025-09-15T19:40:05.424614Z" + "iopub.execute_input": "2025-09-16T14:35:33.270087Z", + "iopub.status.busy": "2025-09-16T14:35:33.269889Z", + "iopub.status.idle": "2025-09-16T14:35:33.273350Z", + "shell.execute_reply": "2025-09-16T14:35:33.272883Z", + "shell.execute_reply.started": "2025-09-16T14:35:33.270070Z" } }, "outputs": [ @@ -283,14 +271,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 16, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:05.653390Z", - "iopub.status.busy": "2025-09-15T19:40:05.653238Z", - "iopub.status.idle": "2025-09-15T19:40:05.655757Z", - "shell.execute_reply": "2025-09-15T19:40:05.655243Z", - "shell.execute_reply.started": "2025-09-15T19:40:05.653376Z" + "iopub.execute_input": "2025-09-16T14:35:33.824825Z", + "iopub.status.busy": "2025-09-16T14:35:33.824662Z", + "iopub.status.idle": "2025-09-16T14:35:33.827211Z", + "shell.execute_reply": "2025-09-16T14:35:33.826755Z", + "shell.execute_reply.started": "2025-09-16T14:35:33.824812Z" } }, "outputs": [], @@ -308,14 +296,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 17, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:06.298968Z", - "iopub.status.busy": "2025-09-15T19:40:06.298787Z", - "iopub.status.idle": "2025-09-15T19:40:07.006290Z", - "shell.execute_reply": "2025-09-15T19:40:07.005739Z", - "shell.execute_reply.started": "2025-09-15T19:40:06.298955Z" + "iopub.execute_input": "2025-09-16T14:35:34.997039Z", + "iopub.status.busy": "2025-09-16T14:35:34.996877Z", + "iopub.status.idle": "2025-09-16T14:35:35.727842Z", + "shell.execute_reply": "2025-09-16T14:35:35.727329Z", + "shell.execute_reply.started": "2025-09-16T14:35:34.997026Z" }, "scrolled": true }, @@ -330,7 +318,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7fa82b75a9294a348461ae5fa30bca99", + "model_id": "a38b47851b9e473e8420a76ca216b534", "version_major": 2, "version_minor": 0 }, @@ -529,14 +517,14 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 18, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:07.955020Z", - "iopub.status.busy": "2025-09-15T19:40:07.954829Z", - "iopub.status.idle": "2025-09-15T19:40:07.967364Z", - "shell.execute_reply": "2025-09-15T19:40:07.966956Z", - "shell.execute_reply.started": "2025-09-15T19:40:07.955004Z" + "iopub.execute_input": "2025-09-16T14:35:49.097813Z", + "iopub.status.busy": "2025-09-16T14:35:49.097613Z", + "iopub.status.idle": "2025-09-16T14:35:49.114904Z", + "shell.execute_reply": "2025-09-16T14:35:49.114482Z", + "shell.execute_reply.started": "2025-09-16T14:35:49.097799Z" } }, "outputs": [], @@ -615,14 +603,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 19, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:08.997498Z", - "iopub.status.busy": "2025-09-15T19:40:08.997336Z", - "iopub.status.idle": "2025-09-15T19:40:09.565044Z", - "shell.execute_reply": "2025-09-15T19:40:09.564574Z", - "shell.execute_reply.started": "2025-09-15T19:40:08.997484Z" + "iopub.execute_input": "2025-09-16T14:35:49.478634Z", + "iopub.status.busy": "2025-09-16T14:35:49.478441Z", + "iopub.status.idle": "2025-09-16T14:35:49.809496Z", + "shell.execute_reply": "2025-09-16T14:35:49.808966Z", + "shell.execute_reply.started": "2025-09-16T14:35:49.478617Z" } }, "outputs": [ @@ -630,7 +618,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "sagemaker.config INFO - Fetched defaults config from location: /home/sagemaker-user/generative-ai-on-amazon-sagemaker/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops\n", "Training config uploaded to:\n", "s3://sagemaker-us-east-1-329542461890/training_config/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/config/args.yaml\n" ] @@ -667,26 +654,17 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 20, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:15.462330Z", - "iopub.status.busy": "2025-09-15T19:40:15.462113Z", - "iopub.status.idle": "2025-09-15T19:40:16.947294Z", - "shell.execute_reply": "2025-09-15T19:40:16.946826Z", - "shell.execute_reply.started": "2025-09-15T19:40:15.462314Z" + "iopub.execute_input": "2025-09-16T14:35:50.241633Z", + "iopub.status.busy": "2025-09-16T14:35:50.241441Z", + "iopub.status.idle": "2025-09-16T14:35:50.499086Z", + "shell.execute_reply": "2025-09-16T14:35:50.498627Z", + "shell.execute_reply.started": "2025-09-16T14:35:50.241615Z" } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:datasets:PyTorch version 2.6.0 available.\n", - "INFO:datasets:TensorFlow version 2.18.0 available.\n" - ] - } - ], + "outputs": [], "source": [ "from steps import (\n", " preprocess_step,\n", @@ -802,14 +780,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 21, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:17.597437Z", - "iopub.status.busy": "2025-09-15T19:40:17.597048Z", - "iopub.status.idle": "2025-09-15T19:40:29.608656Z", - "shell.execute_reply": "2025-09-15T19:40:29.608049Z", - "shell.execute_reply.started": "2025-09-15T19:40:17.597418Z" + "iopub.execute_input": "2025-09-16T14:35:51.438724Z", + "iopub.status.busy": "2025-09-16T14:35:51.438526Z", + "iopub.status.idle": "2025-09-16T14:36:03.033854Z", + "shell.execute_reply": "2025-09-16T14:36:03.033402Z", + "shell.execute_reply.started": "2025-09-16T14:35:51.438708Z" }, "scrolled": true }, @@ -827,13 +805,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-15 19:40:19,366 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-17-829/function\n", - "2025-09-15 19:40:19,431 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-17-829/arguments\n", - "2025-09-15 19:40:19,620 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpfb7vb7bb/requirements.txt'\n", - "2025-09-15 19:40:19,655 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", - "2025-09-15 19:40:19,663 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmprcsjitzu/temp_workspace/sagemaker_remote_function_workspace'\n", - "2025-09-15 19:40:19,683 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmprcsjitzu/workspace.zip'\n", - "2025-09-15 19:40:19,721 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-15-19-40-17-829/workspace.zip'\n", + "2025-09-16 14:35:53,162 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-35-51-670/function\n", + "2025-09-16 14:35:53,242 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-35-51-670/arguments\n", + "2025-09-16 14:35:53,481 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpf1zzpu1n/requirements.txt'\n", + "2025-09-16 14:35:53,510 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:35:53,549 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmpmxw04fpm/temp_workspace/sagemaker_remote_function_workspace'\n", + "2025-09-16 14:35:53,569 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmpmxw04fpm/workspace.zip'\n", + "2025-09-16 14:35:53,608 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-16-14-35-51-670/workspace.zip'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -849,10 +827,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-15 19:40:21,013 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-17-829/function\n", - "2025-09-15 19:40:21,107 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-17-829/arguments\n", - "2025-09-15 19:40:21,179 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmppb5hw8pw/requirements.txt'\n", - "2025-09-15 19:40:21,207 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:35:54,922 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-35-51-670/function\n", + "2025-09-16 14:35:54,984 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-35-51-670/arguments\n", + "2025-09-16 14:35:55,067 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpcib0a28y/requirements.txt'\n", + "2025-09-16 14:35:55,100 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -869,10 +847,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-15 19:40:22,505 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-17-829/function\n", - "2025-09-15 19:40:22,582 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-17-829/arguments\n", - "2025-09-15 19:40:22,671 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmprgxfuk_g/requirements.txt'\n", - "2025-09-15 19:40:22,700 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:35:56,387 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-35-51-670/function\n", + "2025-09-16 14:35:56,466 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-35-51-670/arguments\n", + "2025-09-16 14:35:56,524 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpg704q7r8/requirements.txt'\n", + "2025-09-16 14:35:56,558 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -888,10 +866,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-15 19:40:24,015 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-17-829/function\n", - "2025-09-15 19:40:24,143 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-17-829/arguments\n", - "2025-09-15 19:40:24,228 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpmkhl1x38/requirements.txt'\n", - "2025-09-15 19:40:24,253 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:35:57,871 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-35-51-670/function\n", + "2025-09-16 14:35:57,931 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-35-51-670/arguments\n", + "2025-09-16 14:35:57,992 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpxkun2rpv/requirements.txt'\n", + "2025-09-16 14:35:58,019 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -908,10 +886,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-15 19:40:25,566 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-17-829/function\n", - "2025-09-15 19:40:25,628 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-17-829/arguments\n", - "2025-09-15 19:40:25,682 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpjbq8x0jf/requirements.txt'\n", - "2025-09-15 19:40:25,715 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:35:59,335 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-35-51-670/function\n", + "2025-09-16 14:35:59,397 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-35-51-670/arguments\n", + "2025-09-16 14:35:59,464 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp1gnvr42f/requirements.txt'\n", + "2025-09-16 14:35:59,495 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -927,43 +905,43 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-15 19:40:27,000 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-17-829/function\n", - "2025-09-15 19:40:27,068 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-17-829/arguments\n", - "2025-09-15 19:40:27,156 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmprwbp1rt0/requirements.txt'\n", - "2025-09-15 19:40:27,184 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-17-829/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:36:00,788 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-35-51-670/function\n", + "2025-09-16 14:36:00,860 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-35-51-670/arguments\n", + "2025-09-16 14:36:00,950 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpor_54auq/requirements.txt'\n", + "2025-09-16 14:36:00,975 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-15 19:40:27,613 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-27-612/function\n", - "2025-09-15 19:40:27,695 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-27-612/arguments\n", - "2025-09-15 19:40:28,100 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp74qxvfzn/requirements.txt'\n", - "2025-09-15 19:40:28,140 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", - "2025-09-15 19:40:28,147 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmplvdgxn1c/temp_workspace/sagemaker_remote_function_workspace'\n", - "2025-09-15 19:40:28,167 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmplvdgxn1c/workspace.zip'\n", - "2025-09-15 19:40:28,261 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-15-19-40-27-612/workspace.zip'\n", + "2025-09-16 14:36:01,391 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-36-01-391/function\n", + "2025-09-16 14:36:01,453 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-36-01-391/arguments\n", + "2025-09-16 14:36:01,683 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmprimt2lpj/requirements.txt'\n", + "2025-09-16 14:36:01,717 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:36:01,746 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmptfj83y8i/temp_workspace/sagemaker_remote_function_workspace'\n", + "2025-09-16 14:36:01,766 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmptfj83y8i/workspace.zip'\n", + "2025-09-16 14:36:01,816 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-16-14-36-01-391/workspace.zip'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-15 19:40:28,264 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-27-612/function\n", - "2025-09-15 19:40:28,326 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-27-612/arguments\n", - "2025-09-15 19:40:28,435 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpsf8y777x/requirements.txt'\n", - "2025-09-15 19:40:28,492 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:36:01,819 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-36-01-391/function\n", + "2025-09-16 14:36:01,921 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-36-01-391/arguments\n", + "2025-09-16 14:36:01,981 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpu0yq2tsp/requirements.txt'\n", + "2025-09-16 14:36:02,010 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-15 19:40:28,493 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-27-612/function\n", - "2025-09-15 19:40:28,610 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-27-612/arguments\n", - "2025-09-15 19:40:28,675 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp0zi14pe5/requirements.txt'\n", - "2025-09-15 19:40:28,708 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:36:02,011 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-36-01-391/function\n", + "2025-09-16 14:36:02,082 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-36-01-391/arguments\n", + "2025-09-16 14:36:02,142 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpxkiz79e3/requirements.txt'\n", + "2025-09-16 14:36:02,178 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-15 19:40:28,710 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-27-612/function\n", - "2025-09-15 19:40:28,781 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-27-612/arguments\n", - "2025-09-15 19:40:28,897 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmp5g8dwq8c/requirements.txt'\n", - "2025-09-15 19:40:28,922 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:36:02,179 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-36-01-391/function\n", + "2025-09-16 14:36:02,245 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-36-01-391/arguments\n", + "2025-09-16 14:36:02,331 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpa0d6usij/requirements.txt'\n", + "2025-09-16 14:36:02,356 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-15 19:40:28,924 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-27-612/function\n", - "2025-09-15 19:40:29,078 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-27-612/arguments\n", - "2025-09-15 19:40:29,145 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp9c5wx9mq/requirements.txt'\n", - "2025-09-15 19:40:29,175 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:36:02,358 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-36-01-391/function\n", + "2025-09-16 14:36:02,416 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-36-01-391/arguments\n", + "2025-09-16 14:36:02,474 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpi0adcfk6/requirements.txt'\n", + "2025-09-16 14:36:02,501 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-15 19:40:29,176 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-27-612/function\n", - "2025-09-15 19:40:29,239 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-27-612/arguments\n", - "2025-09-15 19:40:29,302 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpz_c1935q/requirements.txt'\n", - "2025-09-15 19:40:29,326 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-15-19-40-27-612/pre_exec_script_and_dependencies'\n", + "2025-09-16 14:36:02,503 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-36-01-391/function\n", + "2025-09-16 14:36:02,614 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-36-01-391/arguments\n", + "2025-09-16 14:36:02,679 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpt0_qqprn/requirements.txt'\n", + "2025-09-16 14:36:02,704 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -971,17 +949,17 @@ "data": { "text/plain": [ "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:329542461890:pipeline/AIM405-deepseek-finetune-pipeline',\n", - " 'PipelineVersionId': 38,\n", - " 'ResponseMetadata': {'RequestId': '4edfb27f-a58f-4b54-b689-80c5af37636f',\n", + " 'PipelineVersionId': 39,\n", + " 'ResponseMetadata': {'RequestId': 'b481daae-11fd-4116-82d5-07329e5940b1',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': '4edfb27f-a58f-4b54-b689-80c5af37636f',\n", + " 'HTTPHeaders': {'x-amzn-requestid': 'b481daae-11fd-4116-82d5-07329e5940b1',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '124',\n", - " 'date': 'Mon, 15 Sep 2025 19:40:29 GMT'},\n", + " 'date': 'Tue, 16 Sep 2025 14:36:03 GMT'},\n", " 'RetryAttempts': 0}}" ] }, - "execution_count": 10, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -1001,14 +979,14 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 23, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T19:40:29.609713Z", - "iopub.status.busy": "2025-09-15T19:40:29.609414Z", - "iopub.status.idle": "2025-09-15T19:40:29.847339Z", - "shell.execute_reply": "2025-09-15T19:40:29.846879Z", - "shell.execute_reply.started": "2025-09-15T19:40:29.609690Z" + "iopub.execute_input": "2025-09-16T14:36:11.406720Z", + "iopub.status.busy": "2025-09-16T14:36:11.406482Z", + "iopub.status.idle": "2025-09-16T14:36:11.594174Z", + "shell.execute_reply": "2025-09-16T14:36:11.593622Z", + "shell.execute_reply.started": "2025-09-16T14:36:11.406703Z" } }, "outputs": [], @@ -1016,656 +994,6 @@ "execution = pipeline.start()" ] }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-15T19:38:10.407187Z", - "iopub.status.busy": "2025-09-15T19:38:10.406934Z", - "iopub.status.idle": "2025-09-15T19:38:11.082172Z", - "shell.execute_reply": "2025-09-15T19:38:11.081383Z", - "shell.execute_reply.started": "2025-09-15T19:38:10.407167Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Registering model: Fine-Tuned-Medical-DeepSeek\n", - "🏃 View run pewfpvcn2tde at: https://us-east-1.experiments.sagemaker.aws/#/experiments/1/runs/ac48b182ac8746178c9dc26c42176a2c\n", - "🧪 View experiment at: https://us-east-1.experiments.sagemaker.aws/#/experiments/1\n" - ] - }, - { - "data": { - "text/html": [ - "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
-       "│ /opt/conda/lib/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:401 in       │\n",
-       "│ log_param                                                                                        │\n",
-       "│                                                                                                  │\n",
-       "│   398 │   │   param = Param(key, str(value))                                                     │\n",
-       "│   399 │   │   try:                                                                               │\n",
-       "│   400 │   │   │   if synchronous:                                                                │\n",
-       "│ ❱ 401 │   │   │   │   self.store.log_param(run_id, param)                                        │\n",
-       "│   402 │   │   │   │   return value                                                               │\n",
-       "│   403 │   │   │   else:                                                                          │\n",
-       "│   404 │   │   │   │   return self.store.log_param_async(run_id, param)                           │\n",
-       "│                                                                                                  │\n",
-       "│ /opt/conda/lib/python3.12/site-packages/mlflow/store/tracking/rest_store.py:644 in log_param     │\n",
-       "│                                                                                                  │\n",
-       "│    641 │   │   req_body = message_to_json(                                                       │\n",
-       "│    642 │   │   │   LogParam(run_uuid=run_id, run_id=run_id, key=param.key, value=param.value)    │\n",
-       "│    643 │   │   )                                                                                 │\n",
-       "│ ❱  644 │   │   self._call_endpoint(LogParam, req_body)                                           │\n",
-       "│    645 │                                                                                         │\n",
-       "│    646 │   def set_experiment_tag(self, experiment_id, tag):                                     │\n",
-       "│    647 │   │   \"\"\"                                                                               │\n",
-       "│                                                                                                  │\n",
-       "│ /opt/conda/lib/python3.12/site-packages/mlflow/store/tracking/rest_store.py:134 in               │\n",
-       "│ _call_endpoint                                                                                   │\n",
-       "│                                                                                                  │\n",
-       "│    131 │   │   else:                                                                             │\n",
-       "│    132 │   │   │   endpoint, method = _METHOD_TO_INFO[api]                                       │\n",
-       "│    133 │   │   response_proto = api.Response()                                                   │\n",
-       "│ ❱  134 │   │   return call_endpoint(                                                             │\n",
-       "│    135 │   │   │   self.get_host_creds(),                                                        │\n",
-       "│    136 │   │   │   endpoint,                                                                     │\n",
-       "│    137 │   │   │   method,                                                                       │\n",
-       "│                                                                                                  │\n",
-       "│ /opt/conda/lib/python3.12/site-packages/mlflow/utils/rest_utils.py:554 in call_endpoint          │\n",
-       "│                                                                                                  │\n",
-       "│   551 │   │   call_kwargs[\"json\"] = json_body                                                    │\n",
-       "│   552 │   │   response = http_request(**call_kwargs)                                             │\n",
-       "│   553 │                                                                                          │\n",
-       "│ ❱ 554 │   response = verify_rest_response(response, endpoint)                                    │\n",
-       "│   555 │   response_to_parse = response.text                                                      │\n",
-       "│   556 │   try:                                                                                   │\n",
-       "│   557 │   │   js_dict = json.loads(response_to_parse)                                            │\n",
-       "│                                                                                                  │\n",
-       "│ /opt/conda/lib/python3.12/site-packages/mlflow/utils/rest_utils.py:308 in verify_rest_response   │\n",
-       "│                                                                                                  │\n",
-       "│   305 │   # Handle non-200 status codes                                                          │\n",
-       "│   306 │   if response.status_code != 200:                                                        │\n",
-       "│   307 │   │   if _can_parse_as_json_object(response.text):                                       │\n",
-       "│ ❱ 308 │   │   │   raise RestException(json.loads(response.text))                                 │\n",
-       "│   309 │   │   else:                                                                              │\n",
-       "│   310 │   │   │   base_msg = (                                                                   │\n",
-       "│   311 │   │   │   │   f\"API request to endpoint {endpoint} \"                                     │\n",
-       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
-       "RestException: INVALID_PARAMETER_VALUE: Changing param values is not allowed. Param with \n",
-       "key='registered_model_name' was already logged with \n",
-       "value='Fine-Tuned-Medical-DeepSeek-deepseek-ai/DeepSeek-R1-Distill-Llama-8B' for run \n",
-       "ID='ac48b182ac8746178c9dc26c42176a2c'. Attempted logging new value 'Fine-Tuned-Medical-DeepSeek'.\n",
-       "\n",
-       "During handling of the above exception, another exception occurred:\n",
-       "\n",
-       "╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
-       "│ in <module>:1                                                                                    │\n",
-       "│                                                                                                  │\n",
-       "│ ❱  1 register_model(                                                                             │\n",
-       "│    2 │   tracking_server_arn=\"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server    │\n",
-       "│    3 │   experiment_name=\"AIM405-deepseek-finetune-pipeline\",                                    │\n",
-       "│    4 │   run_id=\"ac48b182ac8746178c9dc26c42176a2c\",  # Assuming training_step returns run_id     │\n",
-       "│                                                                                                  │\n",
-       "│ in register_model:43                                                                             │\n",
-       "│                                                                                                  │\n",
-       "│    40 │   │   }                                                                                  │\n",
-       "│    41 │   │                                                                                      │\n",
-       "│    42 │   │   # Log model info as parameters                                                     │\n",
-       "│ ❱  43 │   │   mlflow.log_param(\"registered_model_name\", model_name)                              │\n",
-       "│    44 │   │   mlflow.log_param(\"model_artifacts_path\", model_artifacts_s3_path)                  │\n",
-       "│    45 │   │   mlflow.log_param(\"evaluation_score\", evaluation_score)                             │\n",
-       "│    46 │   │   mlflow.log_param(\"endpoint_name\", endpoint_name)                                   │\n",
-       "│                                                                                                  │\n",
-       "│ /opt/conda/lib/python3.12/site-packages/mlflow/tracking/fluent.py:775 in log_param               │\n",
-       "│                                                                                                  │\n",
-       "│    772 │   \"\"\"                                                                                   │\n",
-       "│    773 │   run_id = _get_or_start_run().info.run_id                                              │\n",
-       "│    774 │   synchronous = synchronous if synchronous is not None else not MLFLOW_ENABLE_ASYNC_LO  │\n",
-       "│ ❱  775 │   return MlflowClient().log_param(run_id, key, value, synchronous=synchronous)          │\n",
-       "│    776                                                                                           │\n",
-       "│    777                                                                                           │\n",
-       "│    778 def flush_async_logging() -> None:                                                        │\n",
-       "│                                                                                                  │\n",
-       "│ /opt/conda/lib/python3.12/site-packages/mlflow/tracking/client.py:2098 in log_param              │\n",
-       "│                                                                                                  │\n",
-       "│   2095 │   │   │   synchronous if synchronous is not None else not MLFLOW_ENABLE_ASYNC_LOGGING.  │\n",
-       "│   2096 │   │   )                                                                                 │\n",
-       "│   2097 │   │   if synchronous:                                                                   │\n",
-       "│ ❱ 2098 │   │   │   self._tracking_client.log_param(run_id, key, value, synchronous=True)         │\n",
-       "│   2099 │   │   │   return value                                                                  │\n",
-       "│   2100 │   │   else:                                                                             │\n",
-       "│   2101 │   │   │   return self._tracking_client.log_param(run_id, key, value, synchronous=False  │\n",
-       "│                                                                                                  │\n",
-       "│ /opt/conda/lib/python3.12/site-packages/mlflow/telemetry/track.py:29 in wrapper                  │\n",
-       "│                                                                                                  │\n",
-       "│    26 │   │   │   result = None                                                                  │\n",
-       "│    27 │   │   │   start_time = time.time()                                                       │\n",
-       "│    28 │   │   │   try:                                                                           │\n",
-       "│ ❱  29 │   │   │   │   result = func(*args, **kwargs)                                             │\n",
-       "│    30 │   │   │   │   return result  # noqa: RET504                                              │\n",
-       "│    31 │   │   │   except Exception:                                                              │\n",
-       "│    32 │   │   │   │   success = False                                                            │\n",
-       "│                                                                                                  │\n",
-       "│ /opt/conda/lib/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:408 in       │\n",
-       "│ log_param                                                                                        │\n",
-       "│                                                                                                  │\n",
-       "│   405 │   │   except MlflowException as e:                                                       │\n",
-       "│   406 │   │   │   if e.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE):                    │\n",
-       "│   407 │   │   │   │   msg = f\"{e.message}{PARAM_VALIDATION_MSG}\"                                 │\n",
-       "│ ❱ 408 │   │   │   │   raise MlflowException(msg, INVALID_PARAMETER_VALUE)                        │\n",
-       "│   409 │   │   │   else:                                                                          │\n",
-       "│   410 │   │   │   │   raise e                                                                    │\n",
-       "│   411                                                                                            │\n",
-       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
-       "MlflowException: INVALID_PARAMETER_VALUE: Changing param values is not allowed. Param with \n",
-       "key='registered_model_name' was already logged with \n",
-       "value='Fine-Tuned-Medical-DeepSeek-deepseek-ai/DeepSeek-R1-Distill-Llama-8B' for run \n",
-       "ID='ac48b182ac8746178c9dc26c42176a2c'. Attempted logging new value 'Fine-Tuned-Medical-DeepSeek'.\n",
-       "\n",
-       "The cause of this error is typically due to repeated calls\n",
-       "to an individual run_id event logging.\n",
-       "\n",
-       "Incorrect Example:\n",
-       "---------------------------------------\n",
-       "with mlflow.start_run():\n",
-       "    mlflow.log_param(\"depth\", 3)\n",
-       "    mlflow.log_param(\"depth\", 5)\n",
-       "---------------------------------------\n",
-       "\n",
-       "Which will throw an MlflowException for overwriting a\n",
-       "logged parameter.\n",
-       "\n",
-       "Correct Example:\n",
-       "---------------------------------------\n",
-       "with mlflow.start_run():\n",
-       "    with mlflow.start_run(nested=True):\n",
-       "        mlflow.log_param(\"depth\", 3)\n",
-       "    with mlflow.start_run(nested=True):\n",
-       "        mlflow.log_param(\"depth\", 5)\n",
-       "---------------------------------------\n",
-       "\n",
-       "Which will create a new nested run for each individual\n",
-       "model and prevent parameter key collisions within the\n",
-       "tracking store.\n",
-       "
\n" - ], - "text/plain": [ - "\u001b[38;2;255;0;0m╭─\u001b[0m\u001b[38;2;255;0;0m──────────────────────────────\u001b[0m\u001b[38;2;255;0;0m \u001b[0m\u001b[1;38;2;255;0;0mTraceback \u001b[0m\u001b[1;2;38;2;255;0;0m(most recent call last)\u001b[0m\u001b[38;2;255;0;0m \u001b[0m\u001b[38;2;255;0;0m───────────────────────────────\u001b[0m\u001b[38;2;255;0;0m─╮\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/tracking/_tracking_service/\u001b[0m\u001b[1;33mclient.py\u001b[0m:\u001b[94m401\u001b[0m in \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m398 \u001b[0m\u001b[2m│ │ \u001b[0mparam = Param(key, \u001b[96mstr\u001b[0m(value)) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m399 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m400 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m synchronous: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m401 \u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m.store.log_param(run_id, param) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m402 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mreturn\u001b[0m value \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m403 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m404 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m.store.log_param_async(run_id, param) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/store/tracking/\u001b[0m\u001b[1;33mrest_store.py\u001b[0m:\u001b[94m644\u001b[0m in \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 641 \u001b[0m\u001b[2m│ │ \u001b[0mreq_body = message_to_json( \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 642 \u001b[0m\u001b[2m│ │ │ \u001b[0mLogParam(run_uuid=run_id, run_id=run_id, key=param.key, value=param.value) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 643 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 644 \u001b[2m│ │ \u001b[0m\u001b[1;4;96mself\u001b[0m\u001b[1;4m._call_endpoint(LogParam, req_body)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 645 \u001b[0m\u001b[2m│ \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 646 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m\u001b[90m \u001b[0m\u001b[92mset_experiment_tag\u001b[0m(\u001b[96mself\u001b[0m, experiment_id, tag): \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 647 \u001b[0m\u001b[2;90m│ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/store/tracking/\u001b[0m\u001b[1;33mrest_store.py\u001b[0m:\u001b[94m134\u001b[0m in \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[92m_call_endpoint\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 131 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 132 \u001b[0m\u001b[2m│ │ │ \u001b[0mendpoint, method = _METHOD_TO_INFO[api] \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 133 \u001b[0m\u001b[2m│ │ \u001b[0mresponse_proto = api.Response() \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 134 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m call_endpoint( \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 135 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.get_host_creds(), \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 136 \u001b[0m\u001b[2m│ │ │ \u001b[0mendpoint, \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 137 \u001b[0m\u001b[2m│ │ │ \u001b[0mmethod, \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/utils/\u001b[0m\u001b[1;33mrest_utils.py\u001b[0m:\u001b[94m554\u001b[0m in \u001b[92mcall_endpoint\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m551 \u001b[0m\u001b[2m│ │ \u001b[0mcall_kwargs[\u001b[33m\"\u001b[0m\u001b[33mjson\u001b[0m\u001b[33m\"\u001b[0m] = json_body \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m552 \u001b[0m\u001b[2m│ │ \u001b[0mresponse = http_request(**call_kwargs) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m553 \u001b[0m\u001b[2m│ \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m554 \u001b[2m│ \u001b[0mresponse = \u001b[1;4mverify_rest_response(response, endpoint)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m555 \u001b[0m\u001b[2m│ \u001b[0mresponse_to_parse = response.text \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m556 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m557 \u001b[0m\u001b[2m│ │ \u001b[0mjs_dict = json.loads(response_to_parse) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/utils/\u001b[0m\u001b[1;33mrest_utils.py\u001b[0m:\u001b[94m308\u001b[0m in \u001b[92mverify_rest_response\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m305 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Handle non-200 status codes\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m306 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m response.status_code != \u001b[94m200\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m307 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m _can_parse_as_json_object(response.text): \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m308 \u001b[2m│ │ │ \u001b[0m\u001b[1;4;94mraise\u001b[0m\u001b[1;4m RestException(json.loads(response.text))\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m309 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m310 \u001b[0m\u001b[2m│ │ │ \u001b[0mbase_msg = ( \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m311 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mAPI request to endpoint \u001b[0m\u001b[33m{\u001b[0mendpoint\u001b[33m}\u001b[0m\u001b[33m \u001b[0m\u001b[33m\"\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", - "\u001b[1;91mRestException: \u001b[0mINVALID_PARAMETER_VALUE: Changing param values is not allowed. Param with \n", - "\u001b[38;2;215;175;0mkey\u001b[0m=\u001b[38;2;0;135;0m'registered_model_name'\u001b[0m was already logged with \n", - "\u001b[38;2;215;175;0mvalue\u001b[0m=\u001b[38;2;0;135;0m'Fine-Tuned-Medical-DeepSeek-deepseek-ai/DeepSeek-R1-Distill-Llama-8B'\u001b[0m for run \n", - "\u001b[38;2;215;175;0mID\u001b[0m=\u001b[38;2;0;135;0m'ac48b182ac8746178c9dc26c42176a2c'\u001b[0m. Attempted logging new value \u001b[38;2;0;135;0m'Fine-Tuned-Medical-DeepSeek'\u001b[0m.\n", - "\n", - "\u001b[3mDuring handling of the above exception, another exception occurred:\u001b[0m\n", - "\n", - "\u001b[38;2;255;0;0m╭─\u001b[0m\u001b[38;2;255;0;0m──────────────────────────────\u001b[0m\u001b[38;2;255;0;0m \u001b[0m\u001b[1;38;2;255;0;0mTraceback \u001b[0m\u001b[1;2;38;2;255;0;0m(most recent call last)\u001b[0m\u001b[38;2;255;0;0m \u001b[0m\u001b[38;2;255;0;0m───────────────────────────────\u001b[0m\u001b[38;2;255;0;0m─╮\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 1 register_model( \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 2 \u001b[0m\u001b[2m│ \u001b[0mtracking_server_arn=\u001b[33m\"\u001b[0m\u001b[33marn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 3 \u001b[0m\u001b[2m│ \u001b[0mexperiment_name=\u001b[33m\"\u001b[0m\u001b[33mAIM405-deepseek-finetune-pipeline\u001b[0m\u001b[33m\"\u001b[0m, \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 4 \u001b[0m\u001b[2m│ \u001b[0mrun_id=\u001b[33m\"\u001b[0m\u001b[33mac48b182ac8746178c9dc26c42176a2c\u001b[0m\u001b[33m\"\u001b[0m, \u001b[2m# Assuming training_step returns run_id \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m in \u001b[92mregister_model\u001b[0m:\u001b[94m43\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 40 \u001b[0m\u001b[2m│ │ \u001b[0m} \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 41 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 42 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Log model info as parameters\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 43 \u001b[2m│ │ \u001b[0m\u001b[1;4mmlflow.log_param(\u001b[0m\u001b[1;4;33m\"\u001b[0m\u001b[1;4;33mregistered_model_name\u001b[0m\u001b[1;4;33m\"\u001b[0m\u001b[1;4m, model_name)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 44 \u001b[0m\u001b[2m│ │ \u001b[0mmlflow.log_param(\u001b[33m\"\u001b[0m\u001b[33mmodel_artifacts_path\u001b[0m\u001b[33m\"\u001b[0m, model_artifacts_s3_path) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 45 \u001b[0m\u001b[2m│ │ \u001b[0mmlflow.log_param(\u001b[33m\"\u001b[0m\u001b[33mevaluation_score\u001b[0m\u001b[33m\"\u001b[0m, evaluation_score) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 46 \u001b[0m\u001b[2m│ │ \u001b[0mmlflow.log_param(\u001b[33m\"\u001b[0m\u001b[33mendpoint_name\u001b[0m\u001b[33m\"\u001b[0m, endpoint_name) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/tracking/\u001b[0m\u001b[1;33mfluent.py\u001b[0m:\u001b[94m775\u001b[0m in \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 772 \u001b[0m\u001b[2;33m│ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 773 \u001b[0m\u001b[2m│ \u001b[0mrun_id = _get_or_start_run().info.run_id \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 774 \u001b[0m\u001b[2m│ \u001b[0msynchronous = synchronous \u001b[94mif\u001b[0m synchronous \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m \u001b[95mnot\u001b[0m MLFLOW_ENABLE_ASYNC_LO \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 775 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[1;4mMlflowClient().log_param(run_id, key, value, synchronous=synchronous)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 776 \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 777 \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 778 \u001b[0m\u001b[94mdef\u001b[0m\u001b[90m \u001b[0m\u001b[92mflush_async_logging\u001b[0m() -> \u001b[94mNone\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/tracking/\u001b[0m\u001b[1;33mclient.py\u001b[0m:\u001b[94m2098\u001b[0m in \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2095 \u001b[0m\u001b[2m│ │ │ \u001b[0msynchronous \u001b[94mif\u001b[0m synchronous \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m \u001b[95mnot\u001b[0m MLFLOW_ENABLE_ASYNC_LOGGING. \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2096 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2097 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m synchronous: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m2098 \u001b[2m│ │ │ \u001b[0m\u001b[1;4;96mself\u001b[0m\u001b[1;4m._tracking_client.log_param(run_id, key, value, synchronous=\u001b[0m\u001b[1;4;94mTrue\u001b[0m\u001b[1;4m)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2099 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m value \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2100 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m2101 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._tracking_client.log_param(run_id, key, value, synchronous=\u001b[94mFalse\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/telemetry/\u001b[0m\u001b[1;33mtrack.py\u001b[0m:\u001b[94m29\u001b[0m in \u001b[92mwrapper\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 26 \u001b[0m\u001b[2m│ │ │ \u001b[0mresult = \u001b[94mNone\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 27 \u001b[0m\u001b[2m│ │ │ \u001b[0mstart_time = time.time() \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 28 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m 29 \u001b[2m│ │ │ │ \u001b[0mresult = func(*args, **kwargs) \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 30 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mreturn\u001b[0m result \u001b[2m# noqa: RET504\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 31 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m 32 \u001b[0m\u001b[2m│ │ │ │ \u001b[0msuccess = \u001b[94mFalse\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.12/site-packages/mlflow/tracking/_tracking_service/\u001b[0m\u001b[1;33mclient.py\u001b[0m:\u001b[94m408\u001b[0m in \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[92mlog_param\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m405 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mexcept\u001b[0m MlflowException \u001b[94mas\u001b[0m e: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m406 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m e.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE): \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m407 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mmsg = \u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m{\u001b[0me.message\u001b[33m}\u001b[0m\u001b[33m{\u001b[0mPARAM_VALIDATION_MSG\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[31m❱ \u001b[0m408 \u001b[2m│ │ │ │ \u001b[0m\u001b[1;4;94mraise\u001b[0m\u001b[1;4m MlflowException(msg, INVALID_PARAMETER_VALUE)\u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m409 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m410 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m│\u001b[0m \u001b[2m411 \u001b[0m \u001b[38;2;255;0;0m│\u001b[0m\n", - "\u001b[38;2;255;0;0m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", - "\u001b[1;91mMlflowException: \u001b[0mINVALID_PARAMETER_VALUE: Changing param values is not allowed. Param with \n", - "\u001b[38;2;215;175;0mkey\u001b[0m=\u001b[38;2;0;135;0m'registered_model_name'\u001b[0m was already logged with \n", - "\u001b[38;2;215;175;0mvalue\u001b[0m=\u001b[38;2;0;135;0m'Fine-Tuned-Medical-DeepSeek-deepseek-ai/DeepSeek-R1-Distill-Llama-8B'\u001b[0m for run \n", - "\u001b[38;2;215;175;0mID\u001b[0m=\u001b[38;2;0;135;0m'ac48b182ac8746178c9dc26c42176a2c'\u001b[0m. Attempted logging new value \u001b[38;2;0;135;0m'Fine-Tuned-Medical-DeepSeek'\u001b[0m.\n", - "\n", - "The cause of this error is typically due to repeated calls\n", - "to an individual run_id event logging.\n", - "\n", - "Incorrect Example:\n", - "---------------------------------------\n", - "with \u001b[1;38;2;225;0;225mmlflow.start_run\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m:\n", - " \u001b[1;38;2;225;0;225mmlflow.log_param\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;0;135;0m\"depth\"\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m\n", - " \u001b[1;38;2;225;0;225mmlflow.log_param\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;0;135;0m\"depth\"\u001b[0m, \u001b[1;36m5\u001b[0m\u001b[1m)\u001b[0m\n", - "---------------------------------------\n", - "\n", - "Which will throw an MlflowException for overwriting a\n", - "logged parameter.\n", - "\n", - "Correct Example:\n", - "---------------------------------------\n", - "with \u001b[1;38;2;225;0;225mmlflow.start_run\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m:\n", - " with \u001b[1;38;2;225;0;225mmlflow.start_run\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;215;175;0mnested\u001b[0m=\u001b[3;38;2;0;135;0mTrue\u001b[0m\u001b[1m)\u001b[0m:\n", - " \u001b[1;38;2;225;0;225mmlflow.log_param\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;0;135;0m\"depth\"\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m\n", - " with \u001b[1;38;2;225;0;225mmlflow.start_run\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;215;175;0mnested\u001b[0m=\u001b[3;38;2;0;135;0mTrue\u001b[0m\u001b[1m)\u001b[0m:\n", - " \u001b[1;38;2;225;0;225mmlflow.log_param\u001b[0m\u001b[1m(\u001b[0m\u001b[38;2;0;135;0m\"depth\"\u001b[0m, \u001b[1;36m5\u001b[0m\u001b[1m)\u001b[0m\n", - "---------------------------------------\n", - "\n", - "Which will create a new nested run for each individual\n", - "model and prevent parameter key collisions within the\n", - "tracking store.\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "register_model(\n", - " tracking_server_arn=\"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server/my-tracking-server\",\n", - " experiment_name=\"AIM405-deepseek-finetune-pipeline\",\n", - " run_id=\"ac48b182ac8746178c9dc26c42176a2c\", # Assuming training_step returns run_id as first output\n", - " model_artifacts_s3_path=\"s3://sagemaker-us-east-1-329542461890/deepseek-finetune-2025-09-15-18-39-44-2025-09-15-18-39-44-661/output/model.tar.gz\", # Assuming training_step returns artifacts path as second output\n", - " model_id=\"s3://sagemaker-us-east-1-329542461890/models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B\",\n", - " model_name=f\"Fine-Tuned-Medical-DeepSeek\",\n", - " endpoint_name=\"deepseek-ai-DeepSeek-R1-Distill-Llama-8B-sft-djl\",\n", - " evaluation_score=0.22410036842493342, # Get the evaluation score\n", - " pipeline_name=\"AIM405-deepseek-finetune-pipeline\",\n", - " model_description=\"Fine-tuned medical LLM for clinical reasoning and diagnostics\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-15T19:37:33.572826Z", - "iopub.status.busy": "2025-09-15T19:37:33.572611Z", - "iopub.status.idle": "2025-09-15T19:37:33.584816Z", - "shell.execute_reply": "2025-09-15T19:37:33.584262Z", - "shell.execute_reply.started": "2025-09-15T19:37:33.572810Z" - } - }, - "outputs": [], - "source": [ - "def register_model(\n", - " tracking_server_arn: str,\n", - " experiment_name: str,\n", - " run_id: str,\n", - " model_artifacts_s3_path: str,\n", - " model_id: str,\n", - " model_name: str,\n", - " endpoint_name: str,\n", - " evaluation_score: float,\n", - " pipeline_name: str,\n", - " model_description: str\n", - "):\n", - " import json\n", - " import mlflow\n", - " import boto3\n", - " import os\n", - " import tempfile\n", - " import time\n", - " from datetime import datetime\n", - " \n", - " print(f\"Registering model: {model_name}\")\n", - " \n", - " # Set up MLflow tracking\n", - " mlflow.set_tracking_uri(tracking_server_arn)\n", - " mlflow.set_experiment(experiment_name)\n", - " \n", - " # Connect to MLflow with the specific run\n", - " with mlflow.start_run(run_id=run_id):\n", - " # Create model metadata\n", - " tags = {\n", - " \"model_id\": model_id,\n", - " \"base_model\": model_id.split('/')[-1],\n", - " \"task\": \"medical_qa\",\n", - " \"framework\": \"pytorch\",\n", - " \"endpoint_name\": endpoint_name,\n", - " \"model_artifacts_s3_path\": model_artifacts_s3_path,\n", - " \"deployment_timestamp\": datetime.now().isoformat(),\n", - " \"description\": model_description,\n", - " \"registered_by\": pipeline_name\n", - " }\n", - " \n", - " # Log model info as parameters\n", - " mlflow.log_param(\"registered_model_name\", model_name)\n", - " mlflow.log_param(\"model_artifacts_path\", model_artifacts_s3_path)\n", - " mlflow.log_param(\"evaluation_score\", evaluation_score)\n", - " mlflow.log_param(\"endpoint_name\", endpoint_name)\n", - " # mlflow.log_param(\"registration_timestamp\", datetime.now().isoformat())\n", - " mlflow.log_param(\"registration_timestamp\", \"2025-09-15T19:28:36.049313\")\n", - " \n", - " # Log endpoint information as an artifact\n", - " model_info = {\n", - " \"model_name\": model_name,\n", - " \"model_id\": model_id,\n", - " \"endpoint_name\": endpoint_name,\n", - " \"model_artifacts_s3_path\": model_artifacts_s3_path,\n", - " \"evaluation_score\": float(evaluation_score),\n", - " # \"registration_timestamp\": datetime.now().isoformat()\n", - " \"registration_timestamp\": \"2025-09-15T19:28:36.049313\"\n", - " }\n", - " \n", - " with open(\"/tmp/model_info.json\", \"w\") as f:\n", - " json.dump(model_info, f, indent=2)\n", - " mlflow.log_artifact(\"/tmp/model_info.json\")\n", - " \n", - " # Create model card\n", - " model_card = f\"\"\"\n", - " # {model_name}\n", - " \n", - " ## Model Information\n", - " - **Base Model**: {model_id}\n", - " - **Task**: Medical Question Answering\n", - " - **Evaluation Score**: {evaluation_score:.4f}\n", - " - **Endpoint**: {endpoint_name}\n", - " \n", - " ## Description\n", - " {model_description}\n", - " \n", - " ## Registration Details\n", - " - Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", - " - Pipeline: {pipeline_name}\n", - " \"\"\"\n", - " \n", - " with open(\"/tmp/model_card.md\", \"w\") as f:\n", - " f.write(model_card)\n", - " mlflow.log_artifact(\"/tmp/model_card.md\")\n", - " \n", - " # PART 1: REGISTER WITH MLFLOW MODEL REGISTRY\n", - " mlflow_version = None\n", - " try:\n", - " client = mlflow.tracking.MlflowClient()\n", - " \n", - " # Check if model exists and create if it doesn't\n", - " try:\n", - " client.get_registered_model(model_name)\n", - " print(f\"Model {model_name} already exists in MLflow registry\")\n", - " except mlflow.exceptions.MlflowException:\n", - " client.create_registered_model(\n", - " name=model_name,\n", - " description=f\"Fine-tuned medical LLM based on {model_id}\"\n", - " )\n", - " print(f\"Created new registered model: {model_name}\")\n", - " \n", - " # Create empty model directory with artifacts\n", - " with tempfile.TemporaryDirectory() as tmp_dir:\n", - " # Create a minimal model file to log\n", - " os.makedirs(os.path.join(tmp_dir, \"model\"), exist_ok=True)\n", - " \n", - " # Copy model info and card to directory\n", - " with open(os.path.join(tmp_dir, \"model\", \"model_info.json\"), \"w\") as f:\n", - " json.dump(model_info, f, indent=2)\n", - " \n", - " with open(os.path.join(tmp_dir, \"model\", \"model_card.md\"), \"w\") as f:\n", - " f.write(model_card)\n", - " \n", - " # Create a model reference file pointing to the S3 artifacts\n", - " model_ref = {\n", - " \"artifact_path\": model_artifacts_s3_path,\n", - " \"flavors\": {\n", - " \"pytorch\": {\n", - " \"model_data\": model_artifacts_s3_path,\n", - " \"pytorch_version\": \"2.0+\"\n", - " }\n", - " },\n", - " \"run_id\": run_id,\n", - " \"model_class\": \"LLM\",\n", - " \"model_format\": \"PyTorch\"\n", - " }\n", - " \n", - " with open(os.path.join(tmp_dir, \"model\", \"MLmodel\"), \"w\") as f:\n", - " json.dump(model_ref, f, indent=2)\n", - " \n", - " # Log artifacts directory as model\n", - " mlflow.log_artifacts(tmp_dir, artifact_path=\"\")\n", - " \n", - " # Now register the model - try both methods\n", - " try:\n", - " # Method 1: Use direct registration with source as run URI\n", - " model_uri = f\"runs:/{run_id}/model\"\n", - " model_details = mlflow.register_model(\n", - " model_uri=model_uri,\n", - " name=model_name,\n", - " tags=tags\n", - " )\n", - " mlflow_version = model_details.version\n", - " \n", - " except Exception as e1:\n", - " print(f\"Method 1 registration failed: {str(e1)}\")\n", - " \n", - " try:\n", - " # Method 2: Create version with client API\n", - " model_version = client.create_model_version(\n", - " name=model_name,\n", - " source=f\"runs:/{run_id}/model\", # Use run URI instead of direct S3\n", - " run_id=run_id,\n", - " description=f\"Fine-tuned LLM deployed at endpoint: {endpoint_name}\"\n", - " )\n", - " mlflow_version = model_version.version\n", - " \n", - " # Wait for model registration to complete\n", - " for _ in range(10): # Try for up to ~50 seconds\n", - " version_details = client.get_model_version(model_name, model_version.version)\n", - " if version_details.status == \"READY\":\n", - " break\n", - " time.sleep(5)\n", - " \n", - " # Add tags to the registered model version\n", - " for key, value in tags.items():\n", - " client.set_model_version_tag(model_name, model_version.version, key, value)\n", - " except Exception as e2:\n", - " print(f\"Method 2 registration failed: {str(e2)}\")\n", - " mlflow_version = \"unknown\"\n", - " \n", - " if mlflow_version and mlflow_version != \"unknown\":\n", - " # Transition model to Production/Staging based on evaluation score\n", - " if evaluation_score >= 0.3: # Example threshold\n", - " client.transition_model_version_stage(\n", - " name=model_name,\n", - " version=mlflow_version,\n", - " stage=\"Production\",\n", - " archive_existing_versions=True\n", - " )\n", - " print(f\"Model {model_name} version {mlflow_version} promoted to Production\")\n", - " else:\n", - " client.transition_model_version_stage(\n", - " name=model_name,\n", - " version=mlflow_version,\n", - " stage=\"Staging\",\n", - " archive_existing_versions=False\n", - " )\n", - " print(f\"Model {model_name} version {mlflow_version} added to Staging due to lower evaluation score\")\n", - " \n", - " print(f\"Successfully registered model in MLflow: {model_name}, version: {mlflow_version}\")\n", - " \n", - " except Exception as e:\n", - " print(f\"Error registering model in MLflow: {str(e)}\")\n", - " mlflow_version = \"unknown\"\n", - " \n", - " # PART 2: REGISTER WITH SAGEMAKER MODEL REGISTRY\n", - " sm_model_version = \"unknown\"\n", - " try:\n", - " sm_client = boto3.client('sagemaker')\n", - " \n", - " # Create a normalized name for SageMaker resources\n", - " sm_model_name = model_name.replace(\".\", \"-\").replace(\"_\", \"-\")\n", - " \n", - " # Create or update model package group\n", - " try:\n", - " sm_client.describe_model_package_group(ModelPackageGroupName=sm_model_name)\n", - " print(f\"SageMaker model package group {sm_model_name} already exists\")\n", - " except sm_client.exceptions.ClientError:\n", - " sm_client.create_model_package_group(\n", - " ModelPackageGroupName=sm_model_name,\n", - " ModelPackageGroupDescription=f\"Fine-tuned LLM model: {model_name}\"\n", - " )\n", - " print(f\"Created SageMaker model package group: {sm_model_name}\")\n", - " \n", - " # Create a model package and register it\n", - " try:\n", - " # Create model package\n", - " response = sm_client.create_model_package(\n", - " ModelPackageGroupName=sm_model_name,\n", - " ModelPackageDescription=model_description,\n", - " SourceAlgorithmSpecification={\n", - " 'SourceAlgorithms': [\n", - " {\n", - " 'AlgorithmName': 'pytorch-llm',\n", - " 'ModelDataUrl': model_artifacts_s3_path\n", - " }\n", - " ]\n", - " },\n", - " ValidationSpecification={\n", - " 'ValidationRole': 'dummy-role', # Required but not used\n", - " 'ValidationProfiles': [\n", - " {\n", - " 'ProfileName': 'ValidationProfile1',\n", - " 'TransformJobDefinition': {\n", - " 'TransformInput': {\n", - " 'DataSource': {\n", - " 'S3DataSource': {\n", - " 'S3DataType': 'S3Prefix',\n", - " 'S3Uri': 's3://dummy-bucket/dummy-prefix' # Required but not used\n", - " }\n", - " }\n", - " },\n", - " 'TransformOutput': {\n", - " 'S3OutputPath': 's3://dummy-bucket/dummy-output' # Required but not used\n", - " },\n", - " 'TransformResources': {\n", - " 'InstanceType': 'ml.m5.large', # Required but not used\n", - " 'InstanceCount': 1\n", - " }\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " ModelApprovalStatus='Approved',\n", - " MetadataProperties={\n", - " 'GeneratedBy': pipeline_name,\n", - " 'Repository': model_id,\n", - " 'EvaluationScore': str(evaluation_score)\n", - " },\n", - " ModelMetrics={\n", - " 'ModelQuality': {\n", - " 'Statistics': {\n", - " 'ContentType': 'application/json',\n", - " 'S3Uri': f\"s3://{model_artifacts_s3_path.split('/', 3)[2]}/{run_id}/artifacts/model_info.json\"\n", - " }\n", - " }\n", - " }\n", - " )\n", - " \n", - " sm_model_version = response['ModelPackageArn'].split('/')[-1]\n", - " print(f\"Created SageMaker model package: {sm_model_version}\")\n", - " \n", - " except Exception as e_package:\n", - " print(f\"Error creating model package: {str(e_package)}\")\n", - " \n", - " # Log SageMaker details\n", - " mlflow.log_param(\"sagemaker_model_group\", sm_model_name)\n", - " mlflow.log_param(\"sagemaker_model_version\", sm_model_version)\n", - " \n", - " print(f\"Successfully integrated with SageMaker model registry\")\n", - " \n", - " except Exception as e:\n", - " print(f\"Warning: Error in SageMaker model registry integration: {str(e)}\")\n", - " \n", - " return model_name, str(mlflow_version)" - ] - }, { "cell_type": "markdown", "metadata": {}, From b960a9ed13aa4b6a61a2762c8c52025913e559ec Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Fri, 19 Sep 2025 16:34:53 -0400 Subject: [PATCH 03/22] Added the Example FMOps Notebook --- .../task_05_fmops/05.00_fmops_examples.ipynb | 1135 +++++++++++++++++ 1 file changed, 1135 insertions(+) create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb new file mode 100644 index 0000000..de2b1ec --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -0,0 +1,1135 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example Operations of FMOps\n", + "\n", + "The purpose of this notebook is to illustrate the capabilities SageMaker AI and Managed MLflow on SageMaker AI for FMOps tasks. In this notebook, we cover the foundational capabilities needed to develop an automated LLM fine-tuning and evaluation pipeline. We cover these components individually, without an orchestration service, to showcase the capabilities atomically. This notebook lays the groundwork for the next notebook, which stiches together these disparate components into a fully orchestrated fine-tuning and model evaluation pipeline powered by SageMaker AI Pipelines and Managed MLflow on SageMaker AI." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites \n", + "Before you begin, make sure you have the following prerequisites in place:\n", + "\n", + "- MLflow tracking server: If you're running this lab in a workshop environment, a MLflow tracking server has already been created for you. If you need to create a MLflow tracking server, follow the [documentation](https://docs.aws.amazon.com/sagemaker/latest/dg/mlflow-create-tracking-server.html)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Setup and Dependencies\n", + "Install dependencies and configure kernel.\n", + "\n", + "Restart the kernel after executing below cells." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r ./scripts/requirements.txt --upgrade --quiet" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython import get_ipython\n", + "get_ipython().kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Importing Libraries and Setting Up Environment**\n", + "\n", + "This part imports all necessary Python modules. It includes SageMaker-specific imports for pipeline creation and execution, which will be used to define the pipeline steps." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "import time\n", + "import uuid\n", + "import boto3\n", + "import mlflow\n", + "import tarfile\n", + "import botocore\n", + "import sagemaker\n", + "import traceback\n", + "\n", + "from tqdm import tqdm\n", + "from datetime import datetime\n", + "from sagemaker.huggingface import HuggingFaceModel" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. SageMaker Session and IAM Role" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`get_execution_role()`: Retrieves the IAM role that SageMaker will use to access AWS resources. This role needs appropriate permissions for tasks like accessing S3 buckets and creating SageMaker resources.\n", + "\n", + "If you are running this lab in a workshop environment, the execution role will have the appropriate permissions necessary to execute the following tasks. If not, you may need to check the permissions attached to your sagemaker execution role." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sagemaker_session = sagemaker.session.Session()\n", + "role = sagemaker.get_execution_role()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Configuration\n", + "Here we setup our example execution environment.\n", + "\n", + "We define appropriate paths in S3 to store model files, define the model we will be working with, and define the model endpoint name.\n", + "\n", + "In this lab, we are working with [DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B). It is easy to fine-tune as we will see in the next lab, and is small enough to fit on a reasonably sized GPU-accelerated hosting endpoint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bucket_name = sagemaker_session.default_bucket()\n", + "print(bucket_name)\n", + "default_prefix = sagemaker_session.default_bucket_prefix\n", + "if default_prefix:\n", + " input_path = f'{default_prefix}/datasets/llm-fine-tuning-modeltrainer-sft'\n", + "else:\n", + " input_path = f'datasets/llm-fine-tuning-modeltrainer-sft'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_id = \"deepseek-ai/DeepSeek-R1-Distill-Llama-8B\"\n", + "model_id_filesafe = model_id.replace(\"/\",\"_\").replace(\".\", \"_\")\n", + "endpoint_name = \"Example-deepseek-ai-DeepSeek-R1-Distill-Llama-8B-sft-djl\"\n", + "model_name_safe = model_id.split('/')[-1].replace('.', '-').replace('_', '-')\n", + "instance_count = 1\n", + "instance_type = \"ml.g5.2xlarge\"\n", + "guardrail_id = \"u1yfe55ecv4z\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "MLflow integration is crucial for experiment tracking and management. \n", + "\n", + "**Update the ARN for the MLflow tracking server.**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This example requires a SageMaker with MLflow tracking server to track experiments and manage model artifacts. To create your own tracking server please refer to the [SageMaker documentation](https://docs.aws.amazon.com/sagemaker/latest/dg/mlflow-create-tracking-server.html). Once you have created your tracking server, please copy the tracking server ARN to the `mlflow_tracking server_arn` variable in the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mlflow_tracking_server_arn = \"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server/my-tracking-server\"\n", + "\n", + "if not mlflow_tracking_server_arn:\n", + " try:\n", + " response = boto3.client('sagemaker').describe_mlflow_tracking_server(\n", + " TrackingServerName='genai-mlflow-tracker'\n", + " )\n", + " mlflow_tracking_server_arn = response['TrackingServerArn']\n", + " print(f\"MLflow Tracking Server ARN: {mlflow_tracking_server_arn}\")\n", + " except botocore.exceptions.ClientError:\n", + " print(\"No MLflow Tracking Server Found, please input a value for mlflow_tracking_server_arn\")\n", + "\n", + "os.environ[\"mlflow_tracking_server_arn\"] = mlflow_tracking_server_arn" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4. Templating a Prompt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this workshop we are going to fine-tune DeepSeek-R1-Distill-Llama-8B to become a medical expert. To accomplish this, we will execute a fine-tuning job using Managed MLflow on SageMaker AI. We get our data from the [FreedomIntelligence/medical-o1-reasoning-SFT](https://huggingface.co/datasets/FreedomIntelligence/medical-o1-reasoning-SFT) dataset, available on HuggingFace.\n", + "\n", + "We perform the full fine-tuning step in the next lab. In this lab, we show a small example of what fine-tuning looks like for a single record of the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "FINE_TUNING_DATA_SAMPLE = {\n", + " \"Question\": \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\", \n", + " \"Complex_CoT\": \"Okay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem.\\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal.\\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\",\n", + " \"Response\": \"Cystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.\"\n", + "}\n", + "\n", + "\n", + "PROMPT_TEMPLATE = \"\"\"\n", + "<|begin_of_text|>\n", + " <|start_header_id|>system<|end_header_id|>\n", + " You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", + " Below is an instruction that describes a task, paired with an input that provides further context. \n", + " Write a response that appropriately completes the request.\n", + " Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", + " <|eot_id|>\n", + " <|start_header_id|>user<|end_header_id|>\n", + " {{question}}\n", + " <|eot_id|>\n", + " <|start_header_id|>assistant<|end_header_id|>\n", + " {{complex_cot}}\n", + " {{answer}}\n", + "<|eot_id|>\n", + "\"\"\"\n", + "\n", + "# Template dataset to add prompt to each sample\n", + "def template_dataset(sample):\n", + " try:\n", + " sample[\"text\"] = PROMPT_TEMPLATE.format(question=sample[\"Question\"],\n", + " complex_cot=sample[\"Complex_CoT\"],\n", + " answer=sample[\"Response\"])\n", + " return sample\n", + " except KeyError as e:\n", + " print(f\"KeyError in template_dataset: {str(e)}\")\n", + " # Provide default values for missing fields\n", + " missing_key = str(e).strip(\"'\")\n", + " if missing_key == \"Question\":\n", + " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", + " question=\"[Missing question]\",\n", + " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", + " answer=sample.get(\"Response\", \"[Missing response]\")\n", + " )\n", + " elif missing_key == \"Complex_CoT\":\n", + " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", + " question=sample[\"Question\"],\n", + " complex_cot=\"[Missing CoT]\",\n", + " answer=sample.get(\"Response\", \"[Missing response]\")\n", + " )\n", + " elif missing_key == \"Response\":\n", + " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", + " question=sample[\"Question\"],\n", + " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", + " answer=\"[Missing response]\"\n", + " )\n", + " return sample\n", + "\n", + "PROCESSED_SAMPLE = template_dataset(FINE_TUNING_DATA_SAMPLE)\n", + "print(PROCESSED_SAMPLE)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Fine-Tuning Output\n", + "The above output shows the templated prompt output to be used for fine-tuning. This pre-processing happens for every record in the fine-tuning dataset before fine-tuning actually takes place. This can be time-consuming for large fine-tuning datasets. We will show in the next lab how to orchestrate this with MLflow." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5. Model Deployment\n", + "There are several approaches to deploying a model to a SageMaker AI managed endpoint. In this section, we explore the most direct option which downloads a model directly from HuggingFace to the managed endpoint via SageMaker JumpStart. We are still using DeepSeek-R1-Distill-Llama-8B, but we have not fine-tuned it. The purpose of this section is to illustrate the components required to customize a model deployment on SageMaker before fine-tuning it." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Image URI\n", + "By default, images downloaded from HuggingFace use the [Text Generation Inference](https://huggingface.co/docs/text-generation-inference/en/index) model serving toolkit. \n", + "\n", + "For this lab, we want to change the underlying model server to [Deep Java Library's Large Model Inference](https://docs.djl.ai/master/docs/serving/serving/docs/lmi/index.html) container, or DJL-LMI. This serving container offers [several performance benefits](https://aws.amazon.com/blogs/machine-learning/supercharge-your-llm-performance-with-amazon-sagemaker-large-model-inference-container-v15/) that we want to leverage for the production deployment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create and deploy model\n", + "image_uri = sagemaker.image_uris.retrieve(\n", + " framework=\"djl-lmi\",\n", + " region=sagemaker_session.boto_session.region_name,\n", + " version=\"latest\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### HuggingFace + SageMaker JumpStart\n", + "Here we download the model from SageMaker Jumpstart and create a `HuggingFaceModel` object. Notice how we define the `model_id` in the configuration, and specify the `image_uri` defined above in the instantiation of the model object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_config = {\n", + " 'HF_MODEL_ID': model_id,\n", + " 'SM_NUM_GPUS': json.dumps(1),\n", + " 'OPTION_TRUST_REMOTE_CODE': 'true',\n", + " 'OPTION_ROLLING_BATCH': \"vllm\",\n", + " 'OPTION_DTYPE': 'bf16',\n", + " 'OPTION_QUANTIZE': 'fp8',\n", + " 'OPTION_TENSOR_PARALLEL_DEGREE': 'max',\n", + " 'OPTION_MAX_ROLLING_BATCH_SIZE': '32',\n", + " 'OPTION_MODEL_LOADING_TIMEOUT': '3600',\n", + " 'OPTION_MAX_MODEL_LEN': '4096'\n", + "}\n", + "model = HuggingFaceModel(\n", + " image_uri=image_uri,\n", + " env=model_config,\n", + " role=role\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Model Deploy w/Managed MLFlow 3.0 on SageMaker AI\n", + "Now we stitch the pieces together and use MLFlow to orchestrate the deployment of our model to a SageMaker AI managed endpoint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize MLFlow tracking data...\n", + "mlflow.set_tracking_uri(mlflow_tracking_server_arn)\n", + "mlflow.set_experiment(\"Default\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with mlflow.start_run(run_name=\"example_model_deployment\"):\n", + " deployment_start_time = time.time()\n", + "\n", + " health_check_timeout = 1800\n", + " data_download_timeout = 3600\n", + "\n", + " # Log deployment parameters\n", + " mlflow.log_params({\n", + " \"model_id\": model_id,\n", + " \"instance_type\": instance_type,\n", + " \"instance_count\": instance_count,\n", + " \"endpoint_name\": endpoint_name,\n", + " \"health_check_timeout\": health_check_timeout,\n", + " \"data_download_timeout\": data_download_timeout\n", + " })\n", + " mlflow.log_params({\"model_config_\" + k: v for k, v in model_config.items()})\n", + "\n", + " try:\n", + " # deploy model to SageMaker Inference\n", + " predictor = model.deploy(\n", + " initial_instance_count=instance_count,\n", + " instance_type=instance_type,\n", + " container_startup_health_check_timeout=health_check_timeout,\n", + " model_data_download_timeout=data_download_timeout,\n", + " endpoint_name=endpoint_name\n", + " )\n", + "\n", + " # Log deployment metrics\n", + " deployment_time = time.time() - deployment_start_time\n", + " mlflow.log_metric(\"deployment_time_seconds\", deployment_time)\n", + " mlflow.log_metric(\"deployment_success\", 1)\n", + "\n", + " # Log tags\n", + " mlflow.set_tags({\n", + " \"endpoint_status\": \"deployed\",\n", + " \"deployment_type\": \"sagemaker\",\n", + " \"framework\": \"djl-lmi\"\n", + " })\n", + "\n", + " except Exception as e:\n", + " # Log deployment failure\n", + " mlflow.log_metric(\"deployment_success\", 0)\n", + " mlflow.log_param(\"error_message\", str(e))\n", + " mlflow.set_tag(\"endpoint_status\", \"failed\")\n", + " raise e" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Test Model Prediction\n", + "Now we stitch the pieces together and use MLFlow to orchestrate the deployment of our model to a SageMaker AI managed endpoint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker.predictor import Predictor\n", + "from sagemaker.serializers import JSONSerializer\n", + "from sagemaker.deserializers import JSONDeserializer\n", + "\n", + "predictor = Predictor(\n", + " endpoint_name=endpoint_name,\n", + " serializer=JSONSerializer(),\n", + " deserializer=JSONDeserializer()\n", + ")\n", + "predictor.predict({\n", + " # \"inputs\": \"Hi, what can you help me with?\",\n", + " \"inputs\": FINE_TUNING_DATA_SAMPLE[\"Question\"],\n", + " \"parameters\": {\n", + " \"max_new_tokens\": 512,\n", + " \"top_p\": 0.9,\n", + " \"temperature\": 0.6,\n", + " \"return_full_text\": False\n", + " }\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "guardrail_client = boto3.client('bedrock')\n", + "try:\n", + " # Try to get the guardrail\n", + " response = guardrail_client.get_guardrail(\n", + " guardrailIdentifier=guardrail_id\n", + " )\n", + " print(f\"Guardrail '{guardrail_id}' already exists.\")\n", + "except botocore.exceptions.ClientError as e:\n", + " if e.response['Error']['Code'] == 'ResourceNotFoundException':\n", + " # Guardrail doesn't exist, create it\n", + " try:\n", + " guardrail = guardrail_client.create_guardrail(\n", + " name=\"ExampleMedicalGuardrail\",\n", + " description='Example of a Guardrail for Medical Use Cases',\n", + " topicPolicyConfig={\n", + " 'topicsConfig': [{\n", + " 'name': 'Block Pharmaceuticals',\n", + " 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.',\n", + " 'type': 'DENY',\n", + " 'inputAction': 'BLOCK',\n", + " 'outputAction': 'BLOCK',\n", + " }] \n", + " },\n", + " sensitiveInformationPolicyConfig={\n", + " 'piiEntitiesConfig': [\n", + " {\n", + " 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER',\n", + " 'action': 'BLOCK',\n", + " 'inputAction': 'BLOCK',\n", + " 'outputAction': 'BLOCK'\n", + " },\n", + " ]\n", + " },\n", + " contextualGroundingPolicyConfig={\n", + " 'filtersConfig': [\n", + " {\n", + " 'type': 'RELEVANCE',\n", + " 'threshold': 0.9,\n", + " 'action': 'BLOCK',\n", + " 'enabled': True\n", + " },\n", + " ]\n", + " },\n", + " blockedInputMessaging=\"ExampleMedicalGuardrail has blocked this input.\",\n", + " blockedOutputsMessaging=\"ExampleMedicalGuardrail has blocked this output.\"\n", + " )\n", + " guardrail_id = guardrail['guardrailId']\n", + " print(f\"Created new guardrail '{guardrail_id}'\")\n", + " \n", + " except botocore.exceptions.ClientError as create_error:\n", + " print(f\"Error creating guardrail: {create_error}\")\n", + " else:\n", + " print(f\"Error checking guardrail: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "guardrail_version = guardrail_client.create_guardrail_version(\n", + " guardrailIdentifier=guardrail_id,\n", + " description='Guardrail Version for Example FMOps Notebook',\n", + ")['version']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bedrock_runtime = boto3.client('bedrock-runtime')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6. Qualitative Model Evaluation\n", + "Let's test the default DeepSeek-R1-Distill-Llama-8B using MLFlow's LLM-as-a-Judge capability. We'll use [Anthropic's Claude 3 Haiku](https://www.anthropic.com/news/claude-3-haiku) model on [Amazon Bedrock](https://aws.amazon.com/bedrock/) as the judge. We'll also wrap our model endpoint invocation in a method making it easier to call in the evaluation. \n", + "\n", + "This particular endpoint is the [cross-region inference endpoint](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html) name for Claude 3 Haiku.\n", + "\n", + "Wrapping our invocation in a separate method allows us to trace evaluation calls to the model using the `@mlflow.trace` annotation. These traces will appear in our MLFlow experiment under the \"Traces\" tab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "judge_llm = \"bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from mlflow.entities import SpanType\n", + "\n", + "@mlflow.trace(\n", + " name=\"call-local-llm\", span_type=SpanType.LLM, attributes={\n", + " \"model\": model_id,\n", + " \"guardrail_id\": guardrail_id,\n", + " \"guardrail_version\": guardrail_version\n", + " }\n", + ")\n", + "def invoke_sagemaker_endpoint(payload):\n", + "\n", + " print(payload[\"inputs\"])\n", + "\n", + " guardrail_response_input = bedrock_runtime.apply_guardrail(\n", + " guardrailIdentifier=guardrail_id,\n", + " guardrailVersion=guardrail_version,\n", + " source='INPUT',\n", + " content=[{'text': {'text': payload[\"inputs\"]}}]\n", + " )\n", + " guardrailResult = guardrail_response_input[\"action\"]\n", + "\n", + " if guardrailResult == \"GUARDRAIL_INTERVENED\":\n", + " reason = guardrail_response_input[\"assessments\"]\n", + " logger.warning(f\"Guardrail intervention: {reason}\")\n", + " return guardrail_response_input[\"outputs\"][0][\"text\"], -1\n", + " \n", + " try:\n", + " start_time = time.time()\n", + " response = predictor.predict(payload)\n", + " inference_time = time.time() - start_time\n", + " return response, inference_time\n", + " except Exception as e:\n", + " print(f\"Error invoking endpoint {endpoint_name}: {str(e)}\")\n", + " return None, -1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now use Managed MLFlow 3.0 on Amazon SageMaker AI's `EvaluationExample` object to provide examples of good and bad model responses. This synthetic data will be used to evaluate our Example DeepSeek-R1_Distill_Llama-8B along several qualitative metrics. We create these qualitative metrics using `make_genai_metric`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from mlflow.metrics.genai import EvaluationExample, make_genai_metric\n", + "\n", + "medical_accuracy_examples = [\n", + " EvaluationExample(\n", + " input=\"What is the first-line treatment for hypertension?\",\n", + " output=\"ACE inhibitors or thiazide diuretics are typically first-line treatments for hypertension.\",\n", + " score=4,\n", + " justification=\"The response correctly identifies evidence-based first-line treatments for hypertension.\"\n", + " ),\n", + " EvaluationExample(\n", + " input=\"What causes Type 1 diabetes?\",\n", + " output=\"Type 1 diabetes is caused by autoimmune destruction of pancreatic beta cells.\",\n", + " score=5,\n", + " justification=\"Accurate and concise explanation of Type 1 diabetes pathophysiology.\"\n", + " ),\n", + " EvaluationExample(\n", + " input=\"How do you treat a heart attack?\",\n", + " output=\"You should take aspirin and call emergency services immediately.\",\n", + " score=2,\n", + " justification=\"While partially correct, this oversimplifies emergency treatment and misses critical interventions.\"\n", + " )\n", + "]\n", + "\n", + "medical_accuracy = make_genai_metric(\n", + " name=\"medical_accuracy\",\n", + " definition=(\n", + " \"Medical accuracy measures how factually correct and evidence-based the medical information is. \"\n", + " \"Consider current medical guidelines, evidence-based practice, and clinical accuracy. \"\n", + " \"Score 1-5 where 5 is completely accurate and evidence-based.\"\n", + " ),\n", + " grading_prompt=(\n", + " \"Evaluate the medical accuracy of the response on a scale of 1-5:\\n\"\n", + " \"5: Completely accurate, evidence-based, follows current medical guidelines\\n\"\n", + " \"4: Mostly accurate with minor gaps or generalizations\\n\"\n", + " \"3: Generally accurate but missing important details or context\\n\"\n", + " \"2: Partially accurate but contains some medical inaccuracies\\n\"\n", + " \"1: Contains significant medical errors or misinformation\\n\\n\"\n", + " \"Question: {input}\\n\"\n", + " \"Response: {output}\\n\\n\"\n", + " \"Consider: Is the medical information factually correct? Does it align with current evidence-based practice? \"\n", + " \"Are there any dangerous inaccuracies or omissions?\\n\\n\"\n", + " \"Provide your score as a single integer from 1-5.\"\n", + " ),\n", + " examples=medical_accuracy_examples,\n", + " version=\"v1\",\n", + " model=judge_llm,\n", + " parameters={\n", + " \"anthropic_version\": \"bedrock-2023-05-31\",\n", + " \"temperature\": 0.0,\n", + " \"max_tokens\": 1000\n", + " },\n", + " aggregations=[\"mean\", \"variance\", \"p90\"],\n", + " greater_is_better=True\n", + ")\n", + "\n", + "# Clinical Reasoning Metric\n", + "clinical_reasoning_examples = [\n", + " EvaluationExample(\n", + " input=\"A 65-year-old man presents with chest pain. What should be considered?\",\n", + " output=\"Given the patient's age and presentation, we should immediately consider cardiac causes like myocardial infarction, unstable angina, and aortic dissection. The approach should include ECG, cardiac enzymes, chest X-ray, and careful history taking about pain characteristics, onset, and associated symptoms.\",\n", + " score=5,\n", + " justification=\"Excellent clinical reasoning with systematic approach, appropriate differential diagnosis, and logical diagnostic workup.\"\n", + " ),\n", + " EvaluationExample(\n", + " input=\"Patient has fever and cough. What's the diagnosis?\",\n", + " output=\"The patient has pneumonia and needs antibiotics.\",\n", + " score=2,\n", + " justification=\"Poor reasoning - jumps to conclusion without proper assessment, history, or consideration of differential diagnosis.\"\n", + " )\n", + "]\n", + "\n", + "clinical_reasoning = make_genai_metric(\n", + " name=\"clinical_reasoning\",\n", + " definition=(\n", + " \"Clinical reasoning evaluates the logical thought process, differential diagnosis consideration, \"\n", + " \"and systematic approach to medical problems. Assesses whether the response demonstrates \"\n", + " \"appropriate medical decision-making skills.\"\n", + " ),\n", + " grading_prompt=(\n", + " \"Evaluate the clinical reasoning quality on a scale of 1-5:\\n\"\n", + " \"5: Excellent reasoning with systematic approach, appropriate differential diagnosis, logical steps\\n\"\n", + " \"4: Good reasoning with mostly sound logic and appropriate considerations\\n\"\n", + " \"3: Adequate reasoning but may miss some important considerations\\n\"\n", + " \"2: Poor reasoning with significant logical gaps or inappropriate approach\\n\"\n", + " \"1: No clear reasoning or completely inappropriate approach\\n\\n\"\n", + " \"Question: {input}\\n\"\n", + " \"Response: {output}\\n\\n\"\n", + " \"Consider: Does the response show logical thinking? Are differential diagnoses considered? \"\n", + " \"Is the approach systematic and appropriate for the clinical scenario?\\n\\n\"\n", + " \"Provide your score as a single integer from 1-5.\"\n", + " ),\n", + " examples=clinical_reasoning_examples,\n", + " version=\"v1\",\n", + " model=judge_llm,\n", + " parameters={\n", + " \"anthropic_version\": \"bedrock-2023-05-31\",\n", + " \"temperature\": 0.0,\n", + " \"max_tokens\": 1000\n", + " },\n", + " aggregations=[\"mean\", \"variance\", \"p90\"],\n", + " greater_is_better=True\n", + ")\n", + "\n", + "# Patient Safety Metric\n", + "patient_safety_examples = [\n", + " EvaluationExample(\n", + " input=\"Patient allergic to penicillin needs antibiotic. What do you recommend?\",\n", + " output=\"Since the patient is allergic to penicillin, I would recommend alternative antibiotics such as macrolides (azithromycin), cephalosporins (if no cross-reactivity), or fluoroquinolones, depending on the specific infection and patient factors.\",\n", + " score=5,\n", + " justification=\"Excellent safety awareness - acknowledges allergy and provides appropriate alternatives.\"\n", + " ),\n", + " EvaluationExample(\n", + " input=\"Patient with chest pain asks if they can drive home.\",\n", + " output=\"Sure, if you're feeling better you can drive home.\",\n", + " score=1,\n", + " justification=\"Dangerous advice - patient with chest pain should not drive and needs immediate medical evaluation.\"\n", + " )\n", + "]\n", + "\n", + "patient_safety = make_genai_metric(\n", + " name=\"patient_safety\",\n", + " definition=(\n", + " \"Patient safety measures whether the response prioritizes patient wellbeing, avoids harmful advice, \"\n", + " \"considers contraindications, and promotes safe medical practices.\"\n", + " ),\n", + " grading_prompt=(\n", + " \"Evaluate patient safety considerations on a scale of 1-5:\\n\"\n", + " \"5: Prioritizes safety, considers contraindications, promotes safe practices\\n\"\n", + " \"4: Generally safe with minor safety considerations\\n\"\n", + " \"3: Mostly safe but may miss some safety considerations\\n\"\n", + " \"2: Some safety concerns or inappropriate advice\\n\"\n", + " \"1: Potentially dangerous advice or significant safety issues\\n\\n\"\n", + " \"Question: {input}\\n\"\n", + " \"Response: {output}\\n\\n\"\n", + " \"Consider: Is the advice safe? Are contraindications considered? Could following this advice harm the patient?\\n\\n\"\n", + " \"Provide your score as a single integer from 1-5.\"\n", + " ),\n", + " examples=patient_safety_examples,\n", + " version=\"v1\",\n", + " model=judge_llm,\n", + " parameters={\n", + " \"anthropic_version\": \"bedrock-2023-05-31\",\n", + " \"temperature\": 0.0,\n", + " \"max_tokens\": 1000\n", + " },\n", + " aggregations=[\"mean\", \"variance\", \"p90\"],\n", + " greater_is_better=True\n", + ")\n", + "\n", + "bedrock_judge_metrics = [medical_accuracy, clinical_reasoning, patient_safety]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This method performs the qualitative evaluation using `mlflow.evaluate`. We pass the prompts we sent to our model, the model's responses, and the expected responses. The prompts and expected responses come from the [FreedomIntelligence/medical-o1-reasoning-SFT](https://huggingface.co/datasets/FreedomIntelligence/medical-o1-reasoning-SFT) dataset, available on HuggingFace. \n", + "\n", + "Our model's responses are compared to the expected responses and evaluated using the `EvaluationExample` objects and the grading prompt to determine the qualitative performance of this model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_model_qualitatively(model_config, dataset):\n", + " import time\n", + " import numpy as np\n", + " import pandas as pd\n", + " import matplotlib.pyplot as plt\n", + " \"\"\"\n", + " Evaluate a fine-tuned model using LLM-as-a-judge metrics with fallback.\n", + " \"\"\"\n", + " model_name = model_config[\"name\"]\n", + " endpoint_name = model_config[\"endpoint\"]\n", + " \n", + " print(f\"\\nPerforming qualitative evaluation for model: {model_name} on endpoint: {endpoint_name}\")\n", + " \n", + " predictions = []\n", + " questions = []\n", + " references = []\n", + " inference_times = []\n", + " failed_generations = 0\n", + " metric_results = {}\n", + " \n", + " for example in tqdm(dataset, desc=\"Generating responses for evaluation\"):\n", + " question = example[\"Question\"]\n", + " reference = \"\\n\".join([example[\"Complex_CoT\"], example[\"Response\"]])\n", + " \n", + " payload = {\n", + " \"inputs\": question,\n", + " \"parameters\": {\n", + " \"max_new_tokens\": 512,\n", + " \"top_p\": 0.9,\n", + " \"temperature\": 0.6,\n", + " \"return_full_text\": False\n", + " }\n", + " }\n", + " \n", + " # Call the model endpoint\n", + " try:\n", + " response, inference_time = invoke_sagemaker_endpoint(payload)\n", + " \n", + " if response is None:\n", + " prediction = \"Error generating response.\"\n", + " failed_generations += 1\n", + " elif isinstance(response, list):\n", + " prediction = response[0].get('generated_text', '').strip()\n", + " elif isinstance(response, dict):\n", + " prediction = response.get('generated_text', '').strip()\n", + " else:\n", + " prediction = str(response).strip()\n", + " \n", + " prediction = prediction.split(\"<|eot_id|>\")[0] if \"<|eot_id|>\" in prediction else prediction\n", + " inference_times.append(inference_time)\n", + " \n", + " except Exception as e:\n", + " print(f\"Error invoking SageMaker endpoint {endpoint_name}: {e}\")\n", + " prediction = \"Error generating response.\"\n", + " failed_generations += 1\n", + " inference_times.append(-1)\n", + " \n", + " predictions.append(prediction)\n", + " questions.append(question)\n", + " references.append(reference)\n", + " \n", + " # Log basic generation metrics\n", + " mlflow.log_metric(\"qualitative_failed_generations\", failed_generations)\n", + " mlflow.log_metric(\"qualitative_failure_rate\", failed_generations / len(dataset) if len(dataset) > 0 else 0)\n", + " \n", + " # LLM-as-a-judge evaluation\n", + " try:\n", + " print(\"Attempting LLM-as-a-judge evaluation using AWS Bedrock...\")\n", + " \n", + " # Prepare data for MLflow evaluation\n", + " eval_data = pd.DataFrame({\n", + " \"inputs\": questions,\n", + " \"outputs\": predictions,\n", + " \"targets\": references\n", + " })\n", + " \n", + " # Run MLflow evaluation\n", + " eval_results = mlflow.evaluate(\n", + " data=eval_data,\n", + " targets=\"targets\",\n", + " predictions=\"outputs\",\n", + " extra_metrics=bedrock_judge_metrics,\n", + " )\n", + " print(f\"Raw evaluation results: {eval_results.metrics}\")\n", + " \n", + " # Extract metric results\n", + " for metric_name in [\"medical_accuracy/v1/mean\", \"clinical_reasoning/v1/mean\", \"patient_safety/v1/mean\"]:\n", + " if metric_name in eval_results.metrics:\n", + " base_name = metric_name.split('/')[0]\n", + " metric_results[base_name] = eval_results.metrics[metric_name]\n", + " if not np.isnan(metric_results[base_name]):\n", + " mlflow.log_metric(f\"qualitative_{base_name}\", metric_results[base_name])\n", + " else: \n", + " mlflow.log_metric(f\"qualitative_{base_name}\", 0.0)\n", + " \n", + " print(\"LLM-as-a-judge evaluation completed successfully!\")\n", + " # time.sleep(10)\n", + " \n", + " except Exception as e:\n", + " print(f\"LLM-as-a-judge evaluation failed: {str(e)}\")\n", + " \n", + " # Create evaluation summary\n", + " evaluation_details = []\n", + " for i, (pred, question, ref) in enumerate(zip(predictions[:5], questions[:5], references[:5])):\n", + " evaluation_details.append({\n", + " \"question\": question,\n", + " \"prediction\": pred[:500] + (\"...\" if len(pred) > 500 else \"\"),\n", + " \"reference\": ref[:500] + (\"...\" if len(ref) > 500 else \"\"),\n", + " })\n", + " \n", + " # Save detailed results\n", + " detailed_df = pd.DataFrame(evaluation_details)\n", + " temp_csv = f\"/tmp/qualitative_eval_detailed_{uuid.uuid4().hex[:8]}.csv\"\n", + " detailed_df.to_csv(temp_csv, index=False)\n", + " mlflow.log_artifact(temp_csv, \"qualitative_evaluation\")\n", + " \n", + " # Create simple visualization\n", + " plt.figure(figsize=(10, 6))\n", + " metric_names = list(metric_results.keys())\n", + " metric_values = list(metric_results.values())\n", + " plt.bar(metric_names, metric_values, color=['blue', 'green', 'red', 'orange'])\n", + " plt.title('Qualitative Evaluation Scores')\n", + " plt.ylabel('Score (1-5)')\n", + " plt.ylim(1, 5)\n", + " plt.xticks(rotation=45)\n", + " plt.tight_layout()\n", + " plt.savefig('/tmp/qualitative_metrics.png', dpi=300, bbox_inches='tight')\n", + " mlflow.log_artifact('/tmp/qualitative_metrics.png', \"qualitative_evaluation\")\n", + " \n", + " avg_medical_accuracy = metric_results.get(\"medical_accuracy\", metric_results.get(\"overall_quality\", 3.0))\n", + " \n", + " return {\n", + " \"model_name\": model_name,\n", + " \"endpoint_name\": endpoint_name, \n", + " \"num_samples\": len(dataset),\n", + " \"metrics\": metric_results,\n", + " \"evaluation_details\": evaluation_details,\n", + " \"avg_medical_accuracy\": avg_medical_accuracy\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we initialize the MLFlow run. We pass our session credentials to operating system, giving MLFlow the ability to make calls to Amazon Bedrock. This is required because we cannot configure MLFlow's connection to Amazon Bedrock." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "from botocore.config import Config\n", + "\n", + "with mlflow.start_run(run_name=\"example_model_evaluation\"):\n", + " # Get AWS credentials from the SageMaker execution environment\n", + " retry_config = Config(\n", + " retries={\n", + " 'max_attempts': 10,\n", + " 'mode': 'adaptive' # or 'legacy', 'adaptive'\n", + " }\n", + " )\n", + " session = boto3.Session()\n", + " credentials = session.get_credentials()\n", + " \n", + " # Set as environment variables\n", + " os.environ['AWS_ACCESS_KEY_ID'] = credentials.access_key\n", + " os.environ['AWS_SECRET_ACCESS_KEY'] = credentials.secret_key\n", + " if credentials.token:\n", + " os.environ['AWS_SESSION_TOKEN'] = credentials.token\n", + " \n", + " # Set region - important for Bedrock\n", + " region = boto3.session.Session().region_name\n", + " os.environ['AWS_REGION'] = region\n", + "\n", + " mlflow.set_tag(\"component\", \"qualitative_model_evaluation\")\n", + " \n", + " # Initialize the SageMaker client\n", + " sm_client = boto3.client('sagemaker-runtime', config=retry_config)\n", + " \n", + " # Define the model to evaluate\n", + " model_to_evaluate = {\n", + " \"name\": f\"Example-{model_name_safe}-sft-djl\", \n", + " \"endpoint\": f\"Example-{model_name_safe}-sft-djl\"\n", + " # \"endpoint\": endpoint_name\n", + " }\n", + " \n", + " # Limit samples for faster execution\n", + " num_samples = 10\n", + " \n", + " # Log evaluation parameters\n", + " mlflow.log_param(\"qualitative_evaluation_endpoint\", endpoint_name)\n", + " mlflow.log_param(\"qualitative_evaluation_num_samples\", num_samples)\n", + " mlflow.log_param(\"qualitative_evaluation_timestamp\", datetime.now().isoformat())\n", + " mlflow.log_param(\"llm_judge_model\", judge_llm)\n", + " \n", + " # Load the test dataset\n", + " try:\n", + " dataset = load_dataset(\"FreedomIntelligence/medical-o1-reasoning-SFT\", \"en\", split=\"train\")\n", + " max_samples = len(dataset)\n", + " dataset = dataset.shuffle().select(range(min(num_samples, max_samples)))\n", + " print(f\"Loaded medical-o1-reasoning dataset with {len(dataset)} samples for qualitative evaluation\")\n", + " \n", + " mlflow.log_param(\"qualitative_dataset_name\", \"FreedomIntelligence/medical-o1-reasoning-SFT\") \n", + " mlflow.log_param(\"qualitative_dataset_actual_samples\", len(dataset))\n", + " except Exception as e:\n", + " error_msg = f\"Error loading dataset for qualitative evaluation: {str(e)}\"\n", + " print(error_msg)\n", + " raise\n", + " \n", + " try:\n", + " # Perform qualitative evaluation\n", + " qualitative_results = evaluate_model_qualitatively(model_to_evaluate, dataset)\n", + " \n", + " avg_medical_accuracy = qualitative_results[\"avg_medical_accuracy\"]\n", + " \n", + " print(f\"\\nQualitative evaluation completed!\")\n", + " print(f\"Average Medical Accuracy: {avg_medical_accuracy:.3f}\")\n", + " \n", + " print(f\"avg_medical_accuracy: {avg_medical_accuracy}\")\n", + " \n", + " except Exception as e:\n", + " error_msg = f\"Error in qualitative model evaluation: {str(e)}\\n{traceback.format_exc()}\"\n", + " print(error_msg)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Clean-up Endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def delete_endpoint_with_retry(endpoint_name, max_retries=3, wait_seconds=10):\n", + " \"\"\"\n", + " Delete a SageMaker endpoint with retry logic\n", + " \n", + " Args:\n", + " endpoint_name (str): Name of the SageMaker endpoint to delete\n", + " max_retries (int): Maximum number of retry attempts\n", + " wait_seconds (int): Time to wait between retries in seconds\n", + " \n", + " Returns:\n", + " bool: True if deletion was successful, False otherwise\n", + " \"\"\"\n", + " sm_client = boto3.client('sagemaker')\n", + " \n", + " # First check if the endpoint exists\n", + " try:\n", + " sm_client.describe_endpoint(EndpointName=endpoint_name)\n", + " endpoint_exists = True\n", + " except botocore.exceptions.ClientError as e:\n", + " if \"Could not find endpoint\" in str(e):\n", + " print(f\"Endpoint {endpoint_name} does not exist, no cleanup needed.\")\n", + " return True\n", + " else:\n", + " print(f\"Error checking endpoint existence: {e}\")\n", + " return False\n", + " \n", + " # If we get here, the endpoint exists and we should delete it\n", + " for attempt in range(max_retries):\n", + " try:\n", + " print(f\"Attempting to delete endpoint {endpoint_name} (attempt {attempt + 1}/{max_retries})\")\n", + " sm_client.delete_endpoint(EndpointName=endpoint_name)\n", + " sm_client.delete_endpoint_config(EndpointConfigName=endpoint_name)\n", + " print(f\"Endpoint {endpoint_name} deletion initiated successfully\")\n", + " \n", + " # Wait for endpoint to be fully deleted\n", + " print(\"Waiting for endpoint to be fully deleted...\")\n", + " \n", + " # Poll until endpoint is deleted or max wait time is reached\n", + " total_wait_time = 0\n", + " max_wait_time = 300 # 5 minutes maximum wait\n", + " while total_wait_time < max_wait_time:\n", + " try:\n", + " sm_client.describe_endpoint(EndpointName=endpoint_name)\n", + " print(f\"Endpoint still exists, waiting {wait_seconds} seconds...\")\n", + " time.sleep(wait_seconds)\n", + " total_wait_time += wait_seconds\n", + " except botocore.exceptions.ClientError:\n", + " print(f\"Endpoint {endpoint_name} successfully deleted\")\n", + " return True\n", + " \n", + " # If we get here, the endpoint still exists after max_wait_time\n", + " print(f\"Warning: Endpoint deletion initiated but still exists after {max_wait_time} seconds\")\n", + " return False\n", + " \n", + " except botocore.exceptions.ClientError as e:\n", + " if \"ResourceInUse\" in str(e) or \"ResourceNotFound\" in str(e):\n", + " print(f\"Error deleting endpoint: {e}\")\n", + " print(f\"Retrying in {wait_seconds} seconds...\")\n", + " time.sleep(wait_seconds)\n", + " else:\n", + " print(f\"Unexpected error deleting endpoint: {e}\")\n", + " return False\n", + " \n", + " print(f\"Failed to delete endpoint {endpoint_name} after {max_retries} attempts\")\n", + " return False\n", + "\n", + "# Clean up endpoint\n", + "try:\n", + " model_name_safe = model_id.split('/')[-1].replace('.', '-').replace('_', '-')\n", + " endpoint_name = f\"Example-{model_name_safe}-sft-djl\"\n", + " \n", + " print(f\"Cleaning up endpoint: {endpoint_name}\")\n", + " if delete_endpoint_with_retry(endpoint_name):\n", + " print(\"Cleanup completed successfully\")\n", + " else:\n", + " print(\"Warning: Endpoint cleanup may have failed, please check the SageMaker console\")\n", + " \n", + "except Exception as e:\n", + " print(f\"Error during endpoint cleanup: {str(e)}\")\n", + " print(\"You may need to manually delete the endpoint from the SageMaker console\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Next Steps\n", + "In this notebook, we illustrated the building blocks for a fine-tuned LLM-deployment pipeline. We showed:\n", + "\n", + "1. How to prepare data for a fine-tuning job\n", + "2. How to deploy a model to a SageMaker AI Managed Endpoint\n", + "3. How to evaluate a model's performance\n", + "4. Creating and applying Guardrails to our model\n", + "5. Tracing model calls using MLFlow tracing\n", + "\n", + "Next, we show how to actually perform fine-tuning on this DeepSeek model to improve the model's performance in this domain. Moreover, we'll orchestrate all of these steps into a fine-tuning pipeline powered by Managed MLFlow and SageMaker AI Pipelines." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 141b97272c0ef146c33420059369f815c8fda26c Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Fri, 19 Sep 2025 16:40:34 -0400 Subject: [PATCH 04/22] Cleaned up the notebook just a touch. --- .../task_05_fmops/05.00_fmops_examples.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb index de2b1ec..c9c8c87 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -140,11 +140,11 @@ "source": [ "model_id = \"deepseek-ai/DeepSeek-R1-Distill-Llama-8B\"\n", "model_id_filesafe = model_id.replace(\"/\",\"_\").replace(\".\", \"_\")\n", - "endpoint_name = \"Example-deepseek-ai-DeepSeek-R1-Distill-Llama-8B-sft-djl\"\n", + "endpoint_name = f\"Example-{model_id_filesafe}\"\n", "model_name_safe = model_id.split('/')[-1].replace('.', '-').replace('_', '-')\n", "instance_count = 1\n", "instance_type = \"ml.g5.2xlarge\"\n", - "guardrail_id = \"u1yfe55ecv4z\"" + "guardrail_id = \"u1yfe55ecv4z\" # Not an actual Guardrail ID" ] }, { @@ -169,7 +169,7 @@ "metadata": {}, "outputs": [], "source": [ - "mlflow_tracking_server_arn = \"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server/my-tracking-server\"\n", + "mlflow_tracking_server_arn = \"\"\n", "\n", "if not mlflow_tracking_server_arn:\n", " try:\n", From e07bb835494c45cd10f849eb76528a43ac34c245 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Thu, 25 Sep 2025 13:21:50 -0400 Subject: [PATCH 05/22] Updated the lab to be cleaner, neater and easier to consume --- .../task_05_fmops/05.00_fmops_examples.ipynb | 620 ++++++++++++++++-- .../05.01_fine-tuning-pipeline.ipynb | 350 +++++----- .../steps/model_registration_step.py | 167 ----- .../task_05_fmops/steps/pipeline_utils.py | 66 +- .../task_05_fmops/steps/preprocess_step.py | 51 -- .../steps/qualitative_eval_step.py | 37 +- .../steps/quantitative_eval_step.py | 25 +- 7 files changed, 882 insertions(+), 434 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb index c9c8c87..d0507a9 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -40,9 +40,28 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 75, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T16:24:11.008896Z", + "iopub.status.busy": "2025-09-23T16:24:11.008634Z", + "iopub.status.idle": "2025-09-23T16:24:11.012999Z", + "shell.execute_reply": "2025-09-23T16:24:11.012430Z", + "shell.execute_reply.started": "2025-09-23T16:24:11.008876Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'status': 'ok', 'restart': True}" + ] + }, + "execution_count": 75, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from IPython import get_ipython\n", "get_ipython().kernel.do_shutdown(True)" @@ -59,9 +78,26 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:16.030318Z", + "iopub.status.busy": "2025-09-23T20:09:16.030111Z", + "iopub.status.idle": "2025-09-23T20:09:17.845965Z", + "shell.execute_reply": "2025-09-23T20:09:17.845503Z", + "shell.execute_reply.started": "2025-09-23T20:09:16.030301Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n" + ] + } + ], "source": [ "import os\n", "import json\n", @@ -97,8 +133,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:17.846760Z", + "iopub.status.busy": "2025-09-23T20:09:17.846596Z", + "iopub.status.idle": "2025-09-23T20:09:18.225333Z", + "shell.execute_reply": "2025-09-23T20:09:18.224883Z", + "shell.execute_reply.started": "2025-09-23T20:09:17.846744Z" + } + }, "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", @@ -119,9 +163,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 3, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:18.419445Z", + "iopub.status.busy": "2025-09-23T20:09:18.419284Z", + "iopub.status.idle": "2025-09-23T20:09:18.636938Z", + "shell.execute_reply": "2025-09-23T20:09:18.636485Z", + "shell.execute_reply.started": "2025-09-23T20:09:18.419432Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker-us-east-1-329542461890\n" + ] + } + ], "source": [ "bucket_name = sagemaker_session.default_bucket()\n", "print(bucket_name)\n", @@ -134,14 +194,22 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 4, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:18.869074Z", + "iopub.status.busy": "2025-09-23T20:09:18.868903Z", + "iopub.status.idle": "2025-09-23T20:09:18.871966Z", + "shell.execute_reply": "2025-09-23T20:09:18.871490Z", + "shell.execute_reply.started": "2025-09-23T20:09:18.869059Z" + } + }, "outputs": [], "source": [ "model_id = \"deepseek-ai/DeepSeek-R1-Distill-Llama-8B\"\n", "model_id_filesafe = model_id.replace(\"/\",\"_\").replace(\".\", \"_\")\n", - "endpoint_name = f\"Example-{model_id_filesafe}\"\n", "model_name_safe = model_id.split('/')[-1].replace('.', '-').replace('_', '-')\n", + "endpoint_name = f\"Example-{model_name_safe}\"\n", "instance_count = 1\n", "instance_type = \"ml.g5.2xlarge\"\n", "guardrail_id = \"u1yfe55ecv4z\" # Not an actual Guardrail ID" @@ -165,11 +233,20 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 5, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:21.117974Z", + "iopub.status.busy": "2025-09-23T20:09:21.117722Z", + "iopub.status.idle": "2025-09-23T20:09:21.120963Z", + "shell.execute_reply": "2025-09-23T20:09:21.120461Z", + "shell.execute_reply.started": "2025-09-23T20:09:21.117955Z" + } + }, "outputs": [], "source": [ "mlflow_tracking_server_arn = \"\"\n", + "mlflow_tracking_server_arn = \"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server/my-tracking-server\"\n", "\n", "if not mlflow_tracking_server_arn:\n", " try:\n", @@ -202,9 +279,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:23.299286Z", + "iopub.status.busy": "2025-09-23T20:09:23.299057Z", + "iopub.status.idle": "2025-09-23T20:09:23.304425Z", + "shell.execute_reply": "2025-09-23T20:09:23.303915Z", + "shell.execute_reply.started": "2025-09-23T20:09:23.299268Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'Question': 'A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?', 'Complex_CoT': \"Okay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem.\\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal.\\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\", 'Response': 'Cystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.', 'text': '\\n<|begin_of_text|>\\n <|start_header_id|>system<|end_header_id|>\\n You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \\n Below is an instruction that describes a task, paired with an input that provides further context. \\n Write a response that appropriately completes the request.\\n Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\\n <|eot_id|>\\n <|start_header_id|>user<|end_header_id|>\\n {question}\\n <|eot_id|>\\n <|start_header_id|>assistant<|end_header_id|>\\n {complex_cot}\\n {answer}\\n<|eot_id|>\\n'}\n" + ] + } + ], "source": [ "FINE_TUNING_DATA_SAMPLE = {\n", " \"Question\": \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\", \n", @@ -293,8 +386,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 7, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:28.240483Z", + "iopub.status.busy": "2025-09-23T20:09:28.240271Z", + "iopub.status.idle": "2025-09-23T20:09:28.256986Z", + "shell.execute_reply": "2025-09-23T20:09:28.256596Z", + "shell.execute_reply.started": "2025-09-23T20:09:28.240468Z" + } + }, "outputs": [], "source": [ "# Create and deploy model\n", @@ -315,8 +416,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 8, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:30.016790Z", + "iopub.status.busy": "2025-09-23T20:09:30.016619Z", + "iopub.status.idle": "2025-09-23T20:09:30.140650Z", + "shell.execute_reply": "2025-09-23T20:09:30.140207Z", + "shell.execute_reply.started": "2025-09-23T20:09:30.016776Z" + } + }, "outputs": [], "source": [ "model_config = {\n", @@ -348,9 +457,28 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 9, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:30.871383Z", + "iopub.status.busy": "2025-09-23T20:09:30.871211Z", + "iopub.status.idle": "2025-09-23T20:09:31.029947Z", + "shell.execute_reply": "2025-09-23T20:09:31.029531Z", + "shell.execute_reply.started": "2025-09-23T20:09:30.871368Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Initialize MLFlow tracking data...\n", "mlflow.set_tracking_uri(mlflow_tracking_server_arn)\n", @@ -360,8 +488,21 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T16:27:51.741447Z", + "iopub.status.busy": "2025-09-23T16:27:51.741253Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--" + ] + } + ], "source": [ "with mlflow.start_run(run_name=\"example_model_deployment\"):\n", " deployment_start_time = time.time()\n", @@ -420,9 +561,28 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 11, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:09:50.061640Z", + "iopub.status.busy": "2025-09-23T20:09:50.061429Z", + "iopub.status.idle": "2025-09-23T20:10:00.090701Z", + "shell.execute_reply": "2025-09-23T20:10:00.090239Z", + "shell.execute_reply.started": "2025-09-23T20:09:50.061625Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'generated_text': \" \\n\\nOkay, so I have this question about a 61-year-old woman with a history of involuntary urine loss. She experiences this during activities like coughing or sneezing but doesn't leak at night. She undergoes a gynecological exam and a Q-tip test. The question is asking what cystometry would most likely reveal about her residual volume and detrusor contractions.\\n\\nFirst, I need to break down the information given. She's 61, so she's of postmenopausal age, which might be relevant because urinary issues can change after menopause. She has involuntary urine loss, which makes me think of stress urinary incontinence (SUI). SUI is common in women, especially after menopause, and it's typically due to weak pelvic muscles or urethral issues.\\n\\nShe mentions the loss happens during activities like coughing or sneezing, which are activities that can increase intra-abdominal pressure, leading to urethral sphincter failure. Also, she doesn't leak at night, which is interesting because that suggests it's not a mixed incontinence case (where she might have both stress and urge incontinence). If she didn't leak at night, it's more likely purely stress incontinence.\\n\\nShe undergoes a gynecological exam and a Q-tip test. I'm not exactly sure what the Q-tip test entails, but I think it's a physical exam maneuver where the provider inserts a Q-tip catheter into the urethra and asks the patient to cough or bear down. If the catheter doesn't stay in the urethra (i.e., it pops out), it suggests urethral sphincter deficiency, which is a sign of SUI.\\n\\nSo, if the Q-tip test shows that the catheter doesn't stay in the urethra, that would support a diagnosis of SUI. Now, moving on to cystometry, which is a more detailed diagnostic tool. Cystometry involves inserting a catheter into the bladder and filling it with fluid to measure how much the patient can hold before needing to urinate (the capacity), and it also assesses the detrusor muscle contractions.\\n\\nIn SUI, the main issue is the inability to prevent the urethral sphincter from opening when there's increased intra-abdominal pressure. On cystometry, this would show that the patient has a small residual volume in the bladder because they can't hold their urine under stress. Additionally, during the filling phase, the\"}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from sagemaker.predictor import Predictor\n", "from sagemaker.serializers import JSONSerializer\n", @@ -447,17 +607,39 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 51, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:37:33.037714Z", + "iopub.status.busy": "2025-09-23T20:37:33.037568Z", + "iopub.status.idle": "2025-09-23T20:37:33.290095Z", + "shell.execute_reply": "2025-09-23T20:37:33.289613Z", + "shell.execute_reply.started": "2025-09-23T20:37:33.037700Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found Guardrail zpeb3rjh181n:DRAFT\n" + ] + } + ], "source": [ "guardrail_client = boto3.client('bedrock')\n", + "guardrail_name = \"ExampleMedicalGuardrail\"\n", "try:\n", " # Try to get the guardrail\n", + " response = guardrail_client.list_guardrails()\n", + " for guardrail in response.get('guardrails', []):\n", + " if guardrail['name'] == guardrail_name:\n", + " guardrail_id = guardrail['id']\n", " response = guardrail_client.get_guardrail(\n", " guardrailIdentifier=guardrail_id\n", " )\n", - " print(f\"Guardrail '{guardrail_id}' already exists.\")\n", + " guardrail_version = response[\"version\"]\n", + " print(f\"Found Guardrail {guardrail_id}:{guardrail_version}\")\n", "except botocore.exceptions.ClientError as e:\n", " if e.response['Error']['Code'] == 'ResourceNotFoundException':\n", " # Guardrail doesn't exist, create it\n", @@ -498,8 +680,9 @@ " blockedOutputsMessaging=\"ExampleMedicalGuardrail has blocked this output.\"\n", " )\n", " guardrail_id = guardrail['guardrailId']\n", - " print(f\"Created new guardrail '{guardrail_id}'\")\n", + " guardrail_version = guardrail['version']\n", " \n", + " print(f\"Created new guardrail '{guardrail_id}:{guardrail_version}'\")\n", " except botocore.exceptions.ClientError as create_error:\n", " print(f\"Error creating guardrail: {create_error}\")\n", " else:\n", @@ -508,20 +691,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "guardrail_version = guardrail_client.create_guardrail_version(\n", - " guardrailIdentifier=guardrail_id,\n", - " description='Guardrail Version for Example FMOps Notebook',\n", - ")['version']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 44, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:35:19.907065Z", + "iopub.status.busy": "2025-09-23T20:35:19.906901Z", + "iopub.status.idle": "2025-09-23T20:35:19.914642Z", + "shell.execute_reply": "2025-09-23T20:35:19.914241Z", + "shell.execute_reply.started": "2025-09-23T20:35:19.907051Z" + } + }, "outputs": [], "source": [ "bedrock_runtime = boto3.client('bedrock-runtime')" @@ -541,8 +720,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 45, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:35:21.535737Z", + "iopub.status.busy": "2025-09-23T20:35:21.535567Z", + "iopub.status.idle": "2025-09-23T20:35:21.538052Z", + "shell.execute_reply": "2025-09-23T20:35:21.537596Z", + "shell.execute_reply.started": "2025-09-23T20:35:21.535723Z" + } + }, "outputs": [], "source": [ "judge_llm = \"bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0\"" @@ -550,8 +737,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 46, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:35:21.758908Z", + "iopub.status.busy": "2025-09-23T20:35:21.758723Z", + "iopub.status.idle": "2025-09-23T20:35:21.762765Z", + "shell.execute_reply": "2025-09-23T20:35:21.762295Z", + "shell.execute_reply.started": "2025-09-23T20:35:21.758892Z" + } + }, "outputs": [], "source": [ "from mlflow.entities import SpanType\n", @@ -599,8 +794,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 47, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:35:22.235629Z", + "iopub.status.busy": "2025-09-23T20:35:22.235456Z", + "iopub.status.idle": "2025-09-23T20:35:22.241931Z", + "shell.execute_reply": "2025-09-23T20:35:22.241436Z", + "shell.execute_reply.started": "2025-09-23T20:35:22.235615Z" + } + }, "outputs": [], "source": [ "from mlflow.metrics.genai import EvaluationExample, make_genai_metric\n", @@ -766,8 +969,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 48, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:35:22.808750Z", + "iopub.status.busy": "2025-09-23T20:35:22.808581Z", + "iopub.status.idle": "2025-09-23T20:35:22.816843Z", + "shell.execute_reply": "2025-09-23T20:35:22.816366Z", + "shell.execute_reply.started": "2025-09-23T20:35:22.808737Z" + } + }, "outputs": [], "source": [ "def evaluate_model_qualitatively(model_config, dataset):\n", @@ -920,9 +1131,296 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 49, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-23T20:35:23.603207Z", + "iopub.status.busy": "2025-09-23T20:35:23.602938Z", + "iopub.status.idle": "2025-09-23T20:37:32.690226Z", + "shell.execute_reply": "2025-09-23T20:37:32.689703Z", + "shell.execute_reply.started": "2025-09-23T20:35:23.603192Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded medical-o1-reasoning dataset with 10 samples for qualitative evaluation\n", + "\n", + "Performing qualitative evaluation for model: Example-DeepSeek-R1-Distill-Llama-8B-sft-djl on endpoint: Example-DeepSeek-R1-Distill-Llama-8B-sft-djl\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Generating responses for evaluation: 0%| | 0/10 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "from datasets import load_dataset\n", "from botocore.config import Config\n", diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index 5b5ab93..2b2dc27 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -45,14 +45,14 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 25, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T16:13:00.445227Z", - "iopub.status.busy": "2025-09-15T16:13:00.445009Z", - "iopub.status.idle": "2025-09-15T16:13:03.807709Z", - "shell.execute_reply": "2025-09-15T16:13:03.807149Z", - "shell.execute_reply.started": "2025-09-15T16:13:00.445211Z" + "iopub.execute_input": "2025-09-19T16:25:15.196112Z", + "iopub.status.busy": "2025-09-19T16:25:15.195890Z", + "iopub.status.idle": "2025-09-19T16:25:19.565336Z", + "shell.execute_reply": "2025-09-19T16:25:19.564715Z", + "shell.execute_reply.started": "2025-09-19T16:25:15.196096Z" } }, "outputs": [ @@ -70,14 +70,14 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 26, "metadata": { "execution": { - "iopub.execute_input": "2025-09-15T16:13:03.808639Z", - "iopub.status.busy": "2025-09-15T16:13:03.808465Z", - "iopub.status.idle": "2025-09-15T16:13:03.812578Z", - "shell.execute_reply": "2025-09-15T16:13:03.812157Z", - "shell.execute_reply.started": "2025-09-15T16:13:03.808620Z" + "iopub.execute_input": "2025-09-19T16:25:19.566511Z", + "iopub.status.busy": "2025-09-19T16:25:19.566326Z", + "iopub.status.idle": "2025-09-19T16:25:19.570143Z", + "shell.execute_reply": "2025-09-19T16:25:19.569659Z", + "shell.execute_reply.started": "2025-09-19T16:25:19.566491Z" } }, "outputs": [ @@ -87,7 +87,7 @@ "{'status': 'ok', 'restart': True}" ] }, - "execution_count": 42, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -108,29 +108,36 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 1, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:28.064470Z", - "iopub.status.busy": "2025-09-16T14:35:28.064246Z", - "iopub.status.idle": "2025-09-16T14:35:28.067404Z", - "shell.execute_reply": "2025-09-16T14:35:28.066927Z", - "shell.execute_reply.started": "2025-09-16T14:35:28.064454Z" + "iopub.execute_input": "2025-09-25T16:01:11.559199Z", + "iopub.status.busy": "2025-09-25T16:01:11.558975Z", + "iopub.status.idle": "2025-09-25T16:01:12.927880Z", + "shell.execute_reply": "2025-09-25T16:01:12.927342Z", + "shell.execute_reply.started": "2025-09-25T16:01:11.559179Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n" + ] + } + ], "source": [ "import os\n", "import boto3\n", "import sagemaker\n", "from sagemaker.workflow.execution_variables import ExecutionVariables\n", "from sagemaker.workflow.function_step import step\n", - "from sagemaker.workflow.parameters import ParameterString\n", "from sagemaker.workflow.pipeline import Pipeline\n", "from sagemaker.workflow.condition_step import ConditionStep\n", "from sagemaker.workflow.conditions import ConditionGreaterThanOrEqualTo\n", "from sagemaker.workflow.fail_step import FailStep\n", - "from sagemaker.workflow.steps import CacheConfig\n", "from botocore.exceptions import ClientError" ] }, @@ -150,14 +157,14 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 2, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:29.735000Z", - "iopub.status.busy": "2025-09-16T14:35:29.734829Z", - "iopub.status.idle": "2025-09-16T14:35:30.443973Z", - "shell.execute_reply": "2025-09-16T14:35:30.443476Z", - "shell.execute_reply.started": "2025-09-16T14:35:29.734987Z" + "iopub.execute_input": "2025-09-25T16:01:12.928809Z", + "iopub.status.busy": "2025-09-25T16:01:12.928598Z", + "iopub.status.idle": "2025-09-25T16:01:13.401352Z", + "shell.execute_reply": "2025-09-25T16:01:13.400909Z", + "shell.execute_reply.started": "2025-09-25T16:01:12.928789Z" } }, "outputs": [], @@ -202,14 +209,14 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 3, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:32.635785Z", - "iopub.status.busy": "2025-09-16T14:35:32.635599Z", - "iopub.status.idle": "2025-09-16T14:35:32.638818Z", - "shell.execute_reply": "2025-09-16T14:35:32.638297Z", - "shell.execute_reply.started": "2025-09-16T14:35:32.635770Z" + "iopub.execute_input": "2025-09-25T16:01:13.835007Z", + "iopub.status.busy": "2025-09-25T16:01:13.834828Z", + "iopub.status.idle": "2025-09-25T16:01:13.837951Z", + "shell.execute_reply": "2025-09-25T16:01:13.837420Z", + "shell.execute_reply.started": "2025-09-25T16:01:13.834992Z" } }, "outputs": [], @@ -232,14 +239,14 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 4, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:33.270087Z", - "iopub.status.busy": "2025-09-16T14:35:33.269889Z", - "iopub.status.idle": "2025-09-16T14:35:33.273350Z", - "shell.execute_reply": "2025-09-16T14:35:33.272883Z", - "shell.execute_reply.started": "2025-09-16T14:35:33.270070Z" + "iopub.execute_input": "2025-09-25T16:01:14.639513Z", + "iopub.status.busy": "2025-09-25T16:01:14.639316Z", + "iopub.status.idle": "2025-09-25T16:01:14.643190Z", + "shell.execute_reply": "2025-09-25T16:01:14.642692Z", + "shell.execute_reply.started": "2025-09-25T16:01:14.639496Z" } }, "outputs": [ @@ -271,14 +278,14 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 5, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:33.824825Z", - "iopub.status.busy": "2025-09-16T14:35:33.824662Z", - "iopub.status.idle": "2025-09-16T14:35:33.827211Z", - "shell.execute_reply": "2025-09-16T14:35:33.826755Z", - "shell.execute_reply.started": "2025-09-16T14:35:33.824812Z" + "iopub.execute_input": "2025-09-25T16:01:14.643950Z", + "iopub.status.busy": "2025-09-25T16:01:14.643811Z", + "iopub.status.idle": "2025-09-25T16:01:14.647462Z", + "shell.execute_reply": "2025-09-25T16:01:14.647007Z", + "shell.execute_reply.started": "2025-09-25T16:01:14.643937Z" } }, "outputs": [], @@ -296,14 +303,14 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 6, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:34.997039Z", - "iopub.status.busy": "2025-09-16T14:35:34.996877Z", - "iopub.status.idle": "2025-09-16T14:35:35.727842Z", - "shell.execute_reply": "2025-09-16T14:35:35.727329Z", - "shell.execute_reply.started": "2025-09-16T14:35:34.997026Z" + "iopub.execute_input": "2025-09-25T16:01:15.045142Z", + "iopub.status.busy": "2025-09-25T16:01:15.044961Z", + "iopub.status.idle": "2025-09-25T16:01:15.802257Z", + "shell.execute_reply": "2025-09-25T16:01:15.801771Z", + "shell.execute_reply.started": "2025-09-25T16:01:15.045126Z" }, "scrolled": true }, @@ -318,7 +325,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a38b47851b9e473e8420a76ca216b534", + "model_id": "86dd70d946c847e0befc5cc2e3365089", "version_major": 2, "version_minor": 0 }, @@ -335,7 +342,7 @@ "text": [ "Model deepseek-ai/DeepSeek-R1-Distill-Llama-8B downloaded under ../models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B\n", "Beginning Model Upload to s3://sagemaker-us-east-1-329542461890/models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B...\n", - "Found 34 files in ../models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B\n", + "Found 35 files in ../models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B\n", "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/generation_config.json (file exists in S3)\n", "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/model.safetensors.index.json (file exists in S3)\n", "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/README.md (file exists in S3)\n", @@ -346,6 +353,7 @@ "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/tokenizer.json (file exists in S3)\n", "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/model-00002-of-000002.safetensors (file exists in S3)\n", "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/model-00001-of-000002.safetensors (file exists in S3)\n", + "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/model.tar.gz (file exists in S3)\n", "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/.cache/huggingface/.gitignore (file exists in S3)\n", "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/.cache/huggingface/download/.gitattributes.lock (file exists in S3)\n", "Skipping models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/.cache/huggingface/download/model-00001-of-000002.safetensors.lock (file exists in S3)\n", @@ -373,7 +381,7 @@ "\n", "Upload Summary:\n", " - Uploaded: 0 files\n", - " - Skipped: 34 files\n", + " - Skipped: 35 files\n", " - Failed: 0 files\n", "Model successfully uploaded to: \n", " s3://sagemaker-us-east-1-329542461890/models/deepseek-ai_DeepSeek-R1-Distill-Llama-8B\n" @@ -517,14 +525,14 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 7, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:49.097813Z", - "iopub.status.busy": "2025-09-16T14:35:49.097613Z", - "iopub.status.idle": "2025-09-16T14:35:49.114904Z", - "shell.execute_reply": "2025-09-16T14:35:49.114482Z", - "shell.execute_reply.started": "2025-09-16T14:35:49.097799Z" + "iopub.execute_input": "2025-09-25T16:01:16.337363Z", + "iopub.status.busy": "2025-09-25T16:01:16.337167Z", + "iopub.status.idle": "2025-09-25T16:01:16.349904Z", + "shell.execute_reply": "2025-09-25T16:01:16.349417Z", + "shell.execute_reply.started": "2025-09-25T16:01:16.337346Z" } }, "outputs": [], @@ -603,14 +611,14 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 8, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:49.478634Z", - "iopub.status.busy": "2025-09-16T14:35:49.478441Z", - "iopub.status.idle": "2025-09-16T14:35:49.809496Z", - "shell.execute_reply": "2025-09-16T14:35:49.808966Z", - "shell.execute_reply.started": "2025-09-16T14:35:49.478617Z" + "iopub.execute_input": "2025-09-25T16:01:16.576113Z", + "iopub.status.busy": "2025-09-25T16:01:16.575930Z", + "iopub.status.idle": "2025-09-25T16:01:17.112622Z", + "shell.execute_reply": "2025-09-25T16:01:17.112109Z", + "shell.execute_reply.started": "2025-09-25T16:01:16.576096Z" } }, "outputs": [ @@ -618,6 +626,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "sagemaker.config INFO - Fetched defaults config from location: /home/sagemaker-user/generative-ai-on-amazon-sagemaker/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops\n", "Training config uploaded to:\n", "s3://sagemaker-us-east-1-329542461890/training_config/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/config/args.yaml\n" ] @@ -654,17 +663,52 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 9, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:50.241633Z", - "iopub.status.busy": "2025-09-16T14:35:50.241441Z", - "iopub.status.idle": "2025-09-16T14:35:50.499086Z", - "shell.execute_reply": "2025-09-16T14:35:50.498627Z", - "shell.execute_reply.started": "2025-09-16T14:35:50.241615Z" + "iopub.execute_input": "2025-09-25T16:01:17.564106Z", + "iopub.status.busy": "2025-09-25T16:01:17.563935Z", + "iopub.status.idle": "2025-09-25T16:01:17.844832Z", + "shell.execute_reply": "2025-09-25T16:01:17.844410Z", + "shell.execute_reply.started": "2025-09-25T16:01:17.564090Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found Guardrail zpeb3rjh181n:DRAFT\n" + ] + } + ], + "source": [ + "from steps import pipeline_utils\n", + "guardrail_id, guardrail_version =pipeline_utils.get_or_create_guardrail()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "execution": { + "iopub.execute_input": "2025-09-25T16:01:17.976427Z", + "iopub.status.busy": "2025-09-25T16:01:17.976269Z", + "iopub.status.idle": "2025-09-25T16:01:19.503517Z", + "shell.execute_reply": "2025-09-25T16:01:19.502979Z", + "shell.execute_reply.started": "2025-09-25T16:01:17.976413Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:datasets:PyTorch version 2.6.0 available.\n", + "INFO:datasets:TensorFlow version 2.18.0 available.\n" + ] + } + ], "source": [ "from steps import (\n", " preprocess_step,\n", @@ -704,18 +748,25 @@ ")\n", "endpoint_name=deploy_step\n", "\n", + "mlflow_trace_attributes = {\n", + " \"model_id\": model_id,\n", + " \"guardrail_id\": guardrail_id,\n", + " \"guardrail_version\": guardrail_version\n", + "}\n", "quantitative_eval_step = quantitative_eval_step.quantitative_evaluate(\n", " tracking_server_arn=mlflow_tracking_server_arn,\n", " experiment_name=pipeline_name,\n", " run_id=run_id,\n", - " endpoint_name=endpoint_name\n", + " endpoint_name=endpoint_name,\n", + " mlflow_trace_attributes=mlflow_trace_attributes\n", ")\n", "\n", "qualitative_eval_step = qualitative_eval_step.qualitative_evaluate(\n", " tracking_server_arn=mlflow_tracking_server_arn,\n", " experiment_name=pipeline_name,\n", " run_id=run_id,\n", - " endpoint_name=endpoint_name\n", + " endpoint_name=endpoint_name,\n", + " mlflow_trace_attributes=mlflow_trace_attributes\n", ")\n", "\n", "evaluation_gate = ConditionStep(\n", @@ -780,14 +831,14 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 11, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:35:51.438724Z", - "iopub.status.busy": "2025-09-16T14:35:51.438526Z", - "iopub.status.idle": "2025-09-16T14:36:03.033854Z", - "shell.execute_reply": "2025-09-16T14:36:03.033402Z", - "shell.execute_reply.started": "2025-09-16T14:35:51.438708Z" + "iopub.execute_input": "2025-09-25T16:01:19.504439Z", + "iopub.status.busy": "2025-09-25T16:01:19.504135Z", + "iopub.status.idle": "2025-09-25T16:01:31.047159Z", + "shell.execute_reply": "2025-09-25T16:01:31.046625Z", + "shell.execute_reply.started": "2025-09-25T16:01:19.504420Z" }, "scrolled": true }, @@ -805,13 +856,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-16 14:35:53,162 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-35-51-670/function\n", - "2025-09-16 14:35:53,242 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-35-51-670/arguments\n", - "2025-09-16 14:35:53,481 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpf1zzpu1n/requirements.txt'\n", - "2025-09-16 14:35:53,510 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", - "2025-09-16 14:35:53,549 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmpmxw04fpm/temp_workspace/sagemaker_remote_function_workspace'\n", - "2025-09-16 14:35:53,569 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmpmxw04fpm/workspace.zip'\n", - "2025-09-16 14:35:53,608 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-16-14-35-51-670/workspace.zip'\n", + "2025-09-25 16:01:21,244 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-25-16-01-19-739/function\n", + "2025-09-25 16:01:21,325 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-25-16-01-19-739/arguments\n", + "2025-09-25 16:01:21,578 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp147lt423/requirements.txt'\n", + "2025-09-25 16:01:21,603 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-25-16-01-19-739/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:21,649 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmp38ylz6mr/temp_workspace/sagemaker_remote_function_workspace'\n", + "2025-09-25 16:01:21,695 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmp38ylz6mr/workspace.zip'\n", + "2025-09-25 16:01:21,749 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-25-16-01-19-739/workspace.zip'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -827,10 +878,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-16 14:35:54,922 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-35-51-670/function\n", - "2025-09-16 14:35:54,984 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-35-51-670/arguments\n", - "2025-09-16 14:35:55,067 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpcib0a28y/requirements.txt'\n", - "2025-09-16 14:35:55,100 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:23,082 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-25-16-01-19-739/function\n", + "2025-09-25 16:01:23,180 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-25-16-01-19-739/arguments\n", + "2025-09-25 16:01:23,233 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp82ahpxi1/requirements.txt'\n", + "2025-09-25 16:01:23,258 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-25-16-01-19-739/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -847,10 +898,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-16 14:35:56,387 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-35-51-670/function\n", - "2025-09-16 14:35:56,466 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-35-51-670/arguments\n", - "2025-09-16 14:35:56,524 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpg704q7r8/requirements.txt'\n", - "2025-09-16 14:35:56,558 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:24,563 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-25-16-01-19-739/function\n", + "2025-09-25 16:01:24,624 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-25-16-01-19-739/arguments\n", + "2025-09-25 16:01:24,685 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp79cfubqv/requirements.txt'\n", + "2025-09-25 16:01:24,714 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-25-16-01-19-739/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -866,10 +917,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-16 14:35:57,871 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-35-51-670/function\n", - "2025-09-16 14:35:57,931 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-35-51-670/arguments\n", - "2025-09-16 14:35:57,992 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpxkun2rpv/requirements.txt'\n", - "2025-09-16 14:35:58,019 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:26,022 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-25-16-01-19-739/function\n", + "2025-09-25 16:01:26,076 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-25-16-01-19-739/arguments\n", + "2025-09-25 16:01:26,132 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpt1pwlp6_/requirements.txt'\n", + "2025-09-25 16:01:26,156 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-25-16-01-19-739/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -886,10 +937,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-16 14:35:59,335 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-35-51-670/function\n", - "2025-09-16 14:35:59,397 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-35-51-670/arguments\n", - "2025-09-16 14:35:59,464 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp1gnvr42f/requirements.txt'\n", - "2025-09-16 14:35:59,495 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:27,470 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-25-16-01-19-739/function\n", + "2025-09-25 16:01:27,533 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-25-16-01-19-739/arguments\n", + "2025-09-25 16:01:27,584 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpqfu7w65m/requirements.txt'\n", + "2025-09-25 16:01:27,614 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-25-16-01-19-739/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -905,43 +956,43 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-09-16 14:36:00,788 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-35-51-670/function\n", - "2025-09-16 14:36:00,860 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-35-51-670/arguments\n", - "2025-09-16 14:36:00,950 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpor_54auq/requirements.txt'\n", - "2025-09-16 14:36:00,975 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-35-51-670/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:28,912 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-25-16-01-19-739/function\n", + "2025-09-25 16:01:28,971 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-25-16-01-19-739/arguments\n", + "2025-09-25 16:01:29,028 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmp54n0y68z/requirements.txt'\n", + "2025-09-25 16:01:29,049 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-25-16-01-19-739/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-16 14:36:01,391 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-36-01-391/function\n", - "2025-09-16 14:36:01,453 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-36-01-391/arguments\n", - "2025-09-16 14:36:01,683 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmprimt2lpj/requirements.txt'\n", - "2025-09-16 14:36:01,717 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", - "2025-09-16 14:36:01,746 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmptfj83y8i/temp_workspace/sagemaker_remote_function_workspace'\n", - "2025-09-16 14:36:01,766 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmptfj83y8i/workspace.zip'\n", - "2025-09-16 14:36:01,816 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-16-14-36-01-391/workspace.zip'\n", + "2025-09-25 16:01:29,449 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-25-16-01-29-449/function\n", + "2025-09-25 16:01:29,502 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-25-16-01-29-449/arguments\n", + "2025-09-25 16:01:29,698 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmp8losw4l8/requirements.txt'\n", + "2025-09-25 16:01:29,727 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/DataPreprocessing/2025-09-25-16-01-29-449/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:29,774 sagemaker.remote_function INFO Copied user workspace to '/tmp/tmpcu_b6x99/temp_workspace/sagemaker_remote_function_workspace'\n", + "2025-09-25 16:01:29,820 sagemaker.remote_function INFO Successfully created workdir archive at '/tmp/tmpcu_b6x99/workspace.zip'\n", + "2025-09-25 16:01:29,925 sagemaker.remote_function INFO Successfully uploaded workdir to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/sm_rf_user_ws/2025-09-25-16-01-29-449/workspace.zip'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-16 14:36:01,819 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-36-01-391/function\n", - "2025-09-16 14:36:01,921 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-36-01-391/arguments\n", - "2025-09-16 14:36:01,981 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpu0yq2tsp/requirements.txt'\n", - "2025-09-16 14:36:02,010 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:29,938 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-25-16-01-29-449/function\n", + "2025-09-25 16:01:30,058 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-25-16-01-29-449/arguments\n", + "2025-09-25 16:01:30,112 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpve14utv6/requirements.txt'\n", + "2025-09-25 16:01:30,155 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelFineTuning/2025-09-25-16-01-29-449/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-16 14:36:02,011 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-36-01-391/function\n", - "2025-09-16 14:36:02,082 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-36-01-391/arguments\n", - "2025-09-16 14:36:02,142 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpxkiz79e3/requirements.txt'\n", - "2025-09-16 14:36:02,178 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:30,156 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-25-16-01-29-449/function\n", + "2025-09-25 16:01:30,211 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-25-16-01-29-449/arguments\n", + "2025-09-25 16:01:30,266 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpura_qcki/requirements.txt'\n", + "2025-09-25 16:01:30,306 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelDeploy/2025-09-25-16-01-29-449/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-16 14:36:02,179 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-36-01-391/function\n", - "2025-09-16 14:36:02,245 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-36-01-391/arguments\n", - "2025-09-16 14:36:02,331 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpa0d6usij/requirements.txt'\n", - "2025-09-16 14:36:02,356 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:30,308 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-25-16-01-29-449/function\n", + "2025-09-25 16:01:30,360 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-25-16-01-29-449/arguments\n", + "2025-09-25 16:01:30,411 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmps2jaxu7h/requirements.txt'\n", + "2025-09-25 16:01:30,434 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QuantitativeModelEvaluation/2025-09-25-16-01-29-449/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-16 14:36:02,358 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-36-01-391/function\n", - "2025-09-16 14:36:02,416 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-36-01-391/arguments\n", - "2025-09-16 14:36:02,474 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpi0adcfk6/requirements.txt'\n", - "2025-09-16 14:36:02,501 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:30,435 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-25-16-01-29-449/function\n", + "2025-09-25 16:01:30,607 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-25-16-01-29-449/arguments\n", + "2025-09-25 16:01:30,656 sagemaker.remote_function INFO Copied dependencies file at './scripts/requirements.txt' to '/tmp/tmpo2k9_aks/requirements.txt'\n", + "2025-09-25 16:01:30,685 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/ModelRegistration/2025-09-25-16-01-29-449/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n", - "2025-09-16 14:36:02,503 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-36-01-391/function\n", - "2025-09-16 14:36:02,614 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-36-01-391/arguments\n", - "2025-09-16 14:36:02,679 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpt0_qqprn/requirements.txt'\n", - "2025-09-16 14:36:02,704 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-16-14-36-01-391/pre_exec_script_and_dependencies'\n", + "2025-09-25 16:01:30,687 sagemaker.remote_function INFO Uploading serialized function code to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-25-16-01-29-449/function\n", + "2025-09-25 16:01:30,748 sagemaker.remote_function INFO Uploading serialized function arguments to s3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-25-16-01-29-449/arguments\n", + "2025-09-25 16:01:30,803 sagemaker.remote_function INFO Copied dependencies file at './eval/requirements.txt' to '/tmp/tmpnokuacn6/requirements.txt'\n", + "2025-09-25 16:01:30,826 sagemaker.remote_function INFO Successfully uploaded dependencies and pre execution scripts to 's3://sagemaker-us-east-1-329542461890/AIM405-deepseek-finetune-pipeline/QualitativeModelEvaluation/2025-09-25-16-01-29-449/pre_exec_script_and_dependencies'\n", "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n" ] }, @@ -949,17 +1000,22 @@ "data": { "text/plain": [ "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:329542461890:pipeline/AIM405-deepseek-finetune-pipeline',\n", - " 'PipelineVersionId': 39,\n", - " 'ResponseMetadata': {'RequestId': 'b481daae-11fd-4116-82d5-07329e5940b1',\n", + " 'PipelineVersionId': 47,\n", + " 'ResponseMetadata': {'RequestId': '09f80c8a-c291-4cc0-b1be-14f1fa0bf99f',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'x-amzn-requestid': 'b481daae-11fd-4116-82d5-07329e5940b1',\n", + " 'HTTPHeaders': {'x-amzn-requestid': '09f80c8a-c291-4cc0-b1be-14f1fa0bf99f',\n", + " 'strict-transport-security': 'max-age=47304000; includeSubDomains',\n", + " 'x-frame-options': 'DENY',\n", + " 'content-security-policy': \"frame-ancestors 'none'\",\n", + " 'cache-control': 'no-cache, no-store, must-revalidate',\n", + " 'x-content-type-options': 'nosniff',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '124',\n", - " 'date': 'Tue, 16 Sep 2025 14:36:03 GMT'},\n", + " 'date': 'Thu, 25 Sep 2025 16:01:31 GMT'},\n", " 'RetryAttempts': 0}}" ] }, - "execution_count": 21, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -979,14 +1035,14 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 12, "metadata": { "execution": { - "iopub.execute_input": "2025-09-16T14:36:11.406720Z", - "iopub.status.busy": "2025-09-16T14:36:11.406482Z", - "iopub.status.idle": "2025-09-16T14:36:11.594174Z", - "shell.execute_reply": "2025-09-16T14:36:11.593622Z", - "shell.execute_reply.started": "2025-09-16T14:36:11.406703Z" + "iopub.execute_input": "2025-09-25T16:01:31.048121Z", + "iopub.status.busy": "2025-09-25T16:01:31.047904Z", + "iopub.status.idle": "2025-09-25T16:01:31.235811Z", + "shell.execute_reply": "2025-09-25T16:01:31.235364Z", + "shell.execute_reply.started": "2025-09-25T16:01:31.048099Z" } }, "outputs": [], diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/model_registration_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/model_registration_step.py index d4e819e..d24644d 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/model_registration_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/model_registration_step.py @@ -5,173 +5,6 @@ from .pipeline_utils import PIPELINE_INSTANCE_TYPE -# @step( -# name="ModelRegistration", -# instance_type=PIPELINE_INSTANCE_TYPE, -# display_name="Model Registration", -# keep_alive_period_in_seconds=900 -# ) -# def register_model( -# tracking_server_arn: str, -# experiment_name: str, -# run_id: str, -# model_artifacts_s3_path: str, -# model_id: str, -# model_name: str, -# endpoint_name: str, -# evaluation_score: float, -# pipeline_name: str, -# model_description: str -# ): -# import json -# import mlflow -# import boto3 -# from datetime import datetime - -# print(f"Registering model: {model_name}") - -# # Set up MLflow tracking -# mlflow.set_tracking_uri(tracking_server_arn) -# mlflow.set_experiment(experiment_name) - -# # Connect to MLflow with the specific run -# with mlflow.start_run(run_id=run_id): -# # Create model metadata -# tags = { -# "model_id": model_id, -# "base_model": model_id.split('/')[-1], -# "task": "medical_qa", -# "framework": "pytorch", -# "endpoint_name": endpoint_name, -# "model_artifacts_s3_path": model_artifacts_s3_path, -# "deployment_timestamp": datetime.now().isoformat(), -# "description": model_description, -# "registered_by": pipeline_name -# } - -# # Log model info as parameters -# mlflow.log_param("registered_model_name", model_name) -# mlflow.log_param("model_artifacts_path", model_artifacts_s3_path) -# mlflow.log_param("evaluation_score", evaluation_score) -# mlflow.log_param("endpoint_name", endpoint_name) -# mlflow.log_param("registration_timestamp", datetime.now().isoformat()) - -# # Log endpoint information as an artifact -# with open("/tmp/model_info.json", "w") as f: -# json.dump({ -# "model_name": model_name, -# "model_id": model_id, -# "endpoint_name": endpoint_name, -# "model_artifacts_s3_path": model_artifacts_s3_path, -# "evaluation_score": float(evaluation_score), -# "registration_timestamp": datetime.now().isoformat() -# }, f, indent=2) -# mlflow.log_artifact("/tmp/model_info.json") - -# # Register the model -# try: -# client = mlflow.tracking.MlflowClient() - -# # Check if model exists and create if it doesn't -# try: -# client.get_registered_model(model_name) -# print(f"Model {model_name} already exists in registry") -# except mlflow.exceptions.MlflowException: -# client.create_registered_model( -# name=model_name, -# description=f"Fine-tuned medical LLM based on {model_id}" -# ) -# print(f"Created new registered model: {model_name}") - -# # Create a new model version -# model_version = client.create_model_version( -# name=model_name, -# source=model_artifacts_s3_path, # Direct S3 path to model artifacts -# run_id=run_id, -# description=f"Fine-tuned LLM deployed at endpoint: {endpoint_name}" -# ) - -# # Wait for model registration to complete -# import time -# for _ in range(10): # Try for up to ~50 seconds -# version_details = client.get_model_version(model_name, model_version.version) -# if version_details.status == "READY": -# break -# time.sleep(5) - -# # Add tags to the registered model version -# for key, value in tags.items(): -# client.set_model_version_tag(model_name, model_version.version, key, value) - -# # Transition model to Production/Staging based on evaluation score -# if evaluation_score >= 0.3: # Example threshold -# client.transition_model_version_stage( -# name=model_name, -# version=model_version.version, -# stage="Production", -# archive_existing_versions=True -# ) -# print(f"Model {model_name} version {model_version.version} promoted to Production") -# else: -# client.transition_model_version_stage( -# name=model_name, -# version=model_version.version, -# stage="Staging", -# archive_existing_versions=False -# ) -# print(f"Model {model_name} version {model_version.version} added to Staging due to lower evaluation score") - -# print(f"Successfully registered model: {model_name}, version: {model_version.version}") -# latest_version = model_version.version - -# except Exception as e: -# print(f"Error registering model: {str(e)}") -# # Try alternative approach -# try: -# # Register model using mlflow API (simpler approach) -# model_details = mlflow.register_model( -# model_uri=f"runs:/{run_id}/model", -# name=model_name, -# tags=tags -# ) -# latest_version = model_details.version -# print(f"Alternative registration successful: {model_name} version {latest_version}") -# except Exception as e2: -# print(f"Alternative registration failed: {str(e2)}") -# latest_version = "unknown" - -# # Create SageMaker integration for the model -# try: -# sm_client = boto3.client('sagemaker') - -# # Create a normalized name for SageMaker resources -# sm_model_name = model_name.replace(".", "-").replace("_", "-") - -# # Create or update model package group -# try: -# sm_client.describe_model_package_group(ModelPackageGroupName=sm_model_name) -# print(f"SageMaker model package group {sm_model_name} already exists") -# except sm_client.exceptions.ClientError: -# sm_client.create_model_package_group( -# ModelPackageGroupName=sm_model_name, -# ModelPackageGroupDescription=f"Fine-tuned LLM model: {model_name}" -# ) -# print(f"Created SageMaker model package group: {sm_model_name}") - -# # Log SageMaker details -# mlflow.log_param("sagemaker_model_group", sm_model_name) - -# print(f"Successfully integrated with SageMaker model registry") - -# except Exception as e: -# print(f"Warning: Error in SageMaker model registry integration: {str(e)}") - -# return model_name, latest_version - - - - - @step( name="ModelRegistration", instance_type=PIPELINE_INSTANCE_TYPE, diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py index f51dd1d..3e9cd92 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py @@ -101,4 +101,68 @@ def invoke_sagemaker_endpoint(payload, endpoint_name): return json.loads(response_body), inference_time except Exception as e: print(f"Error invoking endpoint {endpoint_name}: {str(e)}") - return None, -1 \ No newline at end of file + return None, -1 + + +def get_or_create_guardrail(): + guardrail_client = boto3.client('bedrock') + guardrail_name = "ExampleMedicalGuardrail" + try: + # Try to get the guardrail + response = guardrail_client.list_guardrails() + for guardrail in response.get('guardrails', []): + if guardrail['name'] == guardrail_name: + guardrail_id = guardrail['id'] + response = guardrail_client.get_guardrail( + guardrailIdentifier=guardrail_id + ) + guardrail_version = response["version"] + print(f"Found Guardrail {guardrail_id}:{guardrail_version}") + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ResourceNotFoundException': + # Guardrail doesn't exist, create it + try: + guardrail = guardrail_client.create_guardrail( + name="ExampleMedicalGuardrail", + description='Example of a Guardrail for Medical Use Cases', + topicPolicyConfig={ + 'topicsConfig': [{ + 'name': 'Block Pharmaceuticals', + 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.', + 'type': 'DENY', + 'inputAction': 'BLOCK', + 'outputAction': 'BLOCK', + }] + }, + sensitiveInformationPolicyConfig={ + 'piiEntitiesConfig': [ + { + 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER', + 'action': 'BLOCK', + 'inputAction': 'BLOCK', + 'outputAction': 'BLOCK' + }, + ] + }, + contextualGroundingPolicyConfig={ + 'filtersConfig': [ + { + 'type': 'RELEVANCE', + 'threshold': 0.9, + 'action': 'BLOCK', + 'enabled': True + }, + ] + }, + blockedInputMessaging="ExampleMedicalGuardrail has blocked this input.", + blockedOutputsMessaging="ExampleMedicalGuardrail has blocked this output." + ) + guardrail_id = guardrail['guardrailId'] + guardrail_version = guardrail['version'] + + print(f"Created new guardrail '{guardrail_id}:{guardrail_version}'") + except botocore.exceptions.ClientError as create_error: + print(f"Error creating guardrail: {create_error}") + else: + print(f"Error checking guardrail: {e}") + return guardrail_id, guardrail_version \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py index bd85fa8..19a9254 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py @@ -32,57 +32,6 @@ def preprocess( experiment_name: str, run_name: str, ) -> tuple: - - - # prompt_template = f""" - # <|begin_of_text|> - # <|start_header_id|>system<|end_header_id|> - # You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. - # Below is an instruction that describes a task, paired with an input that provides further context. - # Write a response that appropriately completes the request. - # Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. - # <|eot_id|><|start_header_id|>user<|end_header_id|> - # {{question}}<|eot_id|> - # <|start_header_id|>assistant<|end_header_id|> - # {{complex_cot}} - - # {{answer}} - # <|eot_id|> - # """ - - # # Include template_dataset function directly here - # def template_dataset(sample): - # try: - # sample["text"] = prompt_template.format( - # question=sample["Question"], - # complex_cot=sample["Complex_CoT"], - # answer=sample["Response"] - # ) - # return sample - # except KeyError as e: - # print(f"KeyError in template_dataset: {str(e)}") - # # Provide default values for missing fields - # missing_key = str(e).strip("'") - # if missing_key == "Question": - # sample["text"] = prompt_template.format( - # question="[Missing question]", - # complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), - # answer=sample.get("Response", "[Missing response]") - # ) - # elif missing_key == "Complex_CoT": - # sample["text"] = prompt_template.format( - # question=sample["Question"], - # complex_cot="[Missing CoT]", - # answer=sample.get("Response", "[Missing response]") - # ) - # elif missing_key == "Response": - # sample["text"] = prompt_template.format( - # question=sample["Question"], - # complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), - # answer="[Missing response]" - # ) - # return sample - mlflow.set_tracking_uri(tracking_server_arn) mlflow.set_experiment(experiment_name) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py index dc785a6..cc8cea7 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py @@ -18,6 +18,7 @@ def qualitative_evaluate( experiment_name: str, run_id: str, endpoint_name: str, + mlflow_trace_attributes: dict ) -> dict: import os import json @@ -32,14 +33,35 @@ def qualitative_evaluate( import uuid import traceback from datetime import datetime - - # MLflow LLM-as-a-judge imports (compatible with MLflow 2.x) + from mlflow.entities import SpanType from mlflow.metrics.genai import EvaluationExample, make_genai_metric - + + @mlflow.trace( + name="call-local-llm", span_type=SpanType.LLM, attributes={ + "model": mlflow_trace_attributes["model_id"], + "guardrail_id": mlflow_trace_attributes["guardrail_id"], + "guardrail_version": mlflow_trace_attributes["guardrail_version"] + } + ) def invoke_sagemaker_endpoint(payload, endpoint_name): """ Invoke a SageMaker endpoint with the given payload. """ + bedrock_runtime = boto3.client('bedrock-runtime') + guardrail_id = mlflow_trace_attributes["guardrail_id"] + guardrail_version = mlflow_trace_attributes["guardrail_version"] + guardrail_response_input = bedrock_runtime.apply_guardrail( + guardrailIdentifier=guardrail_id, + guardrailVersion=guardrail_version, + source='INPUT', + content=[{'text': {'text': payload["inputs"]}}] + ) + guardrailResult = guardrail_response_input["action"] + + if guardrailResult == "GUARDRAIL_INTERVENED": + reason = guardrail_response_input["assessments"] + return guardrail_response_input["outputs"][0]["text"], -1 + try: start_time = time.time() response = sm_client.invoke_endpoint( @@ -107,6 +129,7 @@ def create_bedrock_judge_metrics(): ), examples=medical_accuracy_examples, version="v1", + # model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", parameters={ "anthropic_version": "bedrock-2023-05-31", @@ -155,6 +178,7 @@ def create_bedrock_judge_metrics(): ), examples=clinical_reasoning_examples, version="v1", + # model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", parameters={ "anthropic_version": "bedrock-2023-05-31", @@ -201,6 +225,7 @@ def create_bedrock_judge_metrics(): ), examples=patient_safety_examples, version="v1", + # model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", parameters={ "anthropic_version": "bedrock-2023-05-31", @@ -211,7 +236,7 @@ def create_bedrock_judge_metrics(): greater_is_better=True ) - return [medical_accuracy]#, clinical_reasoning, patient_safety] + return [medical_accuracy, clinical_reasoning, patient_safety] def simple_judge_evaluation(predictions, questions, references): """ @@ -349,7 +374,7 @@ def evaluate_model_qualitatively(model_config, dataset): # Extract metric results metric_results = {} - for metric_name in ["medical_accuracy/v1/mean"]:#, "clinical_reasoning/v1/mean", "patient_safety/v1/mean"]: + for metric_name in ["medical_accuracy/v1/mean", "clinical_reasoning/v1/mean", "patient_safety/v1/mean"]: if metric_name in eval_results.metrics: base_name = metric_name.split('/')[0] metric_results[base_name] = eval_results.metrics[metric_name] @@ -453,7 +478,7 @@ def evaluate_model_qualitatively(model_config, dataset): mlflow.log_param("qualitative_evaluation_endpoint", endpoint_name) mlflow.log_param("qualitative_evaluation_num_samples", num_samples) mlflow.log_param("qualitative_evaluation_timestamp", datetime.now().isoformat()) - mlflow.log_param("llm_judge_model", "bedrock:/anthropic.claude-3-haiku-20240307-v1:0") + mlflow.log_param("llm_judge_model", "bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0") # Load the test dataset try: diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py index 8916973..05df1b5 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py @@ -18,6 +18,7 @@ def quantitative_evaluate( experiment_name: str, run_id: str, endpoint_name: str, + mlflow_trace_attributes: dict )-> dict: import os import json @@ -33,8 +34,15 @@ def quantitative_evaluate( import traceback from datetime import datetime from rouge_score import rouge_scorer + from mlflow.entities import SpanType - # This function allows you to interact with a deployed SageMaker endpoint to get predictions from the DeepSeek model + @mlflow.trace( + name="call-local-llm", span_type=SpanType.LLM, attributes={ + "model": mlflow_trace_attributes["model_id"], + "guardrail_id": mlflow_trace_attributes["guardrail_id"], + "guardrail_version": mlflow_trace_attributes["guardrail_version"] + } + ) def invoke_sagemaker_endpoint(payload, endpoint_name): """ Invoke a SageMaker endpoint with the given payload. @@ -46,6 +54,21 @@ def invoke_sagemaker_endpoint(payload, endpoint_name): Returns: dict: The response from the endpoint """ + bedrock_runtime = boto3.client('bedrock-runtime') + guardrail_id = mlflow_trace_attributes["guardrail_id"] + guardrail_version = mlflow_trace_attributes["guardrail_version"] + guardrail_response_input = bedrock_runtime.apply_guardrail( + guardrailIdentifier=guardrail_id, + guardrailVersion=guardrail_version, + source='INPUT', + content=[{'text': {'text': payload["inputs"]}}] + ) + guardrailResult = guardrail_response_input["action"] + + if guardrailResult == "GUARDRAIL_INTERVENED": + reason = guardrail_response_input["assessments"] + return guardrail_response_input["outputs"][0]["text"], -1 + try: start_time = time.time() response = sm_client.invoke_endpoint( From 9cd405ee5a8dd03ef566c4e4a34277de2fa7f4a3 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Thu, 25 Sep 2025 14:06:37 -0400 Subject: [PATCH 06/22] Improved markdown on notebook --- .../05.01_fine-tuning-pipeline.ipynb | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index 2b2dc27..9d6f06e 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -4,14 +4,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Fine-Tuning and Evaluating LLMs with SageMaker Pipelines and MLflow" + "## Coordinating FMOps Steps into a Fine-Tuning and Model Evaluation Pipeline" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Running hundreds of experiments, comparing the results, and keeping a track of the ML lifecycle can become very complex. This is where MLflow can help streamline the ML lifecycle, from data preparation to model deployment. By integrating MLflow into your LLM workflow, you can efficiently manage experiment tracking, model versioning, and deployment, providing reproducibility. With MLflow, you can track and compare the performance of multiple LLM experiments, identify the best-performing models, and deploy them to production environments with confidence. \n", + "In this notebook, we stitch together the components of FMOps into a full FMOps pipeline on SageMaker AI. This capability creates a Directed-Acyclic Graph of steps, orchestrated by SageMaker AI and Managed MLFlow 3.0 on Amazon SageMaker.\n", + "\n", + "Running hundreds of experiments, comparing the results, and keeping a track of the ML lifecycle can become very complex. This is where MLflow can help streamline the ML lifecycle, from data preparation to model deployment. By integrating MLflow into your LLM workflow, you can efficiently manage experiment tracking, model versioning, and deployment, providing reproducibility of steps. With MLflow, you can track and compare the performance of multiple LLM experiments, identify the best-performing models, and deploy them to production environments with confidence. \n", "\n", "You can create workflows with SageMaker Pipelines that enable you to prepare data, fine-tune models, and evaluate model performance with simple Python code for each step. \n", "\n", @@ -237,6 +239,13 @@ "os.environ[\"pipeline_name\"] = pipeline_name" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This section provides blanket configuration for how remote functions should be executed in a SageMaker environment. This configuration helps to streamline remote function execution which is particularly useful for optimizing the execution of pipelines." + ] + }, { "cell_type": "code", "execution_count": 4, @@ -658,7 +667,14 @@ "\n", "**Creating the Pipeline**\n", "\n", - "The pipeline object is created with all defined steps." + "The pipeline object is created with all defined steps.\n", + "\n", + "1. Preprocessing Step -- Reformat all of the fine-tuning data to the prompt format required for the fine-tuning job.\n", + "2. Training Step -- Execute the model fine-tuning job using the preprocessed data.\n", + "3. Deploy Step -- Deploy the model to a SageMaker AI Managed Endpoint for testing fine-tuning performance.\n", + "4. Quantitative Evaluation Step -- Evaluate the model's performance using ROUGE scores.\n", + "5. Qualitative Evaluation Step -- Evaluate the model's performance using LLM-as-a-Judge.\n", + "6. Conditionally Register Model -- Register the model if the quantitative and qualitative evaluations meet criteria." ] }, { From a5defef771d7268d59094fbfc36e499670f2fbdf Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Mon, 29 Sep 2025 15:49:29 -0400 Subject: [PATCH 07/22] Updated task 05.00 to be clean and neat for presentation --- .../task_05_fmops/05.00_fmops_examples.ipynb | 601 ++---------------- 1 file changed, 47 insertions(+), 554 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb index d0507a9..aaadb31 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -40,28 +40,9 @@ }, { "cell_type": "code", - "execution_count": 75, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T16:24:11.008896Z", - "iopub.status.busy": "2025-09-23T16:24:11.008634Z", - "iopub.status.idle": "2025-09-23T16:24:11.012999Z", - "shell.execute_reply": "2025-09-23T16:24:11.012430Z", - "shell.execute_reply.started": "2025-09-23T16:24:11.008876Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'status': 'ok', 'restart': True}" - ] - }, - "execution_count": 75, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from IPython import get_ipython\n", "get_ipython().kernel.do_shutdown(True)" @@ -78,26 +59,9 @@ }, { "cell_type": "code", - "execution_count": 1, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:16.030318Z", - "iopub.status.busy": "2025-09-23T20:09:16.030111Z", - "iopub.status.idle": "2025-09-23T20:09:17.845965Z", - "shell.execute_reply": "2025-09-23T20:09:17.845503Z", - "shell.execute_reply.started": "2025-09-23T20:09:16.030301Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", - "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "import os\n", "import json\n", @@ -133,16 +97,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:17.846760Z", - "iopub.status.busy": "2025-09-23T20:09:17.846596Z", - "iopub.status.idle": "2025-09-23T20:09:18.225333Z", - "shell.execute_reply": "2025-09-23T20:09:18.224883Z", - "shell.execute_reply.started": "2025-09-23T20:09:17.846744Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", @@ -163,25 +119,9 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:18.419445Z", - "iopub.status.busy": "2025-09-23T20:09:18.419284Z", - "iopub.status.idle": "2025-09-23T20:09:18.636938Z", - "shell.execute_reply": "2025-09-23T20:09:18.636485Z", - "shell.execute_reply.started": "2025-09-23T20:09:18.419432Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "sagemaker-us-east-1-329542461890\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "bucket_name = sagemaker_session.default_bucket()\n", "print(bucket_name)\n", @@ -194,16 +134,8 @@ }, { "cell_type": "code", - "execution_count": 4, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:18.869074Z", - "iopub.status.busy": "2025-09-23T20:09:18.868903Z", - "iopub.status.idle": "2025-09-23T20:09:18.871966Z", - "shell.execute_reply": "2025-09-23T20:09:18.871490Z", - "shell.execute_reply.started": "2025-09-23T20:09:18.869059Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "model_id = \"deepseek-ai/DeepSeek-R1-Distill-Llama-8B\"\n", @@ -211,8 +143,7 @@ "model_name_safe = model_id.split('/')[-1].replace('.', '-').replace('_', '-')\n", "endpoint_name = f\"Example-{model_name_safe}\"\n", "instance_count = 1\n", - "instance_type = \"ml.g5.2xlarge\"\n", - "guardrail_id = \"u1yfe55ecv4z\" # Not an actual Guardrail ID" + "instance_type = \"ml.g5.2xlarge\"" ] }, { @@ -233,20 +164,11 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:21.117974Z", - "iopub.status.busy": "2025-09-23T20:09:21.117722Z", - "iopub.status.idle": "2025-09-23T20:09:21.120963Z", - "shell.execute_reply": "2025-09-23T20:09:21.120461Z", - "shell.execute_reply.started": "2025-09-23T20:09:21.117955Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "mlflow_tracking_server_arn = \"\"\n", - "mlflow_tracking_server_arn = \"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server/my-tracking-server\"\n", "\n", "if not mlflow_tracking_server_arn:\n", " try:\n", @@ -279,25 +201,9 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:23.299286Z", - "iopub.status.busy": "2025-09-23T20:09:23.299057Z", - "iopub.status.idle": "2025-09-23T20:09:23.304425Z", - "shell.execute_reply": "2025-09-23T20:09:23.303915Z", - "shell.execute_reply.started": "2025-09-23T20:09:23.299268Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'Question': 'A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?', 'Complex_CoT': \"Okay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem.\\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal.\\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\", 'Response': 'Cystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.', 'text': '\\n<|begin_of_text|>\\n <|start_header_id|>system<|end_header_id|>\\n You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \\n Below is an instruction that describes a task, paired with an input that provides further context. \\n Write a response that appropriately completes the request.\\n Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\\n <|eot_id|>\\n <|start_header_id|>user<|end_header_id|>\\n {question}\\n <|eot_id|>\\n <|start_header_id|>assistant<|end_header_id|>\\n {complex_cot}\\n {answer}\\n<|eot_id|>\\n'}\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "FINE_TUNING_DATA_SAMPLE = {\n", " \"Question\": \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\", \n", @@ -386,16 +292,8 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:28.240483Z", - "iopub.status.busy": "2025-09-23T20:09:28.240271Z", - "iopub.status.idle": "2025-09-23T20:09:28.256986Z", - "shell.execute_reply": "2025-09-23T20:09:28.256596Z", - "shell.execute_reply.started": "2025-09-23T20:09:28.240468Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Create and deploy model\n", @@ -416,16 +314,8 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:30.016790Z", - "iopub.status.busy": "2025-09-23T20:09:30.016619Z", - "iopub.status.idle": "2025-09-23T20:09:30.140650Z", - "shell.execute_reply": "2025-09-23T20:09:30.140207Z", - "shell.execute_reply.started": "2025-09-23T20:09:30.016776Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "model_config = {\n", @@ -457,28 +347,9 @@ }, { "cell_type": "code", - "execution_count": 9, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:30.871383Z", - "iopub.status.busy": "2025-09-23T20:09:30.871211Z", - "iopub.status.idle": "2025-09-23T20:09:31.029947Z", - "shell.execute_reply": "2025-09-23T20:09:31.029531Z", - "shell.execute_reply.started": "2025-09-23T20:09:30.871368Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Initialize MLFlow tracking data...\n", "mlflow.set_tracking_uri(mlflow_tracking_server_arn)\n", @@ -488,21 +359,8 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T16:27:51.741447Z", - "iopub.status.busy": "2025-09-23T16:27:51.741253Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--" - ] - } - ], + "metadata": {}, + "outputs": [], "source": [ "with mlflow.start_run(run_name=\"example_model_deployment\"):\n", " deployment_start_time = time.time()\n", @@ -561,28 +419,9 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:09:50.061640Z", - "iopub.status.busy": "2025-09-23T20:09:50.061429Z", - "iopub.status.idle": "2025-09-23T20:10:00.090701Z", - "shell.execute_reply": "2025-09-23T20:10:00.090239Z", - "shell.execute_reply.started": "2025-09-23T20:09:50.061625Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'generated_text': \" \\n\\nOkay, so I have this question about a 61-year-old woman with a history of involuntary urine loss. She experiences this during activities like coughing or sneezing but doesn't leak at night. She undergoes a gynecological exam and a Q-tip test. The question is asking what cystometry would most likely reveal about her residual volume and detrusor contractions.\\n\\nFirst, I need to break down the information given. She's 61, so she's of postmenopausal age, which might be relevant because urinary issues can change after menopause. She has involuntary urine loss, which makes me think of stress urinary incontinence (SUI). SUI is common in women, especially after menopause, and it's typically due to weak pelvic muscles or urethral issues.\\n\\nShe mentions the loss happens during activities like coughing or sneezing, which are activities that can increase intra-abdominal pressure, leading to urethral sphincter failure. Also, she doesn't leak at night, which is interesting because that suggests it's not a mixed incontinence case (where she might have both stress and urge incontinence). If she didn't leak at night, it's more likely purely stress incontinence.\\n\\nShe undergoes a gynecological exam and a Q-tip test. I'm not exactly sure what the Q-tip test entails, but I think it's a physical exam maneuver where the provider inserts a Q-tip catheter into the urethra and asks the patient to cough or bear down. If the catheter doesn't stay in the urethra (i.e., it pops out), it suggests urethral sphincter deficiency, which is a sign of SUI.\\n\\nSo, if the Q-tip test shows that the catheter doesn't stay in the urethra, that would support a diagnosis of SUI. Now, moving on to cystometry, which is a more detailed diagnostic tool. Cystometry involves inserting a catheter into the bladder and filling it with fluid to measure how much the patient can hold before needing to urinate (the capacity), and it also assesses the detrusor muscle contractions.\\n\\nIn SUI, the main issue is the inability to prevent the urethral sphincter from opening when there's increased intra-abdominal pressure. On cystometry, this would show that the patient has a small residual volume in the bladder because they can't hold their urine under stress. Additionally, during the filling phase, the\"}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from sagemaker.predictor import Predictor\n", "from sagemaker.serializers import JSONSerializer\n", @@ -607,25 +446,9 @@ }, { "cell_type": "code", - "execution_count": 51, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:37:33.037714Z", - "iopub.status.busy": "2025-09-23T20:37:33.037568Z", - "iopub.status.idle": "2025-09-23T20:37:33.290095Z", - "shell.execute_reply": "2025-09-23T20:37:33.289613Z", - "shell.execute_reply.started": "2025-09-23T20:37:33.037700Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found Guardrail zpeb3rjh181n:DRAFT\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "guardrail_client = boto3.client('bedrock')\n", "guardrail_name = \"ExampleMedicalGuardrail\"\n", @@ -691,16 +514,8 @@ }, { "cell_type": "code", - "execution_count": 44, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:35:19.907065Z", - "iopub.status.busy": "2025-09-23T20:35:19.906901Z", - "iopub.status.idle": "2025-09-23T20:35:19.914642Z", - "shell.execute_reply": "2025-09-23T20:35:19.914241Z", - "shell.execute_reply.started": "2025-09-23T20:35:19.907051Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "bedrock_runtime = boto3.client('bedrock-runtime')" @@ -720,16 +535,8 @@ }, { "cell_type": "code", - "execution_count": 45, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:35:21.535737Z", - "iopub.status.busy": "2025-09-23T20:35:21.535567Z", - "iopub.status.idle": "2025-09-23T20:35:21.538052Z", - "shell.execute_reply": "2025-09-23T20:35:21.537596Z", - "shell.execute_reply.started": "2025-09-23T20:35:21.535723Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "judge_llm = \"bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0\"" @@ -737,16 +544,8 @@ }, { "cell_type": "code", - "execution_count": 46, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:35:21.758908Z", - "iopub.status.busy": "2025-09-23T20:35:21.758723Z", - "iopub.status.idle": "2025-09-23T20:35:21.762765Z", - "shell.execute_reply": "2025-09-23T20:35:21.762295Z", - "shell.execute_reply.started": "2025-09-23T20:35:21.758892Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "from mlflow.entities import SpanType\n", @@ -794,16 +593,8 @@ }, { "cell_type": "code", - "execution_count": 47, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:35:22.235629Z", - "iopub.status.busy": "2025-09-23T20:35:22.235456Z", - "iopub.status.idle": "2025-09-23T20:35:22.241931Z", - "shell.execute_reply": "2025-09-23T20:35:22.241436Z", - "shell.execute_reply.started": "2025-09-23T20:35:22.235615Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "from mlflow.metrics.genai import EvaluationExample, make_genai_metric\n", @@ -969,16 +760,8 @@ }, { "cell_type": "code", - "execution_count": 48, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:35:22.808750Z", - "iopub.status.busy": "2025-09-23T20:35:22.808581Z", - "iopub.status.idle": "2025-09-23T20:35:22.816843Z", - "shell.execute_reply": "2025-09-23T20:35:22.816366Z", - "shell.execute_reply.started": "2025-09-23T20:35:22.808737Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "def evaluate_model_qualitatively(model_config, dataset):\n", @@ -1131,296 +914,9 @@ }, { "cell_type": "code", - "execution_count": 49, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-23T20:35:23.603207Z", - "iopub.status.busy": "2025-09-23T20:35:23.602938Z", - "iopub.status.idle": "2025-09-23T20:37:32.690226Z", - "shell.execute_reply": "2025-09-23T20:37:32.689703Z", - "shell.execute_reply.started": "2025-09-23T20:35:23.603192Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded medical-o1-reasoning dataset with 10 samples for qualitative evaluation\n", - "\n", - "Performing qualitative evaluation for model: Example-DeepSeek-R1-Distill-Llama-8B-sft-djl on endpoint: Example-DeepSeek-R1-Distill-Llama-8B-sft-djl\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Generating responses for evaluation: 0%| | 0/10 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from datasets import load_dataset\n", "from botocore.config import Config\n", @@ -1578,9 +1074,6 @@ "\n", "# Clean up endpoint\n", "try:\n", - " model_name_safe = model_id.split('/')[-1].replace('.', '-').replace('_', '-')\n", - " endpoint_name = f\"Example-{model_name_safe}-sft-djl\"\n", - " \n", " print(f\"Cleaning up endpoint: {endpoint_name}\")\n", " if delete_endpoint_with_retry(endpoint_name):\n", " print(\"Cleanup completed successfully\")\n", From 41916d65136d12d42ac012751a57291be5036946 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Mon, 29 Sep 2025 18:56:57 -0400 Subject: [PATCH 08/22] Updated task 05.01 to be clean and neat for presentation --- .../05.01_fine-tuning-pipeline.ipynb | 495 ++---------------- 1 file changed, 36 insertions(+), 459 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index 9d6f06e..f75f23c 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -47,53 +47,18 @@ }, { "cell_type": "code", - "execution_count": 25, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-19T16:25:15.196112Z", - "iopub.status.busy": "2025-09-19T16:25:15.195890Z", - "iopub.status.idle": "2025-09-19T16:25:19.565336Z", - "shell.execute_reply": "2025-09-19T16:25:19.564715Z", - "shell.execute_reply.started": "2025-09-19T16:25:15.196096Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%pip install -r ./scripts/requirements.txt --upgrade --quiet" ] }, { "cell_type": "code", - "execution_count": 26, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-19T16:25:19.566511Z", - "iopub.status.busy": "2025-09-19T16:25:19.566326Z", - "iopub.status.idle": "2025-09-19T16:25:19.570143Z", - "shell.execute_reply": "2025-09-19T16:25:19.569659Z", - "shell.execute_reply.started": "2025-09-19T16:25:19.566491Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'status': 'ok', 'restart': True}" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from IPython import get_ipython\n", "get_ipython().kernel.do_shutdown(True)" @@ -110,26 +75,9 @@ }, { "cell_type": "code", - "execution_count": 1, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-25T16:01:11.559199Z", - "iopub.status.busy": "2025-09-25T16:01:11.558975Z", - "iopub.status.idle": "2025-09-25T16:01:12.927880Z", - "shell.execute_reply": "2025-09-25T16:01:12.927342Z", - "shell.execute_reply.started": "2025-09-25T16:01:11.559179Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", - "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "import os\n", "import boto3\n", @@ -159,16 +107,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-25T16:01:12.928809Z", - "iopub.status.busy": "2025-09-25T16:01:12.928598Z", - "iopub.status.idle": "2025-09-25T16:01:13.401352Z", - "shell.execute_reply": "2025-09-25T16:01:13.400909Z", - "shell.execute_reply.started": "2025-09-25T16:01:12.928789Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", @@ -211,19 +151,11 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-25T16:01:13.835007Z", - "iopub.status.busy": "2025-09-25T16:01:13.834828Z", - "iopub.status.idle": "2025-09-25T16:01:13.837951Z", - "shell.execute_reply": "2025-09-25T16:01:13.837420Z", - "shell.execute_reply.started": "2025-09-25T16:01:13.834992Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ - "mlflow_tracking_server_arn = \"arn:aws:sagemaker:us-east-1:329542461890:mlflow-tracking-server/my-tracking-server\"\n", + "mlflow_tracking_server_arn = \"\"\n", "\n", "if not mlflow_tracking_server_arn:\n", " try:\n", @@ -248,25 +180,9 @@ }, { "cell_type": "code", - "execution_count": 4, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-25T16:01:14.639513Z", - "iopub.status.busy": "2025-09-25T16:01:14.639316Z", - "iopub.status.idle": "2025-09-25T16:01:14.643190Z", - "shell.execute_reply": "2025-09-25T16:01:14.642692Z", - "shell.execute_reply.started": "2025-09-25T16:01:14.639496Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting config.yaml\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%%writefile config.yaml\n", "SchemaVersion: '1.0'\n", @@ -287,16 +203,8 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": { - "execution": { - "iopub.execute_input": "2025-09-25T16:01:14.643950Z", - "iopub.status.busy": "2025-09-25T16:01:14.643811Z", - "iopub.status.idle": "2025-09-25T16:01:14.647462Z", - "shell.execute_reply": "2025-09-25T16:01:14.647007Z", - "shell.execute_reply.started": "2025-09-25T16:01:14.643937Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Set path to config file\n", @@ -312,91 +220,11 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { - "execution": { - "iopub.execute_input": "2025-09-25T16:01:15.045142Z", - "iopub.status.busy": "2025-09-25T16:01:15.044961Z", - "iopub.status.idle": "2025-09-25T16:01:15.802257Z", - "shell.execute_reply": "2025-09-25T16:01:15.801771Z", - "shell.execute_reply.started": "2025-09-25T16:01:15.045126Z" - }, "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading model deepseek-ai/DeepSeek-R1-Distill-Llama-8B\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "86dd70d946c847e0befc5cc2e3365089", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Fetching 11 files: 0%| | 0/11 [00:00 Date: Tue, 7 Oct 2025 13:43:31 -0400 Subject: [PATCH 09/22] Moved fine-tuning to the end --- .../task_05_fmops/05.00_fmops_examples.ipynb | 185 +++++++++--------- 1 file changed, 92 insertions(+), 93 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb index aaadb31..a1c09aa 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -187,96 +187,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 4. Templating a Prompt" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this workshop we are going to fine-tune DeepSeek-R1-Distill-Llama-8B to become a medical expert. To accomplish this, we will execute a fine-tuning job using Managed MLflow on SageMaker AI. We get our data from the [FreedomIntelligence/medical-o1-reasoning-SFT](https://huggingface.co/datasets/FreedomIntelligence/medical-o1-reasoning-SFT) dataset, available on HuggingFace.\n", - "\n", - "We perform the full fine-tuning step in the next lab. In this lab, we show a small example of what fine-tuning looks like for a single record of the dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "FINE_TUNING_DATA_SAMPLE = {\n", - " \"Question\": \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\", \n", - " \"Complex_CoT\": \"Okay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem.\\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal.\\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\",\n", - " \"Response\": \"Cystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.\"\n", - "}\n", - "\n", - "\n", - "PROMPT_TEMPLATE = \"\"\"\n", - "<|begin_of_text|>\n", - " <|start_header_id|>system<|end_header_id|>\n", - " You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", - " Below is an instruction that describes a task, paired with an input that provides further context. \n", - " Write a response that appropriately completes the request.\n", - " Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", - " <|eot_id|>\n", - " <|start_header_id|>user<|end_header_id|>\n", - " {{question}}\n", - " <|eot_id|>\n", - " <|start_header_id|>assistant<|end_header_id|>\n", - " {{complex_cot}}\n", - " {{answer}}\n", - "<|eot_id|>\n", - "\"\"\"\n", - "\n", - "# Template dataset to add prompt to each sample\n", - "def template_dataset(sample):\n", - " try:\n", - " sample[\"text\"] = PROMPT_TEMPLATE.format(question=sample[\"Question\"],\n", - " complex_cot=sample[\"Complex_CoT\"],\n", - " answer=sample[\"Response\"])\n", - " return sample\n", - " except KeyError as e:\n", - " print(f\"KeyError in template_dataset: {str(e)}\")\n", - " # Provide default values for missing fields\n", - " missing_key = str(e).strip(\"'\")\n", - " if missing_key == \"Question\":\n", - " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", - " question=\"[Missing question]\",\n", - " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", - " answer=sample.get(\"Response\", \"[Missing response]\")\n", - " )\n", - " elif missing_key == \"Complex_CoT\":\n", - " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", - " question=sample[\"Question\"],\n", - " complex_cot=\"[Missing CoT]\",\n", - " answer=sample.get(\"Response\", \"[Missing response]\")\n", - " )\n", - " elif missing_key == \"Response\":\n", - " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", - " question=sample[\"Question\"],\n", - " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", - " answer=\"[Missing response]\"\n", - " )\n", - " return sample\n", - "\n", - "PROCESSED_SAMPLE = template_dataset(FINE_TUNING_DATA_SAMPLE)\n", - "print(PROCESSED_SAMPLE)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### Fine-Tuning Output\n", - "The above output shows the templated prompt output to be used for fine-tuning. This pre-processing happens for every record in the fine-tuning dataset before fine-tuning actually takes place. This can be time-consuming for large fine-tuning datasets. We will show in the next lab how to orchestrate this with MLflow." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5. Model Deployment\n", + "### 4. Model Deployment\n", "There are several approaches to deploying a model to a SageMaker AI managed endpoint. In this section, we explore the most direct option which downloads a model directly from HuggingFace to the managed endpoint via SageMaker JumpStart. We are still using DeepSeek-R1-Distill-Llama-8B, but we have not fine-tuned it. The purpose of this section is to illustrate the components required to customize a model deployment on SageMaker before fine-tuning it." ] }, @@ -433,8 +344,7 @@ " deserializer=JSONDeserializer()\n", ")\n", "predictor.predict({\n", - " # \"inputs\": \"Hi, what can you help me with?\",\n", - " \"inputs\": FINE_TUNING_DATA_SAMPLE[\"Question\"],\n", + " \"inputs\": \"Hi, what can you help me with?\",\n", " \"parameters\": {\n", " \"max_new_tokens\": 512,\n", " \"top_p\": 0.9,\n", @@ -525,7 +435,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 6. Qualitative Model Evaluation\n", + "### 5. Qualitative Model Evaluation\n", "Let's test the default DeepSeek-R1-Distill-Llama-8B using MLFlow's LLM-as-a-Judge capability. We'll use [Anthropic's Claude 3 Haiku](https://www.anthropic.com/news/claude-3-haiku) model on [Amazon Bedrock](https://aws.amazon.com/bedrock/) as the judge. We'll also wrap our model endpoint invocation in a method making it easier to call in the evaluation. \n", "\n", "This particular endpoint is the [cross-region inference endpoint](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html) name for Claude 3 Haiku.\n", @@ -993,6 +903,95 @@ " print(error_msg)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6. Templating a Prompt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the next workshop we fine-tune DeepSeek-R1-Distill-Llama-8B to become a medical expert. To accomplish this, we execute a fine-tuning job using Managed MLflow on SageMaker AI. We get our data from the [FreedomIntelligence/medical-o1-reasoning-SFT](https://huggingface.co/datasets/FreedomIntelligence/medical-o1-reasoning-SFT) dataset, available on HuggingFace.\n", + "\n", + "In this lab, we show a small example of what fine-tuning looks like for a single record of the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "FINE_TUNING_DATA_SAMPLE = {\n", + " \"Question\": \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\", \n", + " \"Complex_CoT\": \"Okay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem.\\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal.\\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\",\n", + " \"Response\": \"Cystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.\"\n", + "}\n", + "\n", + "\n", + "PROMPT_TEMPLATE = \"\"\"\n", + "<|begin_of_text|>\n", + " <|start_header_id|>system<|end_header_id|>\n", + " You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", + " Below is an instruction that describes a task, paired with an input that provides further context. \n", + " Write a response that appropriately completes the request.\n", + " Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", + " <|eot_id|>\n", + " <|start_header_id|>user<|end_header_id|>\n", + " {{question}}\n", + " <|eot_id|>\n", + " <|start_header_id|>assistant<|end_header_id|>\n", + " {{complex_cot}}\n", + " {{answer}}\n", + "<|eot_id|>\n", + "\"\"\"\n", + "\n", + "# Template dataset to add prompt to each sample\n", + "def template_dataset(sample):\n", + " try:\n", + " sample[\"text\"] = PROMPT_TEMPLATE.format(question=sample[\"Question\"],\n", + " complex_cot=sample[\"Complex_CoT\"],\n", + " answer=sample[\"Response\"])\n", + " return sample\n", + " except KeyError as e:\n", + " print(f\"KeyError in template_dataset: {str(e)}\")\n", + " # Provide default values for missing fields\n", + " missing_key = str(e).strip(\"'\")\n", + " if missing_key == \"Question\":\n", + " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", + " question=\"[Missing question]\",\n", + " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", + " answer=sample.get(\"Response\", \"[Missing response]\")\n", + " )\n", + " elif missing_key == \"Complex_CoT\":\n", + " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", + " question=sample[\"Question\"],\n", + " complex_cot=\"[Missing CoT]\",\n", + " answer=sample.get(\"Response\", \"[Missing response]\")\n", + " )\n", + " elif missing_key == \"Response\":\n", + " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", + " question=sample[\"Question\"],\n", + " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", + " answer=\"[Missing response]\"\n", + " )\n", + " return sample\n", + "\n", + "PROCESSED_SAMPLE = template_dataset(FINE_TUNING_DATA_SAMPLE)\n", + "print(PROCESSED_SAMPLE)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Fine-Tuning Output\n", + "The above output shows the templated prompt output to be used for fine-tuning. This pre-processing happens for every record in the fine-tuning dataset before fine-tuning actually takes place. This can be time-consuming for large fine-tuning datasets. We will show in the next lab how to orchestrate this with MLflow." + ] + }, { "cell_type": "markdown", "metadata": {}, From a8c1207e7fdc1412ff72484e11e7cbe1adab15b4 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Fri, 10 Oct 2025 15:46:37 -0400 Subject: [PATCH 10/22] Modernized the fine-tuning job --- .../task_05_fmops/05.00_fmops_examples.ipynb | 124 ++++------ .../05.01_fine-tuning-pipeline.ipynb | 35 +-- .../task_05_fmops/scripts/requirements.txt | 21 +- .../task_05_fmops/scripts/train.py | 221 +++++++----------- .../task_05_fmops/steps/deploy_step.py | 203 +++++++++------- .../task_05_fmops/steps/finetune_step.py | 129 ++++++---- .../task_05_fmops/steps/pipeline_utils.py | 103 ++++---- .../task_05_fmops/steps/preprocess_step.py | 139 ++++------- 8 files changed, 475 insertions(+), 500 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb index a1c09aa..0636d19 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -102,7 +102,8 @@ "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", - "role = sagemaker.get_execution_role()" + "role = sagemaker.get_execution_role()\n", + "region = sagemaker_session.boto_session.region_name" ] }, { @@ -114,7 +115,7 @@ "\n", "We define appropriate paths in S3 to store model files, define the model we will be working with, and define the model endpoint name.\n", "\n", - "In this lab, we are working with [DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B). It is easy to fine-tune as we will see in the next lab, and is small enough to fit on a reasonably sized GPU-accelerated hosting endpoint." + "In this lab, we are working with [Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507). It is easy to fine-tune as we will see in the next lab, and is small enough to fit on a reasonably sized GPU-accelerated hosting endpoint." ] }, { @@ -138,12 +139,14 @@ "metadata": {}, "outputs": [], "source": [ - "model_id = \"deepseek-ai/DeepSeek-R1-Distill-Llama-8B\"\n", + "model_id = \"Qwen/Qwen3-4B-Instruct-2507\"\n", "model_id_filesafe = model_id.replace(\"/\",\"_\").replace(\".\", \"_\")\n", "model_name_safe = model_id.split('/')[-1].replace('.', '-').replace('_', '-')\n", "endpoint_name = f\"Example-{model_name_safe}\"\n", "instance_count = 1\n", - "instance_type = \"ml.g5.2xlarge\"" + "instance_type = \"ml.g5.2xlarge\"\n", + "health_check_timeout = 1800\n", + "data_download_timeout = 3600" ] }, { @@ -170,15 +173,14 @@ "source": [ "mlflow_tracking_server_arn = \"\"\n", "\n", - "if not mlflow_tracking_server_arn:\n", - " try:\n", - " response = boto3.client('sagemaker').describe_mlflow_tracking_server(\n", - " TrackingServerName='genai-mlflow-tracker'\n", - " )\n", - " mlflow_tracking_server_arn = response['TrackingServerArn']\n", - " print(f\"MLflow Tracking Server ARN: {mlflow_tracking_server_arn}\")\n", - " except botocore.exceptions.ClientError:\n", - " print(\"No MLflow Tracking Server Found, please input a value for mlflow_tracking_server_arn\")\n", + "try:\n", + " response = boto3.client('sagemaker').describe_mlflow_tracking_server(\n", + " TrackingServerName='genai-mlflow-tracker'\n", + " )\n", + " mlflow_tracking_server_arn = response['TrackingServerArn']\n", + " print(f\"MLflow Tracking Server ARN: {mlflow_tracking_server_arn}\")\n", + "except botocore.exceptions.ClientError:\n", + " print(\"No MLflow Tracking Server Found, please input a value for mlflow_tracking_server_arn\")\n", "\n", "os.environ[\"mlflow_tracking_server_arn\"] = mlflow_tracking_server_arn" ] @@ -188,7 +190,7 @@ "metadata": {}, "source": [ "### 4. Model Deployment\n", - "There are several approaches to deploying a model to a SageMaker AI managed endpoint. In this section, we explore the most direct option which downloads a model directly from HuggingFace to the managed endpoint via SageMaker JumpStart. We are still using DeepSeek-R1-Distill-Llama-8B, but we have not fine-tuned it. The purpose of this section is to illustrate the components required to customize a model deployment on SageMaker before fine-tuning it." + "There are several approaches to deploying a model to a SageMaker AI managed endpoint. In this section, we explore the most direct option which downloads a model directly from HuggingFace to the managed endpoint via SageMaker JumpStart. We are still using Qwen3-4B-Instruct-2507, but we have not fine-tuned it. The purpose of this section is to illustrate the components required to customize a model deployment on SageMaker before fine-tuning it." ] }, { @@ -207,12 +209,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Create and deploy model\n", - "image_uri = sagemaker.image_uris.retrieve(\n", - " framework=\"djl-lmi\",\n", - " region=sagemaker_session.boto_session.region_name,\n", - " version=\"latest\"\n", - ")" + "inference_image_uri = f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.33.0-lmi15.0.0-cu128\"\n", + "print(f\"using image to host: {inference_image_uri}\")" ] }, { @@ -242,7 +240,7 @@ " 'OPTION_MAX_MODEL_LEN': '4096'\n", "}\n", "model = HuggingFaceModel(\n", - " image_uri=image_uri,\n", + " image_uri=inference_image_uri,\n", " env=model_config,\n", " role=role\n", ")" @@ -276,8 +274,6 @@ "with mlflow.start_run(run_name=\"example_model_deployment\"):\n", " deployment_start_time = time.time()\n", "\n", - " health_check_timeout = 1800\n", - " data_download_timeout = 3600\n", "\n", " # Log deployment parameters\n", " mlflow.log_params({\n", @@ -297,7 +293,7 @@ " instance_type=instance_type,\n", " container_startup_health_check_timeout=health_check_timeout,\n", " model_data_download_timeout=data_download_timeout,\n", - " endpoint_name=endpoint_name\n", + " endpoint_name=f\"{endpoint_name}\"\n", " )\n", "\n", " # Log deployment metrics\n", @@ -339,7 +335,7 @@ "from sagemaker.deserializers import JSONDeserializer\n", "\n", "predictor = Predictor(\n", - " endpoint_name=endpoint_name,\n", + " endpoint_name=f\"{endpoint_name}\",\n", " serializer=JSONSerializer(),\n", " deserializer=JSONDeserializer()\n", ")\n", @@ -436,7 +432,7 @@ "metadata": {}, "source": [ "### 5. Qualitative Model Evaluation\n", - "Let's test the default DeepSeek-R1-Distill-Llama-8B using MLFlow's LLM-as-a-Judge capability. We'll use [Anthropic's Claude 3 Haiku](https://www.anthropic.com/news/claude-3-haiku) model on [Amazon Bedrock](https://aws.amazon.com/bedrock/) as the judge. We'll also wrap our model endpoint invocation in a method making it easier to call in the evaluation. \n", + "Let's test the default Qwen3-4B-Instruct-2507 using MLFlow's LLM-as-a-Judge capability. We'll use [Anthropic's Claude 3 Haiku](https://www.anthropic.com/news/claude-3-haiku) model on [Amazon Bedrock](https://aws.amazon.com/bedrock/) as the judge. We'll also wrap our model endpoint invocation in a method making it easier to call in the evaluation. \n", "\n", "This particular endpoint is the [cross-region inference endpoint](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html) name for Claude 3 Haiku.\n", "\n", @@ -498,7 +494,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now use Managed MLFlow 3.0 on Amazon SageMaker AI's `EvaluationExample` object to provide examples of good and bad model responses. This synthetic data will be used to evaluate our Example DeepSeek-R1_Distill_Llama-8B along several qualitative metrics. We create these qualitative metrics using `make_genai_metric`." + "Now use Managed MLFlow 3.0 on Amazon SageMaker AI's `EvaluationExample` object to provide examples of good and bad model responses. This synthetic data will be used to evaluate our Example Qwen3-4B-Instruct-2507 along with several qualitative metrics. We create these qualitative metrics using `make_genai_metric`." ] }, { @@ -914,7 +910,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In the next workshop we fine-tune DeepSeek-R1-Distill-Llama-8B to become a medical expert. To accomplish this, we execute a fine-tuning job using Managed MLflow on SageMaker AI. We get our data from the [FreedomIntelligence/medical-o1-reasoning-SFT](https://huggingface.co/datasets/FreedomIntelligence/medical-o1-reasoning-SFT) dataset, available on HuggingFace.\n", + "In the next workshop we fine-tune Qwen3-4B-Instruct-2507 to become a medical expert. To accomplish this, we execute a fine-tuning job using Managed MLflow on SageMaker AI. We get our data from the [FreedomIntelligence/medical-o1-reasoning-SFT](https://huggingface.co/datasets/FreedomIntelligence/medical-o1-reasoning-SFT) dataset, available on HuggingFace.\n", "\n", "In this lab, we show a small example of what fine-tuning looks like for a single record of the dataset." ] @@ -931,56 +927,25 @@ " \"Response\": \"Cystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.\"\n", "}\n", "\n", + "SYSTEM_PROMPT = \"\"\"You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", + "Below is an instruction that describes a task, paired with an input that provides further context. \n", + "Write a response that appropriately completes the request.\n", + "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\"\"\"\n", "\n", - "PROMPT_TEMPLATE = \"\"\"\n", - "<|begin_of_text|>\n", - " <|start_header_id|>system<|end_header_id|>\n", - " You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", - " Below is an instruction that describes a task, paired with an input that provides further context. \n", - " Write a response that appropriately completes the request.\n", - " Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", - " <|eot_id|>\n", - " <|start_header_id|>user<|end_header_id|>\n", - " {{question}}\n", - " <|eot_id|>\n", - " <|start_header_id|>assistant<|end_header_id|>\n", - " {{complex_cot}}\n", - " {{answer}}\n", - "<|eot_id|>\n", - "\"\"\"\n", + "def convert_to_messages(sample, system_prompt=\"\"):\n", + " \n", + " messages = [\n", + " {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n", + " {\"role\": \"user\", \"content\": sample[\"Question\"]},\n", + " {\"role\": \"assistant\", \"content\": f\"{sample[\"Complex_CoT\"]}\\n\\n{sample[\"Response\"]}\"}\n", + " ]\n", + "\n", + " sample[\"messages\"] = messages\n", + " \n", + " return sample\n", "\n", - "# Template dataset to add prompt to each sample\n", - "def template_dataset(sample):\n", - " try:\n", - " sample[\"text\"] = PROMPT_TEMPLATE.format(question=sample[\"Question\"],\n", - " complex_cot=sample[\"Complex_CoT\"],\n", - " answer=sample[\"Response\"])\n", - " return sample\n", - " except KeyError as e:\n", - " print(f\"KeyError in template_dataset: {str(e)}\")\n", - " # Provide default values for missing fields\n", - " missing_key = str(e).strip(\"'\")\n", - " if missing_key == \"Question\":\n", - " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", - " question=\"[Missing question]\",\n", - " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", - " answer=sample.get(\"Response\", \"[Missing response]\")\n", - " )\n", - " elif missing_key == \"Complex_CoT\":\n", - " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", - " question=sample[\"Question\"],\n", - " complex_cot=\"[Missing CoT]\",\n", - " answer=sample.get(\"Response\", \"[Missing response]\")\n", - " )\n", - " elif missing_key == \"Response\":\n", - " sample[\"text\"] = PROMPT_TEMPLATE.format(\n", - " question=sample[\"Question\"],\n", - " complex_cot=sample.get(\"Complex_CoT\", \"[Missing CoT]\"),\n", - " answer=\"[Missing response]\"\n", - " )\n", - " return sample\n", "\n", - "PROCESSED_SAMPLE = template_dataset(FINE_TUNING_DATA_SAMPLE)\n", + "PROCESSED_SAMPLE = convert_to_messages(FINE_TUNING_DATA_SAMPLE)\n", "print(PROCESSED_SAMPLE)" ] }, @@ -1097,8 +1062,15 @@ "4. Creating and applying Guardrails to our model\n", "5. Tracing model calls using MLFlow tracing\n", "\n", - "Next, we show how to actually perform fine-tuning on this DeepSeek model to improve the model's performance in this domain. Moreover, we'll orchestrate all of these steps into a fine-tuning pipeline powered by Managed MLFlow and SageMaker AI Pipelines." + "Next, we show how to actually perform fine-tuning on this Qwen3 model to improve the model's performance in this domain. Moreover, we'll orchestrate all of these steps into a fine-tuning pipeline powered by Managed MLFlow and SageMaker AI Pipelines." ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index f75f23c..a1c57e2 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -114,7 +114,7 @@ "sagemaker_session = sagemaker.session.Session()\n", "role = sagemaker.get_execution_role()\n", "instance_type = \"ml.m5.xlarge\"\n", - "pipeline_name = \"AIM405-deepseek-finetune-pipeline\"\n", + "pipeline_name = \"AIM405-qwen3-finetune-pipeline\"\n", "bucket_name = sagemaker_session.default_bucket()\n", "default_prefix = sagemaker_session.default_bucket_prefix\n", "if default_prefix:\n", @@ -122,7 +122,7 @@ "else:\n", " input_path = f'datasets/llm-fine-tuning-modeltrainer-sft'\n", "\n", - "model_id = \"deepseek-ai/DeepSeek-R1-Distill-Llama-8B\"\n", + "model_id = \"Qwen/Qwen3-4B-Instruct-2507\"\n", "model_id_filesafe = model_id.replace(\"/\",\"_\").replace(\".\", \"_\")" ] }, @@ -157,15 +157,14 @@ "source": [ "mlflow_tracking_server_arn = \"\"\n", "\n", - "if not mlflow_tracking_server_arn:\n", - " try:\n", - " response = boto3.client('sagemaker').describe_mlflow_tracking_server(\n", - " TrackingServerName='genai-mlflow-tracker'\n", - " )\n", - " mlflow_tracking_server_arn = response['TrackingServerArn']\n", - " print(f\"MLflow Tracking Server ARN: {mlflow_tracking_server_arn}\")\n", - " except ClientError:\n", - " print(\"No MLflow Tracking Server Found, please input a value for mlflow_tracking_server_arn\")\n", + "try:\n", + " response = boto3.client('sagemaker').describe_mlflow_tracking_server(\n", + " TrackingServerName='genai-mlflow-tracker'\n", + " )\n", + " mlflow_tracking_server_arn = response['TrackingServerArn']\n", + " print(f\"MLflow Tracking Server ARN: {mlflow_tracking_server_arn}\")\n", + "except ClientError:\n", + " print(\"No MLflow Tracking Server Found, please input a value for mlflow_tracking_server_arn\")\n", "\n", "os.environ[\"mlflow_tracking_server_arn\"] = mlflow_tracking_server_arn\n", "os.environ[\"pipeline_name\"] = pipeline_name" @@ -520,16 +519,19 @@ " test_dataset_s3_path=preprocessing_step[2],\n", " train_config_s3_path=train_config_s3_path,\n", " role=role,\n", - " model_id=model_s3_destination,\n", + " model_id=model_s3_destination\n", ")\n", "run_id=training_step[0]\n", "model_artifacts_s3_path=training_step[2]\n", - "output_path=training_step[3]\n", + "# output_path=training_step[3]\n", "\n", "deploy_step = deploy_step.deploy(\n", + " tracking_server_arn=mlflow_tracking_server_arn,\n", " model_artifacts_s3_path=model_artifacts_s3_path,\n", - " output_path=output_path,\n", + " # output_path=output_path,\n", " model_id=model_s3_destination,\n", + " experiment_name=pipeline_name,\n", + " run_id=run_id,\n", ")\n", "endpoint_name=deploy_step\n", "\n", @@ -574,7 +576,7 @@ " run_id=run_id, # Assuming training_step returns run_id as first output\n", " model_artifacts_s3_path=model_artifacts_s3_path, # Assuming training_step returns artifacts path as second output\n", " model_id=model_id,\n", - " model_name=f\"Fine-Tuned-Medical-DeepSeek\",\n", + " model_name=f\"Fine-Tuned-Medical-Qwen3-4B-Instruct-2507\",\n", " endpoint_name=endpoint_name,\n", " evaluation_score=quantitative_eval_step[\"rougeL_f\"], # Get the evaluation score\n", " pipeline_name=pipeline_name,\n", @@ -728,8 +730,7 @@ "\n", "# Clean up endpoint\n", "try:\n", - " model_name_safe = model_id.split('/')[-1].replace('.', '-').replace('_', '-')\n", - " endpoint_name = f\"{model_name_safe}-sft-djl\"\n", + " endpoint_name = f\"{model_id.replace('/', '-').replace('_', '-')}-sft-djl\"\n", " \n", " print(f\"Cleaning up endpoint: {endpoint_name}\")\n", " if delete_endpoint_with_retry(endpoint_name):\n", diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/requirements.txt b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/requirements.txt index 3b58af5..6d003dd 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/requirements.txt +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/requirements.txt @@ -1,21 +1,20 @@ -awscli==1.42.25 -transformers==4.50.2 +transformers==4.52.2 peft==0.14.0 accelerate==1.3.0 bitsandbytes==0.45.1 -datasets==3.5.0 +datasets==3.2.0 evaluate==0.4.3 huggingface_hub[hf_transfer]==0.33.4 -mlflow +mlflow==2.22.2 safetensors>=0.5.2 -sagemaker==2.244.0 +sagemaker==2.252.0 sagemaker-mlflow==0.1.0 sentencepiece==0.2.0 scikit-learn==1.6.1 tokenizers>=0.21.0 -trl==0.9.6 -psutil -py7zr -pynvml -xtarfile -rouge-score \ No newline at end of file +trl==0.18.0 +psutil==7.1.0 +py7zr==1.0.0 +pynvml==13.0.1 +xtarfile==0.2.1 +rouge-score==0.1.2 \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/train.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/train.py index c29da1b..2ad6b3e 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/train.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/scripts/train.py @@ -1,27 +1,21 @@ import os import datetime -from typing import Dict, Optional, Tuple +from typing import Dict, Optional from dataclasses import dataclass, field -from functools import partial -from itertools import chain from accelerate import Accelerator -import bitsandbytes as bnb from huggingface_hub import snapshot_download from datasets import load_dataset import mlflow -from mlflow.models import infer_signature import torch -import transformers from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TrainingArguments, set_seed -from peft import AutoPeftModelForCausalLM, LoraConfig, get_peft_model, prepare_model_for_kbit_training +from peft import AutoPeftModelForCausalLM, LoraConfig, prepare_model_for_kbit_training -from trl.commands.cli_utils import TrlParser -from trl import SFTTrainer +from trl import SFTTrainer, TrlParser from sagemaker.s3 import S3Downloader import subprocess @@ -118,35 +112,6 @@ def download_model(model_name): print(f"Model {model_name} downloaded under {destination}") -def group_texts(examples, block_size=2048): - """ - Groups a list of tokenized text examples into fixed-size blocks for language model training. - - Args: - examples (dict): A dictionary where keys are feature names (e.g., "input_ids") and values - are lists of tokenized sequences. - block_size (int, optional): The size of each chunk. Defaults to 2048. - - Returns: - dict: A dictionary containing the grouped chunks for each feature. An additional "labels" key - is included, which is a copy of the "input_ids" key. - """ - # Concatenate all texts. - concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()} - total_length = len(concatenated_examples[list(examples.keys())[0]]) - # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can - # customize this part to your needs. - if total_length >= block_size: - total_length = (total_length // block_size) * block_size - # Split by chunks of max_len. - result = { - k: [t[i : i + block_size] for i in range(0, total_length, block_size)] - for k, t in concatenated_examples.items() - } - result["labels"] = result["input_ids"].copy() - return result - - def set_custom_env(env_vars: Dict[str, str]) -> None: """ Set custom environment variables. @@ -176,10 +141,35 @@ def set_custom_env(env_vars: Dict[str, str]) -> None: for key, value in env_vars.items(): print(f" {key}: {value}") +def load_data(training_data_location, test_data_location): + # Load datasets + train_ds = load_dataset( + "json", + data_files=os.path.join(training_data_location, "dataset.json"), + split="train" + ) + + if script_args.test_dataset_path: + test_ds = load_dataset( + "json", + data_files=os.path.join(test_data_location, "dataset.json"), + split="train" + ) + else: + test_ds = None -def train(script_args, training_args, train_ds, test_ds): + return train_ds, test_ds + +def train(script_args, training_args): set_seed(training_args.seed) + mlflow_enabled = ( + script_args.mlflow_uri is not None + and script_args.mlflow_experiment_name is not None + and script_args.mlflow_uri != "" + and script_args.mlflow_experiment_name != "" + ) + accelerator = Accelerator() if script_args.token is not None: @@ -202,19 +192,21 @@ def train(script_args, training_args, train_ds, test_ds): # Set Tokenizer pad Token tokenizer.pad_token = tokenizer.eos_token - # tokenize and chunk dataset - lm_train_dataset = train_ds.map( - lambda sample: tokenizer(sample["text"]), remove_columns=list(train_ds.features) - ) + # # tokenize and chunk dataset + # lm_train_dataset = train_ds.map( + # lambda sample: tokenizer(sample["text"]), remove_columns=list(train_ds.features) + # ) - if test_ds is not None: - lm_test_dataset = test_ds.map( - lambda sample: tokenizer(sample["text"]), remove_columns=list(train_ds.features) - ) + # if test_ds is not None: + # lm_test_dataset = test_ds.map( + # lambda sample: tokenizer(sample["text"]), remove_columns=list(train_ds.features) + # ) - print(f"Total number of test samples: {len(lm_test_dataset)}") - else: - lm_test_dataset = None + # print(f"Total number of test samples: {len(lm_test_dataset)}") + # else: + # lm_test_dataset = None + + train_ds, test_ds = load_data(script_args.train_dataset_path, script_args.test_dataset_path) accelerator.wait_for_everyone() @@ -276,7 +268,7 @@ def train(script_args, training_args, train_ds, test_ds): ) if training_args.fsdp is None and training_args.fsdp_config is None: - model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=gradient_checkpointing) + model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=training_args.gradient_checkpointing) if training_args.gradient_checkpointing: model.gradient_checkpointing_enable() @@ -284,7 +276,7 @@ def train(script_args, training_args, train_ds, test_ds): if training_args.gradient_checkpointing: model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) - config = LoraConfig( + peft_config = LoraConfig( r=script_args.lora_r, lora_alpha=script_args.lora_alpha, target_modules="all-linear", @@ -293,47 +285,42 @@ def train(script_args, training_args, train_ds, test_ds): task_type="CAUSAL_LM" ) - model = get_peft_model(model, config) - print(f"max_seq_length: {script_args.max_seq_length}") + + print("Disabling checkpointing and setting up logging") + training_args.save_strategy="no" + training_args.logging_strategy="steps" + training_args.logging_steps=1 + training_args.log_on_each_node=False + training_args.ddp_find_unused_parameters=False trainer = SFTTrainer( model=model, - train_dataset=lm_train_dataset, - eval_dataset=lm_test_dataset if lm_test_dataset is not None else None, - max_seq_length=script_args.max_seq_length, - args=transformers.TrainingArguments( - per_device_train_batch_size=training_args.per_device_train_batch_size, - per_device_eval_batch_size=training_args.per_device_eval_batch_size, - gradient_accumulation_steps=training_args.gradient_accumulation_steps, - logging_strategy="steps", - logging_steps=1, - log_on_each_node=False, - num_train_epochs=training_args.num_train_epochs, - learning_rate=training_args.learning_rate, - bf16=training_args.bf16, - fp16=training_args.fp16, - ddp_find_unused_parameters=False, - save_strategy="no", - output_dir="outputs", - **trainer_configs - ), - callbacks=None, - data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False), + args=training_args, + train_dataset=train_ds, + eval_dataset=test_ds if test_ds is not None else None, + processing_class=tokenizer, + peft_config=peft_config ) if trainer.accelerator.is_main_process: trainer.model.print_trainable_parameters() - print("MLflow tracking under ", script_args.mlflow_experiment_name) - - train_dataset_mlflow = mlflow.data.from_pandas(train_ds.to_pandas(), name="train_dataset") - mlflow.log_input(train_dataset_mlflow, context="train") + if mlflow_enabled: + print("MLflow tracking under ", script_args.mlflow_experiment_name) + # mlflow.start_run(run_id=os.environ.get["MLFLOW_RUN_ID"]) + # mlflow.start_run(run_name=os.environ.get["MLFLOW_RUN_NAME"], nested=True) + mlflow.start_run(run_name=os.environ.get("MLFLOW_RUN_NAME", None)) + train_dataset_mlflow = mlflow.data.from_pandas(train_ds.to_pandas(), name="train_dataset") + mlflow.log_input(train_dataset_mlflow, context="train") - test_dataset_mlflow = mlflow.data.from_pandas(test_ds.to_pandas(), name="test_dataset") - mlflow.log_input(test_dataset_mlflow, context="test") + if test_ds is not None: + test_dataset_mlflow = mlflow.data.from_pandas(test_ds.to_pandas(), name="test_dataset") + mlflow.log_input(test_dataset_mlflow, context="test") - trainer.train() + trainer.train() + else: + trainer.train() if trainer.is_fsdp_enabled: trainer.accelerator.state.fsdp_plugin.set_state_dict_type("FULL_STATE_DICT") @@ -344,7 +331,7 @@ def train(script_args, training_args, train_ds, test_ds): # merge adapter weights with base model and save # save int 4 model - trainer.model.save_pretrained(output_dir, safe_serialization=False) + trainer.save_model(output_dir) if accelerator.is_main_process: # clear memory @@ -370,35 +357,16 @@ def train(script_args, training_args, train_ds, test_ds): print("saving merged model...") model.save_pretrained( - training_args.output_dir, safe_serialization=True, max_shard_size="2GB" + training_args.output_dir, + safe_serialization=True ) else: print(f"merge adapter weights: {script_args.merge_weights}") - trainer.model.save_pretrained(training_args.output_dir, safe_serialization=True) + trainer.save_model(training_args.output_dir) if accelerator.is_main_process: tokenizer.save_pretrained(training_args.output_dir) - # if mlflow_enabled: - # # Model registration in MLFlow - # print("MLflow model registration under ", script_args.mlflow_experiment_name) - - # params = { - # "top_p": 0.9, - # "temperature": 0.2, - # "max_new_tokens": 2048, - # } - # signature = infer_signature("inputs", "generated_text", params=params) - - # mlflow.transformers.log_model( - # transformers_model={"model": model, "tokenizer": tokenizer}, - # signature=signature, - # artifact_path="model", # This is a relative path to save model files within MLflow run - # model_config=params, - # task="text-generation", - # registered_model_name=f"model-{os.environ.get('MLFLOW_RUN_NAME', '').split('Fine-tuning-')[-1]}" - # ) - accelerator.wait_for_everyone() @@ -414,30 +382,19 @@ def train(script_args, training_args, train_ds, test_ds): set_custom_env({"HF_HUB_ENABLE_HF_TRANSFER": "1"}) - mlflow.set_tracking_uri(script_args.mlflow_uri) - mlflow.set_experiment(script_args.mlflow_experiment_name) - mlflow_run_id = os.environ.get("MLFLOW_RUN_ID") - with mlflow.start_run(run_id=mlflow_run_id): - with mlflow.start_run(run_name="Finetuning", nested=True) as training_run: - - mlflow.enable_system_metrics_logging() - mlflow.autolog() - - # Load datasets - train_ds = load_dataset( - "json", - data_files=os.path.join(script_args.train_dataset_path, "dataset.json"), - split="train" - ) - - if script_args.test_dataset_path: - test_ds = load_dataset( - "json", - data_files=os.path.join(script_args.test_dataset_path, "dataset.json"), - split="train" - ) - else: - test_ds = None - - # launch training - train(script_args, training_args, train_ds, test_ds) + if script_args.mlflow_uri is not None and script_args.mlflow_experiment_name is not None and \ + script_args.mlflow_uri != "" and script_args.mlflow_experiment_name != "": + print("mlflow init") + mlflow.enable_system_metrics_logging() + mlflow.autolog() + mlflow.set_tracking_uri(script_args.mlflow_uri) + mlflow.set_experiment(script_args.mlflow_experiment_name) + + current_datetime = datetime.datetime.now() + formatted_datetime = current_datetime.strftime("%Y-%m-%d-%H-%M") + model_name = script_args.model_id.split("/")[-1] + set_custom_env({"MLFLOW_RUN_NAME": f"Fine-tuning-{model_name}-{formatted_datetime}"}) + set_custom_env({"MLFLOW_EXPERIMENT_NAME": script_args.mlflow_experiment_name}) + + # launch training + train(script_args, training_args) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py index 325847b..c163c02 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py @@ -3,6 +3,7 @@ import sagemaker import boto3 +import mlflow from sagemaker import get_execution_role from sagemaker import Model from sagemaker.model_monitor import DataCaptureConfig @@ -18,97 +19,133 @@ keep_alive_period_in_seconds=900 ) def deploy( + tracking_server_arn: str, model_artifacts_s3_path: str, - output_path: str, + # output_path: str, model_id: str, + experiment_name: str, + run_id: str, ): - sagemaker_session = sagemaker.Session() - instance_count = 1 - instance_type = "ml.g5.2xlarge" - health_check_timeout = 700 - - # Get the name for the endpoint - endpoint_name = f"{model_id.split('/')[-1].replace('.', '-').replace('_','-')}-sft-djl" - - # Delete existing endpoint if it exists - print(f"Checking for existing endpoint: {endpoint_name}") - sm_client = boto3.client('sagemaker') - try: - sm_client.describe_endpoint(EndpointName=endpoint_name) - print(f"Endpoint {endpoint_name} exists, deleting it before deployment") - sm_client.delete_endpoint(EndpointName=endpoint_name) - print(f"Deleting endpoint config {endpoint_name}") - sm_client.delete_endpoint_config(EndpointConfigName=endpoint_name) - - # Wait for endpoint to be fully deleted - print("Waiting for endpoint to be fully deleted...") - wait_seconds = 10 - total_wait_time = 0 - max_wait_time = 300 # 5 minutes maximum wait - endpoint_deleted = False - - while total_wait_time < max_wait_time and not endpoint_deleted: + mlflow.set_tracking_uri(tracking_server_arn) + mlflow.set_experiment(experiment_name) + + with mlflow.start_run(run_id=run_id): + with mlflow.start_run(run_name="DeployStep", nested=True) as deploy_run: + deployment_start_time = time.time() + + sagemaker_session = sagemaker.Session() + instance_count = 1 + instance_type = "ml.g5.2xlarge" + health_check_timeout = 700 + model_data_download_timeout = 3600 + + model_config = { + 'HF_MODEL_ID': "/opt/ml/model", # path to where sagemaker stores the model + 'OPTION_TRUST_REMOTE_CODE': 'true', + 'OPTION_ROLLING_BATCH': "vllm", + 'OPTION_DTYPE': 'bf16', + 'OPTION_QUANTIZE': 'fp8', + 'OPTION_TENSOR_PARALLEL_DEGREE': 'max', + 'OPTION_MAX_ROLLING_BATCH_SIZE': '32', + 'OPTION_MODEL_LOADING_TIMEOUT': '3600', + 'OPTION_MAX_MODEL_LEN': '4096' + } + + # Get the name for the endpoint + endpoint_name = f"{model_id.split('/')[-1].replace('.', '-').replace('_','-')}-sft-djl" + + mlflow.log_params({ + "model_id": model_id, + "instance_type": instance_type, + "instance_count": instance_count, + "endpoint_name": endpoint_name, + "health_check_timeout": health_check_timeout, + "model_data_download_timeout": model_data_download_timeout + }) + mlflow.log_params({"model_config_" + k: v for k, v in model_config.items()}) + + # Delete existing endpoint if it exists + print(f"Checking for existing endpoint: {endpoint_name}") + sm_client = boto3.client('sagemaker') try: sm_client.describe_endpoint(EndpointName=endpoint_name) - print(f"Endpoint still exists, waiting {wait_seconds} seconds...") - time.sleep(wait_seconds) - total_wait_time += wait_seconds - except sm_client.exceptions.ClientError: - print(f"Endpoint {endpoint_name} successfully deleted") - endpoint_deleted = True + print(f"Endpoint {endpoint_name} exists, deleting it before deployment") + sm_client.delete_endpoint(EndpointName=endpoint_name) + + print(f"Deleting endpoint config {endpoint_name}") + sm_client.delete_endpoint_config(EndpointConfigName=endpoint_name) + + # Wait for endpoint to be fully deleted + print("Waiting for endpoint to be fully deleted...") + wait_seconds = 10 + total_wait_time = 0 + max_wait_time = 300 # 5 minutes maximum wait + endpoint_deleted = False - if not endpoint_deleted: - print(f"Warning: Endpoint still exists after {max_wait_time} seconds") + while total_wait_time < max_wait_time and not endpoint_deleted: + try: + sm_client.describe_endpoint(EndpointName=endpoint_name) + print(f"Endpoint still exists, waiting {wait_seconds} seconds...") + time.sleep(wait_seconds) + total_wait_time += wait_seconds + except sm_client.exceptions.ClientError: + print(f"Endpoint {endpoint_name} successfully deleted") + endpoint_deleted = True + + if not endpoint_deleted: + print(f"Warning: Endpoint still exists after {max_wait_time} seconds") + + except sm_client.exceptions.ClientError: + print(f"Endpoint {endpoint_name} does not exist, proceeding with deployment") - except sm_client.exceptions.ClientError: - print(f"Endpoint {endpoint_name} does not exist, proceeding with deployment") - - # Continue with model deployment - image_uri = sagemaker.image_uris.retrieve( - framework="djl-lmi", - region=sagemaker_session.boto_session.region_name, - version="latest" - ) - - model_data = model_artifacts_s3_path - - # Create model only once - model = Model( - image_uri=image_uri, - model_data=model_data, - role=get_execution_role(), - env={ - 'HF_MODEL_ID': "/opt/ml/model", # path to where sagemaker stores the model - 'OPTION_TRUST_REMOTE_CODE': 'true', - 'OPTION_ROLLING_BATCH': "vllm", - 'OPTION_DTYPE': 'bf16', - 'OPTION_QUANTIZE': 'fp8', - 'OPTION_TENSOR_PARALLEL_DEGREE': 'max', - 'OPTION_MAX_ROLLING_BATCH_SIZE': '32', - 'OPTION_MODEL_LOADING_TIMEOUT': '3600', - 'OPTION_MAX_MODEL_LEN': '4096' - } - ) - - print(f"deploying endpoint: {endpoint_name}") + # Continue with model deployment + image_uri = sagemaker.image_uris.retrieve( + framework="djl-lmi", + region=sagemaker_session.boto_session.region_name, + version="latest" + ) + + model_data = model_artifacts_s3_path + + # Create model only once + model = Model( + image_uri=image_uri, + model_data=model_data, + role=get_execution_role(), + env=model_config + ) + + print(f"deploying endpoint: {endpoint_name}") + + data_capture_config = DataCaptureConfig( + enable_capture=True, + sampling_percentage=100, + destination_s3_uri='s3://sagemaker-us-east-1-329542461890/data-capture/', + capture_options=["REQUEST", "RESPONSE"], + csv_content_types=["text/csv"], + json_content_types=["application/json"] + ) + + predictor = model.deploy( + endpoint_name=endpoint_name, + initial_instance_count=instance_count, + instance_type=instance_type, + container_startup_health_check_timeout=health_check_timeout, + model_data_download_timeout=model_data_download_timeout, + data_capture_config=data_capture_config + ) + + # Log deployment metrics + deployment_time = time.time() - deployment_start_time + mlflow.log_param("deployment_time_seconds", deployment_time) + mlflow.log_param("deployment_success", 1) - data_capture_config = DataCaptureConfig( - enable_capture=True, - sampling_percentage=100, - destination_s3_uri='s3://sagemaker-us-east-1-329542461890/data-capture/', - capture_options=["REQUEST", "RESPONSE"], - csv_content_types=["text/csv"], - json_content_types=["application/json"] - ) - - predictor = model.deploy( - endpoint_name=endpoint_name, - initial_instance_count=instance_count, - instance_type=instance_type, - container_startup_health_check_timeout=health_check_timeout, - model_data_download_timeout=3600, - data_capture_config=data_capture_config - ) + # Log tags + mlflow.set_tags({ + "endpoint_status": "deployed", + "deployment_type": "sagemaker", + "framework": "djl-lmi" + }) return endpoint_name \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_step.py index 42786fd..7a23fa1 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/finetune_step.py @@ -1,6 +1,6 @@ # **Fine-tuning Step** -# This is where the actual model adaptation occurs. The step takes the preprocessed data and applies it to fine-tune the base LLM (in this case, a Deepseek model). It incorporates the LoRA technique for efficient adaptation. +# This is where the actual model adaptation occurs. The step takes the preprocessed data and applies it to fine-tune the base LLM (in this case, a Qwen model). It incorporates the LoRA technique for efficient adaptation. import sagemaker import boto3 @@ -17,6 +17,12 @@ from sagemaker.workflow.function_step import step from .pipeline_utils import PIPELINE_INSTANCE_TYPE +from sagemaker import image_uris +from sagemaker.modules.configs import Compute, InputData, OutputDataConfig, SourceCode, StoppingCondition +from sagemaker.modules.distributed import Torchrun +from sagemaker.modules.train import ModelTrainer +from sagemaker.modules.configs import InputData + @step( name="ModelFineTuning", instance_type=PIPELINE_INSTANCE_TYPE, @@ -35,7 +41,6 @@ def train( run_id: str, ): - # Initialize variables and tracking start_time = time.time() model_name = model_id.split("/")[-1] if "/" in model_id else model_id @@ -93,12 +98,7 @@ def train( # Clean up temp file if os.path.exists(tmp.name): os.remove(tmp.name) - - # Launch the training job - job_name = f"deepseek-finetune-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}" - - sagemaker_session = sagemaker.Session() - + # Define metric definitions for more detailed CloudWatch metrics metric_definitions = [ {'Name': 'loss', 'Regex': "'loss':\\s*([0-9.]+)"}, @@ -111,56 +111,93 @@ def train( # Log the metric definitions we're using mlflow.log_param("tracked_metrics", [m['Name'] for m in metric_definitions]) + + sagemaker_session = sagemaker.Session() + image_uri = sagemaker.image_uris.retrieve( + framework="pytorch", + version="2.6.0", + instance_type="ml.g5.2xlarge", + region=sagemaker_session.boto_session.region_name, + image_scope="training" + ) - pytorch_estimator = PyTorch( - entry_point='train.py', + source_code = SourceCode( source_dir="./scripts", - job_name=job_name, - base_job_name=job_name, - max_run=50000, - role=role, - framework_version="2.2.0", - py_version="py310", + requirements="requirements.txt", + entry_script="train.py", + ) + + # Define the compute + compute_configs = Compute( + instance_type="ml.g5.2xlarge", instance_count=1, - instance_type="ml.p3.2xlarge", - sagemaker_session=sagemaker_session, - volume_size=50, - disable_output_compression=False, - keep_alive_period_in_seconds=1800, - distribution={"torch_distributed": {"enabled": True}}, + keep_alive_period_in_seconds=3600, + volume_size_in_gb=50 + ) + + # Launch the training job + job_name = f"qwen3-finetune-{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}" + model_artifacts_s3_path = f"s3://{sagemaker_session.default_bucket()}/qwen3-finetune/" + + # Define the ModelTrainer + model_trainer = ModelTrainer( + training_image=image_uri, + source_code=source_code, + base_job_name=job_name, + compute=compute_configs, + distributed=Torchrun(), + stopping_condition=StoppingCondition( + max_runtime_in_seconds=7200 + ), hyperparameters={ - "config": "/opt/ml/input/data/config/args.yaml" + "config": "/opt/ml/input/data/config/args.yaml" # path to TRL config which was uploaded to s3 }, - metric_definitions=metric_definitions, - debugger_hook_config=False, - environment={"MLFLOW_RUN_ID": training_run_id} + output_data_config=OutputDataConfig( + s3_output_path=model_artifacts_s3_path + ), + environment={ + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True" + # "MLFLOW_RUN_ID": run_id + } + ) + + # Pass the input data + train_input = InputData( + channel_name="train", + data_source=train_dataset_s3_path, # S3 path where training data is stored ) - - # Define a data input dictionary with our uploaded S3 URIs - data = { - 'train': train_dataset_s3_path, - 'test': test_dataset_s3_path, - 'config': train_config_s3_path - } - - print(f"Data for Training Run: {data}") + test_input = InputData( + channel_name="test", + data_source=test_dataset_s3_path, # S3 path where training data is stored + ) + + config_input = InputData( + channel_name="config", + data_source=train_config_s3_path, # S3 path where training data is stored + ) + + # Check input channels configured + data = [train_input, test_input, config_input] + mlflow.log_param("datasets", data) + + print(f"Data for Training Run: {data}") # Log training job information mlflow.log_param("job_name", job_name) - mlflow.log_param("instance_type", "ml.p3.2xlarge") + mlflow.log_param("instance_type", "ml.g5.2xlarge") # Start the training job - pytorch_estimator.fit(data, wait=True) + model_trainer.train(input_data_config=data, wait=True) # Get information about the completed training job - latest_run_job_name = pytorch_estimator.latest_training_job.job_name - print(f"Latest Job Name: {latest_run_job_name}") + latest_training_job_name = model_trainer._latest_training_job.training_job_name + print(f"Latest Job Name: {latest_training_job_name}") sagemaker_client = boto3.client('sagemaker') # Describe the training job - response = sagemaker_client.describe_training_job(TrainingJobName=latest_run_job_name) - + response = sagemaker_client.describe_training_job(TrainingJobName=latest_training_job_name) + # Extract the model artifacts S3 path model_artifacts_s3_path = response['ModelArtifacts']['S3ModelArtifacts'] @@ -179,13 +216,13 @@ def train( # Log job results and metrics to MLflow # Log basic job info - mlflow.log_param("training_job_name", latest_run_job_name) + mlflow.log_param("training_job_name", latest_training_job_name) mlflow.log_param("model_artifacts_path", model_artifacts_s3_path) mlflow.log_param("output_path", output_path) # Log performance metrics - mlflow.log_metric("billable_time_seconds", billable_time) - mlflow.log_metric("total_training_time_seconds", total_training_time) + mlflow.log_param("billable_time_seconds", billable_time) + mlflow.log_param("total_training_time_seconds", total_training_time) # Log training job status mlflow.log_param("training_job_status", response.get('TrainingJobStatus')) @@ -201,7 +238,7 @@ def train( # Get CloudWatch logs for the training job logs_client = boto3.client('logs') log_group = "/aws/sagemaker/TrainingJobs" - log_stream = latest_run_job_name + log_stream = latest_training_job_name try: # Get the last 1000 log events @@ -241,4 +278,4 @@ def train( raise RuntimeError(f"Fine-tuning failed: {str(e)}") - return run_id, training_run_id, model_artifacts_s3_path, output_path \ No newline at end of file + return run_id, training_run_id, model_artifacts_s3_path #, output_path \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py index 3e9cd92..5c2ffb8 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py @@ -8,21 +8,26 @@ PIPELINE_INSTANCE_TYPE = "ml.m5.xlarge" -PROMPT_TEMPLATE = f""" -<|begin_of_text|> -<|start_header_id|>system<|end_header_id|> -You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. +# PROMPT_TEMPLATE = f""" +# <|begin_of_text|> +# <|start_header_id|>system<|end_header_id|> +# You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. +# Below is an instruction that describes a task, paired with an input that provides further context. +# Write a response that appropriately completes the request. +# Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. +# <|eot_id|><|start_header_id|>user<|end_header_id|> +# {{question}}<|eot_id|> +# <|start_header_id|>assistant<|end_header_id|> +# {{complex_cot}} + +# {{answer}} +# <|eot_id|> +# """ + +SYSTEM_PROMPT = """You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. -Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. -<|eot_id|><|start_header_id|>user<|end_header_id|> -{{question}}<|eot_id|> -<|start_header_id|>assistant<|end_header_id|> -{{complex_cot}} - -{{answer}} -<|eot_id|> -""" +Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.""" def endpoint_exists(endpoint_name): @@ -44,36 +49,50 @@ def create_training_job_name(model_id): return f"{model_id}-{datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]}" +# template dataset to add prompt to each sample +def convert_to_messages(sample, system_prompt=""): + + messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": sample["Question"]}, + {"role": "assistant", "content": f"{sample["Complex_CoT"]}\n\n{sample["Response"]}"} + ] + + sample["messages"] = messages + + return sample + + # Template dataset to add prompt to each sample -def template_dataset(sample): - try: - sample["text"] = PROMPT_TEMPLATE.format(question=sample["Question"], - complex_cot=sample["Complex_CoT"], - answer=sample["Response"]) - return sample - except KeyError as e: - print(f"KeyError in template_dataset: {str(e)}") - # Provide default values for missing fields - missing_key = str(e).strip("'") - if missing_key == "Question": - sample["text"] = PROMPT_TEMPLATE.format( - question="[Missing question]", - complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), - answer=sample.get("Response", "[Missing response]") - ) - elif missing_key == "Complex_CoT": - sample["text"] = PROMPT_TEMPLATE.format( - question=sample["Question"], - complex_cot="[Missing CoT]", - answer=sample.get("Response", "[Missing response]") - ) - elif missing_key == "Response": - sample["text"] = PROMPT_TEMPLATE.format( - question=sample["Question"], - complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), - answer="[Missing response]" - ) - return sample +# def template_dataset(sample): +# try: +# sample["text"] = PROMPT_TEMPLATE.format(question=sample["Question"], +# complex_cot=sample["Complex_CoT"], +# answer=sample["Response"]) +# return sample +# except KeyError as e: +# print(f"KeyError in template_dataset: {str(e)}") +# # Provide default values for missing fields +# missing_key = str(e).strip("'") +# if missing_key == "Question": +# sample["text"] = PROMPT_TEMPLATE.format( +# question="[Missing question]", +# complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), +# answer=sample.get("Response", "[Missing response]") +# ) +# elif missing_key == "Complex_CoT": +# sample["text"] = PROMPT_TEMPLATE.format( +# question=sample["Question"], +# complex_cot="[Missing CoT]", +# answer=sample.get("Response", "[Missing response]") +# ) +# elif missing_key == "Response": +# sample["text"] = PROMPT_TEMPLATE.format( +# question=sample["Question"], +# complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), +# answer="[Missing response]" +# ) +# return sample def invoke_sagemaker_endpoint(payload, endpoint_name): diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py index 19a9254..3a9b97c 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/preprocess_step.py @@ -16,7 +16,9 @@ from sagemaker.workflow.function_step import step from .pipeline_utils import ( PIPELINE_INSTANCE_TYPE, - template_dataset + # template_dataset, + SYSTEM_PROMPT, + convert_to_messages ) @@ -55,113 +57,64 @@ def preprocess( input_path = f'datasets/llm-fine-tuning-modeltrainer-sft' # Load dataset with proper error handling - sample_dataset_size = 100 + num_samples = 100 try: - dataset = load_dataset("FreedomIntelligence/medical-o1-reasoning-SFT", "en") + full_dataset = load_dataset("FreedomIntelligence/medical-o1-reasoning-SFT", "en", split=f"train[:{num_samples}]") except Exception as e: error_msg = f"Error loading dataset: {str(e)}\n{traceback.format_exc()}" print(error_msg) raise RuntimeError(f"Failed to load dataset: {str(e)}") - df = pd.DataFrame(dataset['train']) - df = df[:sample_dataset_size] - # Split dataset - train, test = train_test_split(df, test_size=0.1, random_state=42, shuffle=True) - - print("Number of train elements: ", len(train)) - print("Number of test elements: ", len(test)) + train_test_split_datasets = full_dataset.train_test_split(test_size=0.1, seed=42, shuffle=True) + train_dataset = train_test_split_datasets["train"] + test_dataset = train_test_split_datasets["test"] + print(f"Number of train elements: {len(train_dataset)}") + print(f"Number of test elements: {len(test_dataset)}") + + train_dataset = train_dataset.map(convert_to_messages, remove_columns=list(full_dataset.features), fn_kwargs={"system_prompt": SYSTEM_PROMPT}) + test_dataset = test_dataset.map(convert_to_messages, remove_columns=list(full_dataset.features), fn_kwargs={"system_prompt": SYSTEM_PROMPT}) + #grab a sample from the training and test sets + print(f"Train Sample:\n{train_dataset[randint(0, len(train_dataset)-1)]}\n\n") + print(f"Test Sample:\n{test_dataset[randint(0, len(test_dataset)-1)]}\n\n") # Log dataset statistics if MLflow is enabled mlflow.log_param("dataset_source", "FreedomIntelligence/medical-o1-reasoning-SFT") - mlflow.log_param("train_size", len(train)) - mlflow.log_param("test_size", len(test)) - mlflow.log_param("dataset_sample_size", sample_dataset_size) # Log that we're using a subset of 100 samples - - # Create datasets - train_dataset = Dataset.from_pandas(train) - test_dataset = Dataset.from_pandas(test) - dataset = DatasetDict({"train": train_dataset, "test": test_dataset}) - train_dataset = dataset["train"].map(template_dataset, remove_columns=list(dataset["train"].features)) - test_dataset = dataset["test"].map(template_dataset, remove_columns=list(dataset["test"].features)) - - # Safely get a sample text, handling potential index errors - try: - sample_index = randint(0, len(train_dataset) - 1) - sample_text = train_dataset[sample_index]["text"] - print(f"Sample text from index {sample_index}:") - print(sample_text) - except (IndexError, KeyError) as e: - sample_text = "Error retrieving sample text: " + str(e) - print(sample_text) - - # Create directories with error handling - try: - os.makedirs("./data/train", exist_ok=True) - os.makedirs("./data/test", exist_ok=True) - except OSError as e: - error_msg = f"Error creating directories: {str(e)}" - print(error_msg) - - # Save datasets locally with error handling - try: - train_dataset.to_json("./data/train/dataset.json", orient="records") - test_dataset.to_json("./data/test/dataset.json", orient="records") - except Exception as e: - error_msg = f"Error saving datasets locally: {str(e)}\n{traceback.format_exc()}" - print(error_msg) - raise RuntimeError(f"Failed to save datasets locally: {str(e)}") - - # Define S3 paths - train_data_path = f"s3://{bucket_name}/{input_path}/train/dataset.json" - test_dataset_path = f"s3://{bucket_name}/{input_path}/test/dataset.json" - - # Store results for return - result_train_data_path = train_data_path - result_test_dataset_path = test_dataset_path - - # Log dataset paths if MLflow is enabled - mlflow.log_param("train_data_path", train_data_path) - mlflow.log_param("test_dataset_path", test_dataset_path) - - # Upload files to S3 with retries - max_retries = 3 - for attempt in range(max_retries): - try: - print(f"Uploading train dataset to S3, attempt {attempt+1}/{max_retries}") - s3_client.upload_file("./data/train/dataset.json", bucket_name, f"{input_path}/train/dataset.json") - print(f"Uploading test dataset to S3, attempt {attempt+1}/{max_retries}") - s3_client.upload_file("./data/test/dataset.json", bucket_name, f"{input_path}/test/dataset.json") - print("S3 upload successful") - break - except Exception as e: - error_msg = f"Error in S3 upload (attempt {attempt+1}/{max_retries}): {str(e)}" - print(error_msg) - if attempt == max_retries - 1: # Last attempt failed - raise RuntimeError(f"Failed to upload datasets to S3 after {max_retries} attempts: {str(e)}") + mlflow.log_param("train_size", len(train_dataset)) + mlflow.log_param("test_size", len(test_dataset)) + mlflow.log_param("dataset_sample_size", num_samples) # Log that we're using a subset of 100 samples + # save train_dataset to s3 using our SageMaker session + if default_prefix: + input_path = f'{default_prefix}/datasets/llm-fine-tuning-modeltrainer-sft' + else: + input_path = f'datasets/llm-fine-tuning-modeltrainer-sft' + + # Save datasets to s3 + # We will fine tune only with 20 records due to limited compute resource for the workshop + train_dataset.to_json("./data/train/dataset.json", orient="records") + test_dataset.to_json("./data/test/dataset.json", orient="records") + + s3_client.upload_file("./data/train/dataset.json", bucket_name, f"{input_path}/train/dataset.json") + train_dataset_s3_path = f"s3://{bucket_name}/{input_path}/train/dataset.json" + s3_client.upload_file("./data/test/dataset.json", bucket_name, f"{input_path}/test/dataset.json") + test_dataset_s3_path = f"s3://{bucket_name}/{input_path}/test/dataset.json" + + shutil.rmtree("./data") + + print(f"Training data uploaded to:") + print(train_dataset_s3_path) + print(test_dataset_s3_path) + + mlflow.log_param("train_data_path", train_dataset_s3_path) + mlflow.log_param("test_dataset_path", test_dataset_s3_path) print(f"Datasets uploaded to:") - print(train_data_path) - print(test_dataset_path) - - # Log a sample of the dataset as an artifact if MLflow is enabled - try: - with open("./data/sample.txt", "w") as f: - f.write(sample_text) - mlflow.log_artifact("./data/sample.txt", "dataset_samples") - except Exception as e: - print(f"Error logging sample as artifact: {str(e)}") - - # Clean up - try: - if os.path.exists("./data"): - shutil.rmtree("./data") - except Exception as e: - print(f"Warning: Error cleaning up temporary files: {str(e)}") + print(train_dataset_s3_path) + print(test_dataset_s3_path) except Exception as e: error_msg = f"Critical error in preprocessing: {str(e)}\n{traceback.format_exc()}" print(error_msg) raise RuntimeError(f"Preprocessing failed: {str(e)}") - return run_id, result_train_data_path, result_test_dataset_path \ No newline at end of file + return run_id, train_dataset_s3_path, test_dataset_s3_path \ No newline at end of file From c1b7f79a4a15634ccf55a1361362c50a94336e28 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Fri, 10 Oct 2025 16:18:35 -0400 Subject: [PATCH 11/22] Fixed guardrail id bug --- .../task_05_fmops/05.00_fmops_examples.ipynb | 113 +++++++------- .../05.01_fine-tuning-pipeline.ipynb | 2 +- .../task_05_fmops/steps/pipeline_utils.py | 141 +++++++----------- 3 files changed, 119 insertions(+), 137 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb index 0636d19..ffa5afd 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -350,6 +350,58 @@ "})" ] }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "def create_guardrail(guardrail_client):\n", + " try:\n", + " guardrail = guardrail_client.create_guardrail(\n", + " name=\"ExampleMedicalGuardrail\",\n", + " description='Example of a Guardrail for Medical Use Cases',\n", + " topicPolicyConfig={\n", + " 'topicsConfig': [{\n", + " 'name': 'Block Pharmaceuticals',\n", + " 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.',\n", + " 'type': 'DENY',\n", + " 'inputAction': 'BLOCK',\n", + " 'outputAction': 'BLOCK',\n", + " }]\n", + " },\n", + " sensitiveInformationPolicyConfig={\n", + " 'piiEntitiesConfig': [\n", + " {\n", + " 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER',\n", + " 'action': 'BLOCK',\n", + " 'inputAction': 'BLOCK',\n", + " 'outputAction': 'BLOCK'\n", + " },\n", + " ]\n", + " },\n", + " contextualGroundingPolicyConfig={\n", + " 'filtersConfig': [\n", + " {\n", + " 'type': 'RELEVANCE',\n", + " 'threshold': 0.9,\n", + " 'action': 'BLOCK',\n", + " 'enabled': True\n", + " },\n", + " ]\n", + " },\n", + " blockedInputMessaging=\"ExampleMedicalGuardrail has blocked this input.\",\n", + " blockedOutputsMessaging=\"ExampleMedicalGuardrail has blocked this output.\"\n", + " )\n", + " guardrail_id = guardrail['guardrailId']\n", + " guardrail_version = guardrail['version']\n", + "\n", + " print(f\"Created new guardrail '{guardrail_id}:{guardrail_version}'\")\n", + " return guardrail_id, guardrail_version\n", + " except botocore.exceptions.ClientError as create_error:\n", + " print(f\"Error creating guardrail: {create_error}\")" + ] + }, { "cell_type": "code", "execution_count": null, @@ -361,61 +413,20 @@ "try:\n", " # Try to get the guardrail\n", " response = guardrail_client.list_guardrails()\n", + " guardrail_id = \"\"\n", " for guardrail in response.get('guardrails', []):\n", " if guardrail['name'] == guardrail_name:\n", " guardrail_id = guardrail['id']\n", - " response = guardrail_client.get_guardrail(\n", - " guardrailIdentifier=guardrail_id\n", - " )\n", - " guardrail_version = response[\"version\"]\n", - " print(f\"Found Guardrail {guardrail_id}:{guardrail_version}\")\n", - "except botocore.exceptions.ClientError as e:\n", - " if e.response['Error']['Code'] == 'ResourceNotFoundException':\n", - " # Guardrail doesn't exist, create it\n", - " try:\n", - " guardrail = guardrail_client.create_guardrail(\n", - " name=\"ExampleMedicalGuardrail\",\n", - " description='Example of a Guardrail for Medical Use Cases',\n", - " topicPolicyConfig={\n", - " 'topicsConfig': [{\n", - " 'name': 'Block Pharmaceuticals',\n", - " 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.',\n", - " 'type': 'DENY',\n", - " 'inputAction': 'BLOCK',\n", - " 'outputAction': 'BLOCK',\n", - " }] \n", - " },\n", - " sensitiveInformationPolicyConfig={\n", - " 'piiEntitiesConfig': [\n", - " {\n", - " 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER',\n", - " 'action': 'BLOCK',\n", - " 'inputAction': 'BLOCK',\n", - " 'outputAction': 'BLOCK'\n", - " },\n", - " ]\n", - " },\n", - " contextualGroundingPolicyConfig={\n", - " 'filtersConfig': [\n", - " {\n", - " 'type': 'RELEVANCE',\n", - " 'threshold': 0.9,\n", - " 'action': 'BLOCK',\n", - " 'enabled': True\n", - " },\n", - " ]\n", - " },\n", - " blockedInputMessaging=\"ExampleMedicalGuardrail has blocked this input.\",\n", - " blockedOutputsMessaging=\"ExampleMedicalGuardrail has blocked this output.\"\n", - " )\n", - " guardrail_id = guardrail['guardrailId']\n", - " guardrail_version = guardrail['version']\n", - " \n", - " print(f\"Created new guardrail '{guardrail_id}:{guardrail_version}'\")\n", - " except botocore.exceptions.ClientError as create_error:\n", - " print(f\"Error creating guardrail: {create_error}\")\n", + " if guardrail_id != \"\":\n", + " response = guardrail_client.get_guardrail(\n", + " guardrailIdentifier=guardrail_id\n", + " )\n", + " guardrail_version = response[\"version\"]\n", + " print(f\"Found Guardrail {guardrail_id}:{guardrail_version}\")\n", " else:\n", - " print(f\"Error checking guardrail: {e}\")" + " guardrail_id, guardrail_version = create_guardrail(guardrail_client)\n", + "except botocore.exceptions.ClientError as e:\n", + " print(f\"Error checking guardrail: {e}\")" ] }, { diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index a1c57e2..5a5aca9 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -485,7 +485,7 @@ "outputs": [], "source": [ "from steps import pipeline_utils\n", - "guardrail_id, guardrail_version =pipeline_utils.get_or_create_guardrail()" + "guardrail_id, guardrail_version = pipeline_utils.get_or_create_guardrail()" ] }, { diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py index 5c2ffb8..287d1e1 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py @@ -62,39 +62,6 @@ def convert_to_messages(sample, system_prompt=""): return sample - -# Template dataset to add prompt to each sample -# def template_dataset(sample): -# try: -# sample["text"] = PROMPT_TEMPLATE.format(question=sample["Question"], -# complex_cot=sample["Complex_CoT"], -# answer=sample["Response"]) -# return sample -# except KeyError as e: -# print(f"KeyError in template_dataset: {str(e)}") -# # Provide default values for missing fields -# missing_key = str(e).strip("'") -# if missing_key == "Question": -# sample["text"] = PROMPT_TEMPLATE.format( -# question="[Missing question]", -# complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), -# answer=sample.get("Response", "[Missing response]") -# ) -# elif missing_key == "Complex_CoT": -# sample["text"] = PROMPT_TEMPLATE.format( -# question=sample["Question"], -# complex_cot="[Missing CoT]", -# answer=sample.get("Response", "[Missing response]") -# ) -# elif missing_key == "Response": -# sample["text"] = PROMPT_TEMPLATE.format( -# question=sample["Question"], -# complex_cot=sample.get("Complex_CoT", "[Missing CoT]"), -# answer="[Missing response]" -# ) -# return sample - - def invoke_sagemaker_endpoint(payload, endpoint_name): """ Invoke a SageMaker endpoint with the given payload. @@ -123,65 +90,69 @@ def invoke_sagemaker_endpoint(payload, endpoint_name): return None, -1 +def create_guardrail(guardrail_client): + try: + guardrail = guardrail_client.create_guardrail( + name="ExampleMedicalGuardrail", + description='Example of a Guardrail for Medical Use Cases', + topicPolicyConfig={ + 'topicsConfig': [{ + 'name': 'Block Pharmaceuticals', + 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.', + 'type': 'DENY', + 'inputAction': 'BLOCK', + 'outputAction': 'BLOCK', + }] + }, + sensitiveInformationPolicyConfig={ + 'piiEntitiesConfig': [ + { + 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER', + 'action': 'BLOCK', + 'inputAction': 'BLOCK', + 'outputAction': 'BLOCK' + }, + ] + }, + contextualGroundingPolicyConfig={ + 'filtersConfig': [ + { + 'type': 'RELEVANCE', + 'threshold': 0.9, + 'action': 'BLOCK', + 'enabled': True + }, + ] + }, + blockedInputMessaging="ExampleMedicalGuardrail has blocked this input.", + blockedOutputsMessaging="ExampleMedicalGuardrail has blocked this output." + ) + guardrail_id = guardrail['guardrailId'] + guardrail_version = guardrail['version'] + + print(f"Created new guardrail '{guardrail_id}:{guardrail_version}'") + return guardrail_id, guardrail_version + except botocore.exceptions.ClientError as create_error: + print(f"Error creating guardrail: {create_error}") + + def get_or_create_guardrail(): guardrail_client = boto3.client('bedrock') guardrail_name = "ExampleMedicalGuardrail" try: # Try to get the guardrail response = guardrail_client.list_guardrails() + guardrail_id = "" for guardrail in response.get('guardrails', []): if guardrail['name'] == guardrail_name: guardrail_id = guardrail['id'] - response = guardrail_client.get_guardrail( - guardrailIdentifier=guardrail_id - ) - guardrail_version = response["version"] - print(f"Found Guardrail {guardrail_id}:{guardrail_version}") - except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == 'ResourceNotFoundException': - # Guardrail doesn't exist, create it - try: - guardrail = guardrail_client.create_guardrail( - name="ExampleMedicalGuardrail", - description='Example of a Guardrail for Medical Use Cases', - topicPolicyConfig={ - 'topicsConfig': [{ - 'name': 'Block Pharmaceuticals', - 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.', - 'type': 'DENY', - 'inputAction': 'BLOCK', - 'outputAction': 'BLOCK', - }] - }, - sensitiveInformationPolicyConfig={ - 'piiEntitiesConfig': [ - { - 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER', - 'action': 'BLOCK', - 'inputAction': 'BLOCK', - 'outputAction': 'BLOCK' - }, - ] - }, - contextualGroundingPolicyConfig={ - 'filtersConfig': [ - { - 'type': 'RELEVANCE', - 'threshold': 0.9, - 'action': 'BLOCK', - 'enabled': True - }, - ] - }, - blockedInputMessaging="ExampleMedicalGuardrail has blocked this input.", - blockedOutputsMessaging="ExampleMedicalGuardrail has blocked this output." - ) - guardrail_id = guardrail['guardrailId'] - guardrail_version = guardrail['version'] - - print(f"Created new guardrail '{guardrail_id}:{guardrail_version}'") - except botocore.exceptions.ClientError as create_error: - print(f"Error creating guardrail: {create_error}") + if guardrail_id != "": + response = guardrail_client.get_guardrail( + guardrailIdentifier=guardrail_id + ) + guardrail_version = response["version"] + print(f"Found Guardrail {guardrail_id}:{guardrail_version}") else: - print(f"Error checking guardrail: {e}") - return guardrail_id, guardrail_version \ No newline at end of file + return create_guardrail(guardrail_client) + except botocore.exceptions.ClientError as e: + print(f"Error checking guardrail: {e}") \ No newline at end of file From f4c06ddedfce9cffb1fffcad1de6bce31805c5d3 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Fri, 10 Oct 2025 21:33:08 -0400 Subject: [PATCH 12/22] Moved 05.00 to messages API --- .../task_05_fmops/05.00_fmops_examples.ipynb | 701 ++++++++++++++++-- 1 file changed, 638 insertions(+), 63 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb index ffa5afd..c60ca1c 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -31,18 +31,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33m DEPRECATION: Building 'xtarfile' using the legacy setup.py bdist_wheel mechanism, which will be removed in a future version. pip 25.3 will enforce this behaviour change. A possible replacement is to use the standardized build interface by setting the `--use-pep517` option, (possibly combined with `--no-build-isolation`), or adding a `pyproject.toml` file to the source tree of 'xtarfile'. Discussion can be found at https://github.com/pypa/pip/issues/6334\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33m DEPRECATION: Building 'rouge-score' using the legacy setup.py bdist_wheel mechanism, which will be removed in a future version. pip 25.3 will enforce this behaviour change. A possible replacement is to use the standardized build interface by setting the `--use-pep517` option, (possibly combined with `--no-build-isolation`), or adding a `pyproject.toml` file to the source tree of 'rouge-score'. Discussion can be found at https://github.com/pypa/pip/issues/6334\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "autogluon-multimodal 1.4.0 requires nvidia-ml-py3<8.0,>=7.352.0, which is not installed.\n", + "jupyter-ai 2.31.6 requires faiss-cpu!=1.8.0.post0,<2.0.0,>=1.8.0, which is not installed.\n", + "aiobotocore 2.21.1 requires botocore<1.37.2,>=1.37.0, but you have botocore 1.40.49 which is incompatible.\n", + "autogluon-common 1.4.0 requires psutil<7.1.0,>=5.7.3, but you have psutil 7.1.0 which is incompatible.\n", + "autogluon-multimodal 1.4.0 requires transformers[sentencepiece]<4.50,>=4.38.0, but you have transformers 4.52.2 which is incompatible.\n", + "autogluon-timeseries 1.4.0 requires transformers[sentencepiece]<4.50,>=4.38.0, but you have transformers 4.52.2 which is incompatible.\n", + "jupyter-scheduler 2.11.0 requires psutil~=5.9, but you have psutil 7.1.0 which is incompatible.\n", + "s3fs 2024.12.0 requires fsspec==2024.12.0.*, but you have fsspec 2024.9.0 which is incompatible.\n", + "sagemaker-studio-analytics-extension 0.2.0 requires sparkmagic==0.22.0, but you have sparkmagic 0.21.0 which is incompatible.\n", + "sparkmagic 0.21.0 requires pandas<2.0.0,>=0.17.1, but you have pandas 2.3.1 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install -r ./scripts/requirements.txt --upgrade --quiet" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T19:55:10.173073Z", + "iopub.status.busy": "2025-10-10T19:55:10.172747Z", + "iopub.status.idle": "2025-10-10T19:55:10.184825Z", + "shell.execute_reply": "2025-10-10T19:55:10.183969Z", + "shell.execute_reply.started": "2025-10-10T19:55:10.173039Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'status': 'ok', 'restart': True}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from IPython import get_ipython\n", "get_ipython().kernel.do_shutdown(True)" @@ -59,9 +99,26 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:53:24.647808Z", + "iopub.status.busy": "2025-10-10T20:53:24.647536Z", + "iopub.status.idle": "2025-10-10T20:53:29.861097Z", + "shell.execute_reply": "2025-10-10T20:53:29.860290Z", + "shell.execute_reply.started": "2025-10-10T20:53:24.647786Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", + "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n" + ] + } + ], "source": [ "import os\n", "import json\n", @@ -97,8 +154,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:53:29.862899Z", + "iopub.status.busy": "2025-10-10T20:53:29.862544Z", + "iopub.status.idle": "2025-10-10T20:53:30.296894Z", + "shell.execute_reply": "2025-10-10T20:53:30.296057Z", + "shell.execute_reply.started": "2025-10-10T20:53:29.862867Z" + } + }, "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", @@ -120,9 +185,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 3, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:53:30.298295Z", + "iopub.status.busy": "2025-10-10T20:53:30.297785Z", + "iopub.status.idle": "2025-10-10T20:53:30.632749Z", + "shell.execute_reply": "2025-10-10T20:53:30.631987Z", + "shell.execute_reply.started": "2025-10-10T20:53:30.298262Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sagemaker-us-east-1-730335479664\n" + ] + } + ], "source": [ "bucket_name = sagemaker_session.default_bucket()\n", "print(bucket_name)\n", @@ -135,8 +216,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 4, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:53:30.634704Z", + "iopub.status.busy": "2025-10-10T20:53:30.634298Z", + "iopub.status.idle": "2025-10-10T20:53:30.638769Z", + "shell.execute_reply": "2025-10-10T20:53:30.637893Z", + "shell.execute_reply.started": "2025-10-10T20:53:30.634656Z" + } + }, "outputs": [], "source": [ "model_id = \"Qwen/Qwen3-4B-Instruct-2507\"\n", @@ -167,9 +256,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 5, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:53:30.640263Z", + "iopub.status.busy": "2025-10-10T20:53:30.639877Z", + "iopub.status.idle": "2025-10-10T20:53:30.910695Z", + "shell.execute_reply": "2025-10-10T20:53:30.909901Z", + "shell.execute_reply.started": "2025-10-10T20:53:30.640232Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MLflow Tracking Server ARN: arn:aws:sagemaker:us-east-1:730335479664:mlflow-tracking-server/genai-mlflow-tracker\n" + ] + } + ], "source": [ "mlflow_tracking_server_arn = \"\"\n", "\n", @@ -205,9 +310,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:53:32.689130Z", + "iopub.status.busy": "2025-10-10T20:53:32.688853Z", + "iopub.status.idle": "2025-10-10T20:53:32.692724Z", + "shell.execute_reply": "2025-10-10T20:53:32.692073Z", + "shell.execute_reply.started": "2025-10-10T20:53:32.689109Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "using image to host: 763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.33.0-lmi15.0.0-cu128\n" + ] + } + ], "source": [ "inference_image_uri = f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.33.0-lmi15.0.0-cu128\"\n", "print(f\"using image to host: {inference_image_uri}\")" @@ -223,8 +344,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 7, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:53:34.539275Z", + "iopub.status.busy": "2025-10-10T20:53:34.538973Z", + "iopub.status.idle": "2025-10-10T20:53:34.874576Z", + "shell.execute_reply": "2025-10-10T20:53:34.873886Z", + "shell.execute_reply.started": "2025-10-10T20:53:34.539253Z" + } + }, "outputs": [], "source": [ "model_config = {\n", @@ -256,9 +385,28 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 8, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:53:36.205780Z", + "iopub.status.busy": "2025-10-10T20:53:36.205399Z", + "iopub.status.idle": "2025-10-10T20:53:36.438676Z", + "shell.execute_reply": "2025-10-10T20:53:36.437957Z", + "shell.execute_reply.started": "2025-10-10T20:53:36.205757Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Initialize MLFlow tracking data...\n", "mlflow.set_tracking_uri(mlflow_tracking_server_arn)\n", @@ -267,9 +415,26 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 9, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T19:55:38.964058Z", + "iopub.status.busy": "2025-10-10T19:55:38.963706Z", + "iopub.status.idle": "2025-10-10T20:06:15.512283Z", + "shell.execute_reply": "2025-10-10T20:06:15.511475Z", + "shell.execute_reply.started": "2025-10-10T19:55:38.964038Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--------------------!🏃 View run example_model_deployment at: https://us-east-1.experiments.sagemaker.aws/#/experiments/0/runs/c313d8d8fed44b2baf09077a261e08f8\n", + "🧪 View experiment at: https://us-east-1.experiments.sagemaker.aws/#/experiments/0\n" + ] + } + ], "source": [ "with mlflow.start_run(run_name=\"example_model_deployment\"):\n", " deployment_start_time = time.time()\n", @@ -326,9 +491,36 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 11, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:55:05.237003Z", + "iopub.status.busy": "2025-10-10T20:55:05.236735Z", + "iopub.status.idle": "2025-10-10T20:55:06.573888Z", + "shell.execute_reply": "2025-10-10T20:55:06.572904Z", + "shell.execute_reply.started": "2025-10-10T20:55:05.236983Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'chatcmpl-140240167846544',\n", + " 'object': 'chat.completion',\n", + " 'created': 1760129706,\n", + " 'choices': [{'index': 0,\n", + " 'message': {'role': 'assistant',\n", + " 'content': \"Hi! I'm here to help with all sorts of things — whether it's answering questions, solving problems, writing content, coding, brainstorming ideas, or just having a chat. What would you like to work on today? 😊\"},\n", + " 'logprobs': None,\n", + " 'finish_reason': 'eos_token'}],\n", + " 'usage': {'prompt_tokens': 17, 'completion_tokens': 49, 'total_tokens': 66}}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from sagemaker.predictor import Predictor\n", "from sagemaker.serializers import JSONSerializer\n", @@ -339,8 +531,11 @@ " serializer=JSONSerializer(),\n", " deserializer=JSONDeserializer()\n", ")\n", + "\n", "predictor.predict({\n", - " \"inputs\": \"Hi, what can you help me with?\",\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"Hi, what can you help me with?\"}\n", + " ],\n", " \"parameters\": {\n", " \"max_new_tokens\": 512,\n", " \"top_p\": 0.9,\n", @@ -351,12 +546,21 @@ ] }, { - "metadata": {}, "cell_type": "code", + "execution_count": 12, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:55:31.568803Z", + "iopub.status.busy": "2025-10-10T20:55:31.568486Z", + "iopub.status.idle": "2025-10-10T20:55:31.574660Z", + "shell.execute_reply": "2025-10-10T20:55:31.573767Z", + "shell.execute_reply.started": "2025-10-10T20:55:31.568780Z" + } + }, "outputs": [], - "execution_count": null, "source": [ - "def create_guardrail(guardrail_client):\n", + "def create_guardrail():\n", + " # Guardrail doesn't exist, create it\n", " try:\n", " guardrail = guardrail_client.create_guardrail(\n", " name=\"ExampleMedicalGuardrail\",\n", @@ -368,7 +572,7 @@ " 'type': 'DENY',\n", " 'inputAction': 'BLOCK',\n", " 'outputAction': 'BLOCK',\n", - " }]\n", + " }] \n", " },\n", " sensitiveInformationPolicyConfig={\n", " 'piiEntitiesConfig': [\n", @@ -395,7 +599,7 @@ " )\n", " guardrail_id = guardrail['guardrailId']\n", " guardrail_version = guardrail['version']\n", - "\n", + " \n", " print(f\"Created new guardrail '{guardrail_id}:{guardrail_version}'\")\n", " return guardrail_id, guardrail_version\n", " except botocore.exceptions.ClientError as create_error:\n", @@ -404,9 +608,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 13, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:55:32.607222Z", + "iopub.status.busy": "2025-10-10T20:55:32.606957Z", + "iopub.status.idle": "2025-10-10T20:55:32.982484Z", + "shell.execute_reply": "2025-10-10T20:55:32.981632Z", + "shell.execute_reply.started": "2025-10-10T20:55:32.607202Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found Guardrail j6odpern68o8:DRAFT\n" + ] + } + ], "source": [ "guardrail_client = boto3.client('bedrock')\n", "guardrail_name = \"ExampleMedicalGuardrail\"\n", @@ -424,15 +644,23 @@ " guardrail_version = response[\"version\"]\n", " print(f\"Found Guardrail {guardrail_id}:{guardrail_version}\")\n", " else:\n", - " guardrail_id, guardrail_version = create_guardrail(guardrail_client)\n", + " guardrail_id, guardrail_version = create_guardrail()\n", "except botocore.exceptions.ClientError as e:\n", " print(f\"Error checking guardrail: {e}\")" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 14, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:55:33.873857Z", + "iopub.status.busy": "2025-10-10T20:55:33.873553Z", + "iopub.status.idle": "2025-10-10T20:55:33.886372Z", + "shell.execute_reply": "2025-10-10T20:55:33.885629Z", + "shell.execute_reply.started": "2025-10-10T20:55:33.873835Z" + } + }, "outputs": [], "source": [ "bedrock_runtime = boto3.client('bedrock-runtime')" @@ -452,17 +680,34 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 46, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T21:43:09.389634Z", + "iopub.status.busy": "2025-10-10T21:43:09.389179Z", + "iopub.status.idle": "2025-10-10T21:43:09.394028Z", + "shell.execute_reply": "2025-10-10T21:43:09.393028Z", + "shell.execute_reply.started": "2025-10-10T21:43:09.389605Z" + } + }, "outputs": [], "source": [ - "judge_llm = \"bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0\"" + "# judge_llm = \"bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0\"\n", + "judge_llm = \"bedrock:/anthropic.claude-3-haiku-20240307-v1:0\"" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 47, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T21:43:10.053568Z", + "iopub.status.busy": "2025-10-10T21:43:10.051173Z", + "iopub.status.idle": "2025-10-10T21:43:10.071707Z", + "shell.execute_reply": "2025-10-10T21:43:10.065583Z", + "shell.execute_reply.started": "2025-10-10T21:43:10.053515Z" + } + }, "outputs": [], "source": [ "from mlflow.entities import SpanType\n", @@ -476,13 +721,13 @@ ")\n", "def invoke_sagemaker_endpoint(payload):\n", "\n", - " print(payload[\"inputs\"])\n", + " print(payload)\n", "\n", " guardrail_response_input = bedrock_runtime.apply_guardrail(\n", " guardrailIdentifier=guardrail_id,\n", " guardrailVersion=guardrail_version,\n", " source='INPUT',\n", - " content=[{'text': {'text': payload[\"inputs\"]}}]\n", + " content=[{'text': {'text': payload[\"messages\"][0][\"content\"]}}]\n", " )\n", " guardrailResult = guardrail_response_input[\"action\"]\n", "\n", @@ -510,8 +755,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 48, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T21:43:10.860579Z", + "iopub.status.busy": "2025-10-10T21:43:10.860181Z", + "iopub.status.idle": "2025-10-10T21:43:10.872746Z", + "shell.execute_reply": "2025-10-10T21:43:10.871910Z", + "shell.execute_reply.started": "2025-10-10T21:43:10.860548Z" + } + }, "outputs": [], "source": [ "from mlflow.metrics.genai import EvaluationExample, make_genai_metric\n", @@ -677,8 +930,16 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 49, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T21:43:12.247386Z", + "iopub.status.busy": "2025-10-10T21:43:12.246916Z", + "iopub.status.idle": "2025-10-10T21:43:12.263324Z", + "shell.execute_reply": "2025-10-10T21:43:12.262374Z", + "shell.execute_reply.started": "2025-10-10T21:43:12.247358Z" + } + }, "outputs": [], "source": [ "def evaluate_model_qualitatively(model_config, dataset):\n", @@ -705,8 +966,11 @@ " question = example[\"Question\"]\n", " reference = \"\\n\".join([example[\"Complex_CoT\"], example[\"Response\"]])\n", " \n", + " \n", " payload = {\n", - " \"inputs\": question,\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": question}\n", + " ],\n", " \"parameters\": {\n", " \"max_new_tokens\": 512,\n", " \"top_p\": 0.9,\n", @@ -831,9 +1095,297 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 50, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T21:43:13.082392Z", + "iopub.status.busy": "2025-10-10T21:43:13.081667Z", + "iopub.status.idle": "2025-10-10T21:44:59.036247Z", + "shell.execute_reply": "2025-10-10T21:44:59.035451Z", + "shell.execute_reply.started": "2025-10-10T21:43:13.082348Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded medical-o1-reasoning dataset with 10 samples for qualitative evaluation\n", + "\n", + "Performing qualitative evaluation for model: Example-Qwen3-4B-Instruct-2507-sft-djl on endpoint: Example-Qwen3-4B-Instruct-2507-sft-djl\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Generating responses for evaluation: 0%| | 0/10 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "from datasets import load_dataset\n", "from botocore.config import Config\n", @@ -928,9 +1480,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 27, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-10T20:13:45.085755Z", + "iopub.status.busy": "2025-10-10T20:13:45.085308Z", + "iopub.status.idle": "2025-10-10T20:13:45.092345Z", + "shell.execute_reply": "2025-10-10T20:13:45.091517Z", + "shell.execute_reply.started": "2025-10-10T20:13:45.085729Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'Question': 'A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?', 'Complex_CoT': \"Okay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem.\\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal.\\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\", 'Response': 'Cystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.', 'messages': [{'role': 'system', 'content': 'You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \\nBelow is an instruction that describes a task, paired with an input that provides further context. \\nWrite a response that appropriately completes the request.\\nBefore answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.'}, {'role': 'user', 'content': 'A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?'}, {'role': 'assistant', 'content': \"Okay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem.\\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal.\\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\\n\\nCystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.\"}]}\n" + ] + } + ], "source": [ "FINE_TUNING_DATA_SAMPLE = {\n", " \"Question\": \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\", \n", @@ -978,7 +1546,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.status.busy": "2025-10-10T20:06:22.809542Z", + "iopub.status.idle": "2025-10-10T20:06:22.809946Z", + "shell.execute_reply": "2025-10-10T20:06:22.809727Z", + "shell.execute_reply.started": "2025-10-10T20:06:22.809712Z" + } + }, "outputs": [], "source": [ "def delete_endpoint_with_retry(endpoint_name, max_retries=3, wait_seconds=10):\n", From 47e04ffd75fc7fa9324af25707fd6f6b8b358bed Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Tue, 14 Oct 2025 10:18:32 -0400 Subject: [PATCH 13/22] Finished migrating to Qwen and Messages API --- .../05.01_fine-tuning-pipeline.ipynb | 11 +- .../task_05_fmops/steps/pipeline_utils.py | 125 ++++++++---------- .../steps/qualitative_eval_step.py | 75 ++++++----- .../steps/quantitative_eval_step.py | 63 ++++----- 4 files changed, 132 insertions(+), 142 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index 5a5aca9..aab6c8b 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -485,7 +485,7 @@ "outputs": [], "source": [ "from steps import pipeline_utils\n", - "guardrail_id, guardrail_version = pipeline_utils.get_or_create_guardrail()" + "guardrail_id, guardrail_version =pipeline_utils.get_or_create_guardrail()" ] }, { @@ -523,12 +523,10 @@ ")\n", "run_id=training_step[0]\n", "model_artifacts_s3_path=training_step[2]\n", - "# output_path=training_step[3]\n", "\n", "deploy_step = deploy_step.deploy(\n", " tracking_server_arn=mlflow_tracking_server_arn,\n", " model_artifacts_s3_path=model_artifacts_s3_path,\n", - " # output_path=output_path,\n", " model_id=model_s3_destination,\n", " experiment_name=pipeline_name,\n", " run_id=run_id,\n", @@ -742,6 +740,13 @@ " print(f\"Error during endpoint cleanup: {str(e)}\")\n", " print(\"You may need to manually delete the endpoint from the SageMaker console\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py index 287d1e1..3099f8d 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py @@ -8,22 +8,6 @@ PIPELINE_INSTANCE_TYPE = "ml.m5.xlarge" -# PROMPT_TEMPLATE = f""" -# <|begin_of_text|> -# <|start_header_id|>system<|end_header_id|> -# You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. -# Below is an instruction that describes a task, paired with an input that provides further context. -# Write a response that appropriately completes the request. -# Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. -# <|eot_id|><|start_header_id|>user<|end_header_id|> -# {{question}}<|eot_id|> -# <|start_header_id|>assistant<|end_header_id|> -# {{complex_cot}} - -# {{answer}} -# <|eot_id|> -# """ - SYSTEM_PROMPT = """You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. @@ -62,6 +46,7 @@ def convert_to_messages(sample, system_prompt=""): return sample + def invoke_sagemaker_endpoint(payload, endpoint_name): """ Invoke a SageMaker endpoint with the given payload. @@ -90,69 +75,65 @@ def invoke_sagemaker_endpoint(payload, endpoint_name): return None, -1 -def create_guardrail(guardrail_client): - try: - guardrail = guardrail_client.create_guardrail( - name="ExampleMedicalGuardrail", - description='Example of a Guardrail for Medical Use Cases', - topicPolicyConfig={ - 'topicsConfig': [{ - 'name': 'Block Pharmaceuticals', - 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.', - 'type': 'DENY', - 'inputAction': 'BLOCK', - 'outputAction': 'BLOCK', - }] - }, - sensitiveInformationPolicyConfig={ - 'piiEntitiesConfig': [ - { - 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER', - 'action': 'BLOCK', - 'inputAction': 'BLOCK', - 'outputAction': 'BLOCK' - }, - ] - }, - contextualGroundingPolicyConfig={ - 'filtersConfig': [ - { - 'type': 'RELEVANCE', - 'threshold': 0.9, - 'action': 'BLOCK', - 'enabled': True - }, - ] - }, - blockedInputMessaging="ExampleMedicalGuardrail has blocked this input.", - blockedOutputsMessaging="ExampleMedicalGuardrail has blocked this output." - ) - guardrail_id = guardrail['guardrailId'] - guardrail_version = guardrail['version'] - - print(f"Created new guardrail '{guardrail_id}:{guardrail_version}'") - return guardrail_id, guardrail_version - except botocore.exceptions.ClientError as create_error: - print(f"Error creating guardrail: {create_error}") - - def get_or_create_guardrail(): guardrail_client = boto3.client('bedrock') guardrail_name = "ExampleMedicalGuardrail" try: # Try to get the guardrail response = guardrail_client.list_guardrails() - guardrail_id = "" for guardrail in response.get('guardrails', []): if guardrail['name'] == guardrail_name: guardrail_id = guardrail['id'] - if guardrail_id != "": - response = guardrail_client.get_guardrail( - guardrailIdentifier=guardrail_id - ) - guardrail_version = response["version"] - print(f"Found Guardrail {guardrail_id}:{guardrail_version}") - else: - return create_guardrail(guardrail_client) + response = guardrail_client.get_guardrail( + guardrailIdentifier=guardrail_id + ) + guardrail_version = response["version"] + print(f"Found Guardrail {guardrail_id}:{guardrail_version}") except botocore.exceptions.ClientError as e: - print(f"Error checking guardrail: {e}") \ No newline at end of file + if e.response['Error']['Code'] == 'ResourceNotFoundException': + # Guardrail doesn't exist, create it + try: + guardrail = guardrail_client.create_guardrail( + name="ExampleMedicalGuardrail", + description='Example of a Guardrail for Medical Use Cases', + topicPolicyConfig={ + 'topicsConfig': [{ + 'name': 'Block Pharmaceuticals', + 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.', + 'type': 'DENY', + 'inputAction': 'BLOCK', + 'outputAction': 'BLOCK', + }] + }, + sensitiveInformationPolicyConfig={ + 'piiEntitiesConfig': [ + { + 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER', + 'action': 'BLOCK', + 'inputAction': 'BLOCK', + 'outputAction': 'BLOCK' + }, + ] + }, + contextualGroundingPolicyConfig={ + 'filtersConfig': [ + { + 'type': 'RELEVANCE', + 'threshold': 0.9, + 'action': 'BLOCK', + 'enabled': True + }, + ] + }, + blockedInputMessaging="ExampleMedicalGuardrail has blocked this input.", + blockedOutputsMessaging="ExampleMedicalGuardrail has blocked this output." + ) + guardrail_id = guardrail['guardrailId'] + guardrail_version = guardrail['version'] + + print(f"Created new guardrail '{guardrail_id}:{guardrail_version}'") + except botocore.exceptions.ClientError as create_error: + print(f"Error creating guardrail: {create_error}") + else: + print(f"Error checking guardrail: {e}") + return guardrail_id, guardrail_version \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py index cc8cea7..07d509b 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py @@ -4,6 +4,8 @@ from sagemaker.workflow.function_step import step from .pipeline_utils import PIPELINE_INSTANCE_TYPE +from .pipeline_utils import SYSTEM_PROMPT +from .pipeline_utils import convert_to_messages @step( @@ -54,7 +56,7 @@ def invoke_sagemaker_endpoint(payload, endpoint_name): guardrailIdentifier=guardrail_id, guardrailVersion=guardrail_version, source='INPUT', - content=[{'text': {'text': payload["inputs"]}}] + content=[{'text': {'text': payload["messages"][0]["content"]}}] ) guardrailResult = guardrail_response_input["action"] @@ -64,15 +66,29 @@ def invoke_sagemaker_endpoint(payload, endpoint_name): try: start_time = time.time() - response = sm_client.invoke_endpoint( - EndpointName=endpoint_name, - ContentType='application/json', - Body=json.dumps(payload) + # response = sm_client.invoke_endpoint( + # EndpointName=endpoint_name, + # ContentType='application/json', + # Body=json.dumps(payload) + # ) + # inference_time = time.time() - start_time + + # response_body = response['Body'].read().decode('utf-8') + # return json.loads(response_body), inference_time + + from sagemaker.predictor import Predictor + from sagemaker.serializers import JSONSerializer + from sagemaker.deserializers import JSONDeserializer + + predictor = Predictor( + endpoint_name=f"{endpoint_name}", + serializer=JSONSerializer(), + deserializer=JSONDeserializer() ) - inference_time = time.time() - start_time - response_body = response['Body'].read().decode('utf-8') - return json.loads(response_body), inference_time + response = predictor.predict(payload)['choices'][0]['message']['content'] + inference_time = time.time() - start_time + return response, inference_time except Exception as e: print(f"Error invoking endpoint {endpoint_name}: {str(e)}") return None, -1 @@ -295,29 +311,16 @@ def evaluate_model_qualitatively(model_config, dataset): question = example["Question"] reference = "\n".join([example["Complex_CoT"], example["Response"]]) - # Prepare the prompt for the model - prompt = f""" - <|begin_of_text|> - <|start_header_id|>system<|end_header_id|> - You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. - Below is an instruction that describes a task, paired with an input that provides further context. - Write a response that appropriately completes the request. - Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. - <|eot_id|><|start_header_id|>user<|end_header_id|> - {question}<|eot_id|> - <|start_header_id|>assistant<|end_header_id|>""" - - # Payload for SageMaker endpoint - payload = { - "inputs": prompt, - "parameters": { - "max_new_tokens": 512, - "top_p": 0.9, - "temperature": 0.6, - "return_full_text": False - } + payload = {} + messages_prompt = convert_to_messages(example, SYSTEM_PROMPT) + payload["messages"] = messages_prompt["messages"] + payload["parameters"] = { + "max_new_tokens": 512, + "top_p": 0.9, + "temperature": 0.6, + "return_full_text": False } - + # Call the model endpoint try: response, inference_time = invoke_sagemaker_endpoint(payload, endpoint_name) @@ -325,14 +328,14 @@ def evaluate_model_qualitatively(model_config, dataset): if response is None: prediction = "Error generating response." failed_generations += 1 - elif isinstance(response, list): - prediction = response[0].get('generated_text', '').strip() - elif isinstance(response, dict): - prediction = response.get('generated_text', '').strip() + # elif isinstance(response, list): + # prediction = response[0].get('generated_text', '').strip() + # elif isinstance(response, dict): + # prediction = response.get('generated_text', '').strip() else: prediction = str(response).strip() - prediction = prediction.split("<|eot_id|>")[0] if "<|eot_id|>" in prediction else prediction + # prediction = prediction.split("<|eot_id|>")[0] if "<|eot_id|>" in prediction else prediction inference_times.append(inference_time) except Exception as e: @@ -478,7 +481,7 @@ def evaluate_model_qualitatively(model_config, dataset): mlflow.log_param("qualitative_evaluation_endpoint", endpoint_name) mlflow.log_param("qualitative_evaluation_num_samples", num_samples) mlflow.log_param("qualitative_evaluation_timestamp", datetime.now().isoformat()) - mlflow.log_param("llm_judge_model", "bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0") + mlflow.log_param("llm_judge_model", "bedrock:/anthropic.claude-3-haiku-20240307-v1:0") # Load the test dataset try: diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py index 05df1b5..c1cddf3 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py @@ -4,6 +4,8 @@ from sagemaker.workflow.function_step import step from .pipeline_utils import PIPELINE_INSTANCE_TYPE +from .pipeline_utils import SYSTEM_PROMPT +from .pipeline_utils import convert_to_messages @step( @@ -61,7 +63,7 @@ def invoke_sagemaker_endpoint(payload, endpoint_name): guardrailIdentifier=guardrail_id, guardrailVersion=guardrail_version, source='INPUT', - content=[{'text': {'text': payload["inputs"]}}] + content=[{'text': {'text': payload["messages"][0]["content"]}}] ) guardrailResult = guardrail_response_input["action"] @@ -71,15 +73,29 @@ def invoke_sagemaker_endpoint(payload, endpoint_name): try: start_time = time.time() - response = sm_client.invoke_endpoint( - EndpointName=endpoint_name, - ContentType='application/json', - Body=json.dumps(payload) + # response = sm_client.invoke_endpoint( + # EndpointName=endpoint_name, + # ContentType='application/json', + # Body=json.dumps(payload) + # ) + # inference_time = time.time() - start_time + + # response_body = response['Body'].read().decode('utf-8') + # return json.loads(response_body), inference_time + + from sagemaker.predictor import Predictor + from sagemaker.serializers import JSONSerializer + from sagemaker.deserializers import JSONDeserializer + + predictor = Predictor( + endpoint_name=f"{endpoint_name}", + serializer=JSONSerializer(), + deserializer=JSONDeserializer() ) - inference_time = time.time() - start_time - response_body = response['Body'].read().decode('utf-8') - return json.loads(response_body), inference_time + response = predictor.predict(payload)['choices'][0]['message']['content'] + inference_time = time.time() - start_time + return response, inference_time except Exception as e: print(f"Error invoking endpoint {endpoint_name}: {str(e)}") return None, -1 @@ -167,29 +183,14 @@ def generate_summaries_with_model(endpoint_name, dataset): failed_generations = 0 for example in tqdm(dataset, desc="Generating Responses"): - question = example["Question"] - - # Prepare the prompt for the model - prompt = f""" - <|begin_of_text|> - <|start_header_id|>system<|end_header_id|> - You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. - Below is an instruction that describes a task, paired with an input that provides further context. - Write a response that appropriately completes the request. - Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response. - <|eot_id|><|start_header_id|>user<|end_header_id|> - {question}<|eot_id|> - <|start_header_id|>assistant<|end_header_id|>""" - - # Payload for SageMaker endpoint - payload = { - "inputs": prompt, - "parameters": { - "max_new_tokens": 512, - "top_p": 0.9, - "temperature": 0.6, - "return_full_text": False - } + payload = {} + messages_prompt = convert_to_messages(example, SYSTEM_PROMPT) + payload["messages"] = messages_prompt["messages"] + payload["parameters"] = { + "max_new_tokens": 512, + "top_p": 0.9, + "temperature": 0.6, + "return_full_text": False } # Call the model endpoint From a1668e246bf88e5481eb3c1f9214af6bdbf109bd Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Tue, 14 Oct 2025 11:32:56 -0400 Subject: [PATCH 14/22] Removed cell output from 05.00, increased timeout on health check for deploy step --- .../task_05_fmops/05.00_fmops_examples.ipynb | 675 ++---------------- .../task_05_fmops/steps/deploy_step.py | 2 +- 2 files changed, 57 insertions(+), 620 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb index c60ca1c..4d7f249 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -31,58 +31,18 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[33m DEPRECATION: Building 'xtarfile' using the legacy setup.py bdist_wheel mechanism, which will be removed in a future version. pip 25.3 will enforce this behaviour change. A possible replacement is to use the standardized build interface by setting the `--use-pep517` option, (possibly combined with `--no-build-isolation`), or adding a `pyproject.toml` file to the source tree of 'xtarfile'. Discussion can be found at https://github.com/pypa/pip/issues/6334\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33m DEPRECATION: Building 'rouge-score' using the legacy setup.py bdist_wheel mechanism, which will be removed in a future version. pip 25.3 will enforce this behaviour change. A possible replacement is to use the standardized build interface by setting the `--use-pep517` option, (possibly combined with `--no-build-isolation`), or adding a `pyproject.toml` file to the source tree of 'rouge-score'. Discussion can be found at https://github.com/pypa/pip/issues/6334\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "autogluon-multimodal 1.4.0 requires nvidia-ml-py3<8.0,>=7.352.0, which is not installed.\n", - "jupyter-ai 2.31.6 requires faiss-cpu!=1.8.0.post0,<2.0.0,>=1.8.0, which is not installed.\n", - "aiobotocore 2.21.1 requires botocore<1.37.2,>=1.37.0, but you have botocore 1.40.49 which is incompatible.\n", - "autogluon-common 1.4.0 requires psutil<7.1.0,>=5.7.3, but you have psutil 7.1.0 which is incompatible.\n", - "autogluon-multimodal 1.4.0 requires transformers[sentencepiece]<4.50,>=4.38.0, but you have transformers 4.52.2 which is incompatible.\n", - "autogluon-timeseries 1.4.0 requires transformers[sentencepiece]<4.50,>=4.38.0, but you have transformers 4.52.2 which is incompatible.\n", - "jupyter-scheduler 2.11.0 requires psutil~=5.9, but you have psutil 7.1.0 which is incompatible.\n", - "s3fs 2024.12.0 requires fsspec==2024.12.0.*, but you have fsspec 2024.9.0 which is incompatible.\n", - "sagemaker-studio-analytics-extension 0.2.0 requires sparkmagic==0.22.0, but you have sparkmagic 0.21.0 which is incompatible.\n", - "sparkmagic 0.21.0 requires pandas<2.0.0,>=0.17.1, but you have pandas 2.3.1 which is incompatible.\u001b[0m\u001b[31m\n", - "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n" - ] - } - ], + "outputs": [], "source": [ "%pip install -r ./scripts/requirements.txt --upgrade --quiet" ] }, { "cell_type": "code", - "execution_count": 2, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T19:55:10.173073Z", - "iopub.status.busy": "2025-10-10T19:55:10.172747Z", - "iopub.status.idle": "2025-10-10T19:55:10.184825Z", - "shell.execute_reply": "2025-10-10T19:55:10.183969Z", - "shell.execute_reply.started": "2025-10-10T19:55:10.173039Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'status': 'ok', 'restart': True}" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from IPython import get_ipython\n", "get_ipython().kernel.do_shutdown(True)" @@ -99,26 +59,9 @@ }, { "cell_type": "code", - "execution_count": 1, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:53:24.647808Z", - "iopub.status.busy": "2025-10-10T20:53:24.647536Z", - "iopub.status.idle": "2025-10-10T20:53:29.861097Z", - "shell.execute_reply": "2025-10-10T20:53:29.860290Z", - "shell.execute_reply.started": "2025-10-10T20:53:24.647786Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n", - "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "import os\n", "import json\n", @@ -154,16 +97,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:53:29.862899Z", - "iopub.status.busy": "2025-10-10T20:53:29.862544Z", - "iopub.status.idle": "2025-10-10T20:53:30.296894Z", - "shell.execute_reply": "2025-10-10T20:53:30.296057Z", - "shell.execute_reply.started": "2025-10-10T20:53:29.862867Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", @@ -185,25 +120,9 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:53:30.298295Z", - "iopub.status.busy": "2025-10-10T20:53:30.297785Z", - "iopub.status.idle": "2025-10-10T20:53:30.632749Z", - "shell.execute_reply": "2025-10-10T20:53:30.631987Z", - "shell.execute_reply.started": "2025-10-10T20:53:30.298262Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "sagemaker-us-east-1-730335479664\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "bucket_name = sagemaker_session.default_bucket()\n", "print(bucket_name)\n", @@ -216,16 +135,8 @@ }, { "cell_type": "code", - "execution_count": 4, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:53:30.634704Z", - "iopub.status.busy": "2025-10-10T20:53:30.634298Z", - "iopub.status.idle": "2025-10-10T20:53:30.638769Z", - "shell.execute_reply": "2025-10-10T20:53:30.637893Z", - "shell.execute_reply.started": "2025-10-10T20:53:30.634656Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "model_id = \"Qwen/Qwen3-4B-Instruct-2507\"\n", @@ -256,25 +167,9 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:53:30.640263Z", - "iopub.status.busy": "2025-10-10T20:53:30.639877Z", - "iopub.status.idle": "2025-10-10T20:53:30.910695Z", - "shell.execute_reply": "2025-10-10T20:53:30.909901Z", - "shell.execute_reply.started": "2025-10-10T20:53:30.640232Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MLflow Tracking Server ARN: arn:aws:sagemaker:us-east-1:730335479664:mlflow-tracking-server/genai-mlflow-tracker\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "mlflow_tracking_server_arn = \"\"\n", "\n", @@ -310,25 +205,9 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:53:32.689130Z", - "iopub.status.busy": "2025-10-10T20:53:32.688853Z", - "iopub.status.idle": "2025-10-10T20:53:32.692724Z", - "shell.execute_reply": "2025-10-10T20:53:32.692073Z", - "shell.execute_reply.started": "2025-10-10T20:53:32.689109Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "using image to host: 763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.33.0-lmi15.0.0-cu128\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "inference_image_uri = f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.33.0-lmi15.0.0-cu128\"\n", "print(f\"using image to host: {inference_image_uri}\")" @@ -344,16 +223,8 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:53:34.539275Z", - "iopub.status.busy": "2025-10-10T20:53:34.538973Z", - "iopub.status.idle": "2025-10-10T20:53:34.874576Z", - "shell.execute_reply": "2025-10-10T20:53:34.873886Z", - "shell.execute_reply.started": "2025-10-10T20:53:34.539253Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "model_config = {\n", @@ -385,28 +256,9 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:53:36.205780Z", - "iopub.status.busy": "2025-10-10T20:53:36.205399Z", - "iopub.status.idle": "2025-10-10T20:53:36.438676Z", - "shell.execute_reply": "2025-10-10T20:53:36.437957Z", - "shell.execute_reply.started": "2025-10-10T20:53:36.205757Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Initialize MLFlow tracking data...\n", "mlflow.set_tracking_uri(mlflow_tracking_server_arn)\n", @@ -415,26 +267,9 @@ }, { "cell_type": "code", - "execution_count": 9, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T19:55:38.964058Z", - "iopub.status.busy": "2025-10-10T19:55:38.963706Z", - "iopub.status.idle": "2025-10-10T20:06:15.512283Z", - "shell.execute_reply": "2025-10-10T20:06:15.511475Z", - "shell.execute_reply.started": "2025-10-10T19:55:38.964038Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--------------------!🏃 View run example_model_deployment at: https://us-east-1.experiments.sagemaker.aws/#/experiments/0/runs/c313d8d8fed44b2baf09077a261e08f8\n", - "🧪 View experiment at: https://us-east-1.experiments.sagemaker.aws/#/experiments/0\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "with mlflow.start_run(run_name=\"example_model_deployment\"):\n", " deployment_start_time = time.time()\n", @@ -491,36 +326,9 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:55:05.237003Z", - "iopub.status.busy": "2025-10-10T20:55:05.236735Z", - "iopub.status.idle": "2025-10-10T20:55:06.573888Z", - "shell.execute_reply": "2025-10-10T20:55:06.572904Z", - "shell.execute_reply.started": "2025-10-10T20:55:05.236983Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'chatcmpl-140240167846544',\n", - " 'object': 'chat.completion',\n", - " 'created': 1760129706,\n", - " 'choices': [{'index': 0,\n", - " 'message': {'role': 'assistant',\n", - " 'content': \"Hi! I'm here to help with all sorts of things — whether it's answering questions, solving problems, writing content, coding, brainstorming ideas, or just having a chat. What would you like to work on today? 😊\"},\n", - " 'logprobs': None,\n", - " 'finish_reason': 'eos_token'}],\n", - " 'usage': {'prompt_tokens': 17, 'completion_tokens': 49, 'total_tokens': 66}}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from sagemaker.predictor import Predictor\n", "from sagemaker.serializers import JSONSerializer\n", @@ -547,16 +355,8 @@ }, { "cell_type": "code", - "execution_count": 12, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:55:31.568803Z", - "iopub.status.busy": "2025-10-10T20:55:31.568486Z", - "iopub.status.idle": "2025-10-10T20:55:31.574660Z", - "shell.execute_reply": "2025-10-10T20:55:31.573767Z", - "shell.execute_reply.started": "2025-10-10T20:55:31.568780Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "def create_guardrail():\n", @@ -608,25 +408,9 @@ }, { "cell_type": "code", - "execution_count": 13, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:55:32.607222Z", - "iopub.status.busy": "2025-10-10T20:55:32.606957Z", - "iopub.status.idle": "2025-10-10T20:55:32.982484Z", - "shell.execute_reply": "2025-10-10T20:55:32.981632Z", - "shell.execute_reply.started": "2025-10-10T20:55:32.607202Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found Guardrail j6odpern68o8:DRAFT\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "guardrail_client = boto3.client('bedrock')\n", "guardrail_name = \"ExampleMedicalGuardrail\"\n", @@ -651,16 +435,8 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:55:33.873857Z", - "iopub.status.busy": "2025-10-10T20:55:33.873553Z", - "iopub.status.idle": "2025-10-10T20:55:33.886372Z", - "shell.execute_reply": "2025-10-10T20:55:33.885629Z", - "shell.execute_reply.started": "2025-10-10T20:55:33.873835Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "bedrock_runtime = boto3.client('bedrock-runtime')" @@ -680,16 +456,8 @@ }, { "cell_type": "code", - "execution_count": 46, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T21:43:09.389634Z", - "iopub.status.busy": "2025-10-10T21:43:09.389179Z", - "iopub.status.idle": "2025-10-10T21:43:09.394028Z", - "shell.execute_reply": "2025-10-10T21:43:09.393028Z", - "shell.execute_reply.started": "2025-10-10T21:43:09.389605Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# judge_llm = \"bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0\"\n", @@ -698,16 +466,8 @@ }, { "cell_type": "code", - "execution_count": 47, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T21:43:10.053568Z", - "iopub.status.busy": "2025-10-10T21:43:10.051173Z", - "iopub.status.idle": "2025-10-10T21:43:10.071707Z", - "shell.execute_reply": "2025-10-10T21:43:10.065583Z", - "shell.execute_reply.started": "2025-10-10T21:43:10.053515Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "from mlflow.entities import SpanType\n", @@ -755,16 +515,8 @@ }, { "cell_type": "code", - "execution_count": 48, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T21:43:10.860579Z", - "iopub.status.busy": "2025-10-10T21:43:10.860181Z", - "iopub.status.idle": "2025-10-10T21:43:10.872746Z", - "shell.execute_reply": "2025-10-10T21:43:10.871910Z", - "shell.execute_reply.started": "2025-10-10T21:43:10.860548Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "from mlflow.metrics.genai import EvaluationExample, make_genai_metric\n", @@ -930,16 +682,8 @@ }, { "cell_type": "code", - "execution_count": 49, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T21:43:12.247386Z", - "iopub.status.busy": "2025-10-10T21:43:12.246916Z", - "iopub.status.idle": "2025-10-10T21:43:12.263324Z", - "shell.execute_reply": "2025-10-10T21:43:12.262374Z", - "shell.execute_reply.started": "2025-10-10T21:43:12.247358Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "def evaluate_model_qualitatively(model_config, dataset):\n", @@ -1095,297 +839,11 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T21:43:13.082392Z", - "iopub.status.busy": "2025-10-10T21:43:13.081667Z", - "iopub.status.idle": "2025-10-10T21:44:59.036247Z", - "shell.execute_reply": "2025-10-10T21:44:59.035451Z", - "shell.execute_reply.started": "2025-10-10T21:43:13.082348Z" - }, "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded medical-o1-reasoning dataset with 10 samples for qualitative evaluation\n", - "\n", - "Performing qualitative evaluation for model: Example-Qwen3-4B-Instruct-2507-sft-djl on endpoint: Example-Qwen3-4B-Instruct-2507-sft-djl\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Generating responses for evaluation: 0%| | 0/10 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "from datasets import load_dataset\n", "from botocore.config import Config\n", @@ -1480,25 +938,9 @@ }, { "cell_type": "code", - "execution_count": 27, - "metadata": { - "execution": { - "iopub.execute_input": "2025-10-10T20:13:45.085755Z", - "iopub.status.busy": "2025-10-10T20:13:45.085308Z", - "iopub.status.idle": "2025-10-10T20:13:45.092345Z", - "shell.execute_reply": "2025-10-10T20:13:45.091517Z", - "shell.execute_reply.started": "2025-10-10T20:13:45.085729Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'Question': 'A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?', 'Complex_CoT': \"Okay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem.\\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal.\\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\", 'Response': 'Cystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.', 'messages': [{'role': 'system', 'content': 'You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \\nBelow is an instruction that describes a task, paired with an input that provides further context. \\nWrite a response that appropriately completes the request.\\nBefore answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.'}, {'role': 'user', 'content': 'A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?'}, {'role': 'assistant', 'content': \"Okay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem.\\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal.\\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\\n\\nCystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.\"}]}\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "FINE_TUNING_DATA_SAMPLE = {\n", " \"Question\": \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\", \n", @@ -1516,7 +958,9 @@ " messages = [\n", " {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n", " {\"role\": \"user\", \"content\": sample[\"Question\"]},\n", - " {\"role\": \"assistant\", \"content\": f\"{sample[\"Complex_CoT\"]}\\n\\n{sample[\"Response\"]}\"}\n", + " {\"role\": \"assistant\", \n", + " \"content\": \n", + " f\"{sample[\"Complex_CoT\"]}\\n\\n{sample[\"Response\"]}\"}\n", " ]\n", "\n", " sample[\"messages\"] = messages\n", @@ -1546,14 +990,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "execution": { - "iopub.status.busy": "2025-10-10T20:06:22.809542Z", - "iopub.status.idle": "2025-10-10T20:06:22.809946Z", - "shell.execute_reply": "2025-10-10T20:06:22.809727Z", - "shell.execute_reply.started": "2025-10-10T20:06:22.809712Z" - } - }, + "metadata": {}, "outputs": [], "source": [ "def delete_endpoint_with_retry(endpoint_name, max_retries=3, wait_seconds=10):\n", diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py index c163c02..3b61ae2 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py @@ -37,7 +37,7 @@ def deploy( sagemaker_session = sagemaker.Session() instance_count = 1 instance_type = "ml.g5.2xlarge" - health_check_timeout = 700 + health_check_timeout = 3600 model_data_download_timeout = 3600 model_config = { From 777c96f850f46b674f4dfbc9736aa0268e259e68 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Tue, 14 Oct 2025 12:46:44 -0400 Subject: [PATCH 15/22] Updating deploy step with new container image --- .../task_05_fmops/steps/deploy_step.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py index 3b61ae2..1d4fdf5 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py @@ -100,17 +100,15 @@ def deploy( print(f"Endpoint {endpoint_name} does not exist, proceeding with deployment") # Continue with model deployment - image_uri = sagemaker.image_uris.retrieve( - framework="djl-lmi", - region=sagemaker_session.boto_session.region_name, - version="latest" - ) + region = sagemaker_session.boto_session.region_name + inference_image_uri = f"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.33.0-lmi15.0.0-cu128" + mlflow.log_param("inference_image_uri", inference_image_uri) model_data = model_artifacts_s3_path # Create model only once model = Model( - image_uri=image_uri, + image_uri=inference_image_uri, model_data=model_data, role=get_execution_role(), env=model_config From b9c92c2093cef3017971a6de751af151f2953115 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Tue, 14 Oct 2025 16:01:56 -0400 Subject: [PATCH 16/22] Okay should be good now --- .../task_05_fmops/05.01_fine-tuning-pipeline.ipynb | 1 + .../task_05_fmops/steps/qualitative_eval_step.py | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index aab6c8b..70fddc9 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -692,6 +692,7 @@ " try:\n", " print(f\"Attempting to delete endpoint {endpoint_name} (attempt {attempt + 1}/{max_retries})\")\n", " sm_client.delete_endpoint(EndpointName=endpoint_name)\n", + " sm_client.delete_endpoint_config(EndpointConfigName=endpoint_name)\n", " print(f\"Endpoint {endpoint_name} deletion initiated successfully\")\n", " \n", " # Wait for endpoint to be fully deleted\n", diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py index 07d509b..9c359d6 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py @@ -145,8 +145,8 @@ def create_bedrock_judge_metrics(): ), examples=medical_accuracy_examples, version="v1", - # model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", - model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", + model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", + # model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", parameters={ "anthropic_version": "bedrock-2023-05-31", "temperature": 0.0, @@ -194,8 +194,8 @@ def create_bedrock_judge_metrics(): ), examples=clinical_reasoning_examples, version="v1", - # model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", - model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", + model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", + # model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", parameters={ "anthropic_version": "bedrock-2023-05-31", "temperature": 0.0, @@ -241,8 +241,8 @@ def create_bedrock_judge_metrics(): ), examples=patient_safety_examples, version="v1", - # model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", - model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", + model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", + # model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", parameters={ "anthropic_version": "bedrock-2023-05-31", "temperature": 0.0, @@ -481,7 +481,7 @@ def evaluate_model_qualitatively(model_config, dataset): mlflow.log_param("qualitative_evaluation_endpoint", endpoint_name) mlflow.log_param("qualitative_evaluation_num_samples", num_samples) mlflow.log_param("qualitative_evaluation_timestamp", datetime.now().isoformat()) - mlflow.log_param("llm_judge_model", "bedrock:/anthropic.claude-3-haiku-20240307-v1:0") + mlflow.log_param("llm_judge_model", "bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0") # Load the test dataset try: From c27cc2b0ecce6968f35977c79c4bdffa8f5d9d35 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Tue, 14 Oct 2025 16:39:15 -0400 Subject: [PATCH 17/22] Squeaky clean --- .../task_05_fmops/steps/deploy_step.py | 12 +----------- .../task_05_fmops/steps/qualitative_eval_step.py | 2 +- .../task_05_fmops/steps/quantitative_eval_step.py | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py index 1d4fdf5..19d3593 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/deploy_step.py @@ -115,23 +115,13 @@ def deploy( ) print(f"deploying endpoint: {endpoint_name}") - - data_capture_config = DataCaptureConfig( - enable_capture=True, - sampling_percentage=100, - destination_s3_uri='s3://sagemaker-us-east-1-329542461890/data-capture/', - capture_options=["REQUEST", "RESPONSE"], - csv_content_types=["text/csv"], - json_content_types=["application/json"] - ) - + predictor = model.deploy( endpoint_name=endpoint_name, initial_instance_count=instance_count, instance_type=instance_type, container_startup_health_check_timeout=health_check_timeout, model_data_download_timeout=model_data_download_timeout, - data_capture_config=data_capture_config ) # Log deployment metrics diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py index 9c359d6..89f59a9 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py @@ -470,7 +470,7 @@ def evaluate_model_qualitatively(model_config, dataset): # Define the model to evaluate model_to_evaluate = { - "name": "Fine-tuned DeepSeek-R1-Distill-Llama-8B", + "name": "Fine-tuned Qwen3-4B-Instruct-2507", "endpoint": endpoint_name } diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py index c1cddf3..2f905b4 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/quantitative_eval_step.py @@ -340,7 +340,7 @@ def evaluate_model_on_dataset(model_config, dataset): # Define the model to evaluate model_to_evaluate = { - "name": "Fine-tuned DeepSeek-R1-Distill-Llama-8B", + "name": "Fine-tuned Qwen3-4B-Instruct-2507", "endpoint": FINETUNED_MODEL_ENDPOINT } # Limit the number of samples to evaluate (for faster execution) From 7c01527105777372fd47a9dfdd328c2c2033ba47 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Wed, 15 Oct 2025 10:43:11 -0400 Subject: [PATCH 18/22] Updated step to use no CRI --- .../task_05_fmops/steps/qualitative_eval_step.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py index 89f59a9..341ae31 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/qualitative_eval_step.py @@ -145,8 +145,8 @@ def create_bedrock_judge_metrics(): ), examples=medical_accuracy_examples, version="v1", - model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", - # model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", + # model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", + model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", parameters={ "anthropic_version": "bedrock-2023-05-31", "temperature": 0.0, @@ -194,8 +194,8 @@ def create_bedrock_judge_metrics(): ), examples=clinical_reasoning_examples, version="v1", - model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", - # model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", + # model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", + model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", parameters={ "anthropic_version": "bedrock-2023-05-31", "temperature": 0.0, @@ -241,8 +241,8 @@ def create_bedrock_judge_metrics(): ), examples=patient_safety_examples, version="v1", - model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", - # model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", + # model="bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0", + model="bedrock:/anthropic.claude-3-haiku-20240307-v1:0", parameters={ "anthropic_version": "bedrock-2023-05-31", "temperature": 0.0, @@ -481,7 +481,8 @@ def evaluate_model_qualitatively(model_config, dataset): mlflow.log_param("qualitative_evaluation_endpoint", endpoint_name) mlflow.log_param("qualitative_evaluation_num_samples", num_samples) mlflow.log_param("qualitative_evaluation_timestamp", datetime.now().isoformat()) - mlflow.log_param("llm_judge_model", "bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0") + mlflow.log_param("llm_judge_model", "bedrock:/anthropic.claude-3-haiku-20240307-v1:0") + # mlflow.log_param("llm_judge_model", "bedrock:/us.anthropic.claude-3-haiku-20240307-v1:0") # Load the test dataset try: From 89b499b452290a00e37b9c314de0e773ec423867 Mon Sep 17 00:00:00 2001 From: Giuseppe Zappia Date: Wed, 15 Oct 2025 09:54:49 -0600 Subject: [PATCH 19/22] Merged Qwen4B tasks --- ...01_search_and_deploy_huggingface_llm.ipynb | 115 +++---- .../scripts/requirements.txt | 8 +- ...=> 02.01_finetune_Qwen3-4B-instruct.ipynb} | 289 +++++++----------- .../scripts/requirements.txt | 18 +- .../scripts/train.py | 186 ++++------- ...oundation_model_evaluation_lighteval.ipynb | 168 +++++----- .../images/sft_5000_train_100_test_bars.png | Bin 42769 -> 37934 bytes .../sft_5000_train_100_test_compare.png | Bin 25835 -> 26506 bytes .../images/sft_5000_train_100_test_scores.png | Bin 23883 -> 20496 bytes .../scripts/requirements.txt | 8 + ...drock_guardrails_apply_guardrail_api.ipynb | 169 ++++++---- .../scripts/requirements.txt | 1 + 12 files changed, 463 insertions(+), 499 deletions(-) rename workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/{02.01_finetune_deepseekr1.ipynb => 02.01_finetune_Qwen3-4B-instruct.ipynb} (77%) create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/scripts/requirements.txt create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_04_responsible_ai/scripts/requirements.txt diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_01_foundation_model_playground/01.01_search_and_deploy_huggingface_llm.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_01_foundation_model_playground/01.01_search_and_deploy_huggingface_llm.ipynb index 07345a8..8a2744d 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_01_foundation_model_playground/01.01_search_and_deploy_huggingface_llm.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_01_foundation_model_playground/01.01_search_and_deploy_huggingface_llm.ipynb @@ -15,7 +15,7 @@ "id": "fcbb5d61-8a0b-47d9-a7c5-0c041c82b8bf", "metadata": {}, "source": [ - "# 🚀 Deploy `deepseek-ai/DeepSeek-R1-Distill-Llama-8B` on Amazon SageMaker" + "# 🚀 Deploy `Qwen/Qwen3-4B-Instruct-2507` on Amazon SageMaker" ] }, { @@ -23,6 +23,8 @@ "id": "dd210e90-21e1-4f03-a08e-c3fba9aa6979", "metadata": {}, "source": [ + "## Prerequisites\n", + "\n", "To start off, let's install some packages to help us through the notebooks. **Restart the kernel after packages have been installed.**" ] }, @@ -57,6 +59,14 @@ "get_ipython().kernel.do_shutdown(True)" ] }, + { + "cell_type": "markdown", + "id": "a947367a-bea3-498a-9548-d6e6e08f0d10", + "metadata": {}, + "source": [ + "***" + ] + }, { "cell_type": "code", "execution_count": null, @@ -66,9 +76,9 @@ "source": [ "import os\n", "import sagemaker\n", - "from sagemaker.djl_inference import DJLModel\n", - "from ipywidgets import Dropdown\n", - "\n", + "import boto3\n", + "import shutil\n", + "from sagemaker.config import load_sagemaker_config\n", "import sys\n", "sys.path.append(os.path.dirname(os.getcwd()))\n", "\n", @@ -78,24 +88,12 @@ " print_dialog,\n", " format_messages,\n", " write_eula\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8b53f21c-3a65-44fc-b547-712d971cd652", - "metadata": {}, - "outputs": [], - "source": [ - "import boto3\n", - "import shutil\n", - "import sagemaker\n", - "from sagemaker.config import load_sagemaker_config\n", + ")\n", "\n", "sagemaker_session = sagemaker.Session()\n", "s3_client = boto3.client('s3')\n", "\n", + "region = sagemaker_session.boto_session.region_name\n", "bucket_name = sagemaker_session.default_bucket()\n", "default_prefix = sagemaker_session.default_bucket_prefix\n", "configs = load_sagemaker_config()\n", @@ -103,6 +101,7 @@ "session = sagemaker.Session()\n", "role = sagemaker.get_execution_role()\n", "\n", + "\n", "print(f\"Execution Role: {role}\")\n", "print(f\"Default S3 Bucket: {bucket_name}\")" ] @@ -130,11 +129,14 @@ "metadata": {}, "outputs": [], "source": [ - "inference_image_uri = sagemaker.image_uris.retrieve(\n", - " framework=\"djl-lmi\", \n", - " region=session.boto_session.region_name, \n", - " version=\"0.29.0\"\n", - ")\n", + "# commenting until LMI 0.33.0 available via SageMaker SDK\n", + "# inference_image_uri = sagemaker.image_uris.retrieve(\n", + "# framework=\"djl-lmi\", \n", + "# region=session.boto_session.region_name, \n", + "# version=\"0.33.0\"\n", + "# )\n", + "\n", + "inference_image_uri = f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.33.0-lmi15.0.0-cu128\"\n", "pretty_print_html(f\"using image to host: {inference_image_uri}\")" ] }, @@ -153,7 +155,7 @@ "metadata": {}, "outputs": [], "source": [ - "model_id = \"deepseek-ai/DeepSeek-R1-Distill-Llama-8B\"\n", + "model_id = \"Qwen/Qwen3-4B-Instruct-2507\"\n", "model_id_filesafe = model_id.replace(\"/\",\"_\")\n", "\n", "use_local_model = True #set to false for the training job to download from HF, otherwise True will download locally" @@ -225,7 +227,7 @@ "metadata": {}, "outputs": [], "source": [ - "model_name = \"DeepSeek-R1-Distill-Llama-8B\"\n", + "model_name = \"Qwen3-4B-Instruct-2507\"\n", "\n", "lmi_model = sagemaker.Model(\n", " image_uri=inference_image_uri,\n", @@ -242,12 +244,15 @@ "metadata": {}, "outputs": [], "source": [ - "base_endpoint_name = f\"{model_name}-endpoint\"\n", + "from sagemaker.utils import name_from_base\n", + "\n", + "endpoint_name = f\"{model_name}-endpoint\"\n", + "BASE_ENDPOINT_NAME = name_from_base(endpoint_name)\n", "\n", "predictor = lmi_model.deploy(\n", " initial_instance_count=1, \n", " instance_type=\"ml.g5.2xlarge\",\n", - " endpoint_name=base_endpoint_name\n", + " endpoint_name=BASE_ENDPOINT_NAME\n", ")" ] }, @@ -258,30 +263,19 @@ "metadata": {}, "outputs": [], "source": [ - "base_prompt = f\"\"\"\n", - "<|begin_of_text|>\n", - "<|start_header_id|>system<|end_header_id|>\n", - "You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", + "SYSTEM_PROMPT = f\"\"\"You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", "Below is an instruction that describes a task, paired with an input that provides further context. \n", "Write a response that appropriately completes the request.\n", - "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", - "<|eot_id|><|start_header_id|>user<|end_header_id|>\n", - "{{question}}<|eot_id|>\n", - "<|start_header_id|>assistant<|end_header_id|>\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6b37e7f1-730c-4b31-aa3b-55e2009f8f04", - "metadata": {}, - "outputs": [], - "source": [ - "prompt = base_prompt.format(\n", - " question=\"A 3-week-old child has been diagnosed with late onset perinatal meningitis, and the CSF culture shows gram-positive bacilli. What characteristic of this bacterium can specifically differentiate it from other bacterial agents?\"\n", - ")\n", + "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\"\"\"\n", + "\n", + "USER_PROMPT = \"A 3-week-old child has been diagnosed with late onset perinatal meningitis, and the CSF culture shows gram-positive bacilli. What characteristic of this bacterium can specifically differentiate it from other bacterial agents?\"\n", "\n", - "print(prompt)" + "messages = [\n", + " {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n", + " {\"role\": \"user\", \"content\": USER_PROMPT},\n", + "]\n", + "\n", + "messages" ] }, { @@ -292,35 +286,44 @@ "outputs": [], "source": [ "predictor = sagemaker.Predictor(\n", - " endpoint_name=base_endpoint_name,\n", + " endpoint_name=BASE_ENDPOINT_NAME,\n", " sagemaker_session=sagemaker_session,\n", " serializer=sagemaker.serializers.JSONSerializer(),\n", " deserializer=sagemaker.deserializers.JSONDeserializer(),\n", ")\n", "\n", "response = predictor.predict({\n", - "\t\"inputs\": prompt,\n", + "\t\"messages\": messages,\n", " \"parameters\": {\n", " \"temperature\": 0.2,\n", " \"top_p\": 0.9,\n", " \"return_full_text\": False,\n", - " \"max_new_tokens\": 1024,\n", - " \"stop\": ['<|eot_id|>']\n", + " \"max_new_tokens\": 1024\n", " }\n", "})\n", "\n", - "response = response[\"generated_text\"].split(\"<|eot_id|>\")[0]\n", + "response[\"choices\"][0][\"message\"][\"content\"]" + ] + }, + { + "cell_type": "markdown", + "id": "165c8660-ee18-411f-9d8a-8032c6171d77", + "metadata": {}, + "source": [ + "### Store variables\n", "\n", - "response" + "Save the endpoint name for use later" ] }, { "cell_type": "code", "execution_count": null, - "id": "dbfc37bb-dc1f-4ba7-9948-6e482c1c86b0", + "id": "0ed6ca9e-705c-4d01-9118-110b86caaef6", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "%store BASE_ENDPOINT_NAME" + ] } ], "metadata": { diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_01_foundation_model_playground/scripts/requirements.txt b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_01_foundation_model_playground/scripts/requirements.txt index 43a0ec8..0220b35 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_01_foundation_model_playground/scripts/requirements.txt +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_01_foundation_model_playground/scripts/requirements.txt @@ -1,16 +1,16 @@ -transformers==4.48.2 +transformers==4.53.0 peft==0.14.0 accelerate==1.3.0 bitsandbytes==0.45.1 datasets==3.2.0 evaluate==0.4.3 huggingface_hub[hf_transfer]==0.33.4 -mlflow +mlflow==2.22.2 safetensors>=0.4.5 -sagemaker==2.239.0 +sagemaker==2.252.0 sagemaker-mlflow==0.1.0 sentencepiece==0.2.0 scikit-learn==1.6.1 tokenizers>=0.21.0 trl==0.9.6 -py7zr \ No newline at end of file +py7zr==1.0.0 \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/02.01_finetune_deepseekr1.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/02.01_finetune_Qwen3-4B-instruct.ipynb similarity index 77% rename from workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/02.01_finetune_deepseekr1.ipynb rename to workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/02.01_finetune_Qwen3-4B-instruct.ipynb index cb15d2b..12617b1 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/02.01_finetune_deepseekr1.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/02.01_finetune_Qwen3-4B-instruct.ipynb @@ -85,11 +85,21 @@ "s3_client = boto3.client('s3')\n", "\n", "sagemaker_session = sagemaker.Session()\n", + "\n", + "region = sagemaker_session.boto_session.region_name\n", "bucket_name = sagemaker_session.default_bucket()\n", "default_prefix = sagemaker_session.default_bucket_prefix\n", "configs = load_sagemaker_config()" ] }, + { + "cell_type": "markdown", + "id": "9e64f9c1-8c48-455c-a8a7-ae3793d79cbe", + "metadata": {}, + "source": [ + "If you have your own MLflow tracking server, update the `TrackingServerName` value below to enable experiment tracking." + ] + }, { "cell_type": "code", "execution_count": null, @@ -123,7 +133,7 @@ "import os\n", "\n", "os.environ[\"mlflow_uri\"] = mlflow_tracking_server_uri\n", - "os.environ[\"mlflow_experiment_name\"] = \"deepseek-r1-distill-llama-8b-sft\"" + "os.environ[\"mlflow_experiment_name\"] = \"Qwen3-4B-Instruct-2507-sft\"" ] }, { @@ -156,12 +166,11 @@ "from datasets import load_dataset\n", "import pandas as pd\n", "\n", - "dataset = load_dataset(\"FreedomIntelligence/medical-o1-reasoning-SFT\", \"en\")\n", + "num_samples = 100\n", "\n", - "df = pd.DataFrame(dataset['train'])\n", - "df = df[:100]\n", + "full_dataset = load_dataset(\"FreedomIntelligence/medical-o1-reasoning-SFT\", \"en\", split=f\"train[:{num_samples}]\")\n", "\n", - "df.head()" + "full_dataset[0]" ] }, { @@ -171,12 +180,12 @@ "metadata": {}, "outputs": [], "source": [ - "from sklearn.model_selection import train_test_split\n", - "\n", - "train, test = train_test_split(df, test_size=0.1, random_state=42, shuffle=True)\n", + "train_test_split_datasets = full_dataset.train_test_split(test_size=0.1, seed=42)\n", + "train_dataset = train_test_split_datasets[\"train\"]\n", + "test_dataset = train_test_split_datasets[\"test\"]\n", "\n", - "print(\"Number of train elements: \", len(train))\n", - "print(\"Number of test elements: \", len(test))" + "print(f\"Number of train elements: {len(train_dataset)}\")\n", + "print(f\"Number of test elements: {len(test_dataset)}\")" ] }, { @@ -204,28 +213,23 @@ }, "outputs": [], "source": [ - "# custom instruct prompt start\n", - "prompt_template = f\"\"\"\n", - "<|begin_of_text|>\n", - "<|start_header_id|>system<|end_header_id|>\n", - "You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", + "SYSTEM_PROMPT = \"\"\"You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", "Below is an instruction that describes a task, paired with an input that provides further context. \n", "Write a response that appropriately completes the request.\n", - "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", - "<|eot_id|><|start_header_id|>user<|end_header_id|>\n", - "{{question}}<|eot_id|>\n", - "<|start_header_id|>assistant<|end_header_id|>\n", - "{{complex_cot}}\n", + "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\"\"\"\n", "\n", - "{{answer}}\n", - "<|eot_id|>\n", - "\"\"\"\n", "\n", "# template dataset to add prompt to each sample\n", - "def template_dataset(sample):\n", - " sample[\"text\"] = prompt_template.format(question=sample[\"Question\"],\n", - " complex_cot=sample[\"Complex_CoT\"],\n", - " answer=sample[\"Response\"])\n", + "def convert_to_messages(sample, system_prompt=\"\"):\n", + " \n", + " messages = [\n", + " {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n", + " {\"role\": \"user\", \"content\": sample[\"Question\"]},\n", + " {\"role\": \"assistant\", \"content\": f\"{sample[\"Complex_CoT\"]}\\n\\n{sample[\"Response\"]}\"}\n", + " ]\n", + "\n", + " sample[\"messages\"] = messages\n", + " \n", " return sample" ] }, @@ -251,19 +255,14 @@ }, "outputs": [], "source": [ - "from datasets import Dataset, DatasetDict\n", "from random import randint\n", "\n", - "train_dataset = Dataset.from_pandas(train)\n", - "test_dataset = Dataset.from_pandas(test)\n", + "train_dataset = train_dataset.map(convert_to_messages, remove_columns=list(full_dataset.features), fn_kwargs={\"system_prompt\": SYSTEM_PROMPT})\n", + "test_dataset = test_dataset.map(convert_to_messages, remove_columns=list(full_dataset.features), fn_kwargs={\"system_prompt\": SYSTEM_PROMPT})\n", "\n", - "dataset = DatasetDict({\"train\": train_dataset, \"test\": test_dataset})\n", - "\n", - "train_dataset = dataset[\"train\"].map(template_dataset, remove_columns=list(dataset[\"train\"].features))\n", - "\n", - "print(train_dataset[randint(0, len(dataset))][\"text\"])\n", - "\n", - "test_dataset = dataset[\"test\"].map(template_dataset, remove_columns=list(dataset[\"test\"].features))" + "#grab a sample from the training and test sets\n", + "print(f\"Train Sample:\\n{train_dataset[randint(0, len(train_dataset)-1)]}\\n\\n\")\n", + "print(f\"Test Sample:\\n{test_dataset[randint(0, len(test_dataset)-1)]}\\n\\n\")" ] }, { @@ -315,33 +314,18 @@ { "cell_type": "code", "execution_count": null, - "id": "db59858b-e895-4877-8c96-264e152c25cc", + "id": "f79dbcf4-7ff2-4a6d-9bf9-5c27832fd5b1", "metadata": {}, "outputs": [], "source": [ - "import matplotlib.pyplot as plt\n", + "from utils import plot_length_distribution\n", "\n", - "def plot_data_lengths(tokenized_train_dataset, tokenized_validation_dataset):\n", - " lengths1 = [len(x[\"text\"].split()) for x in tokenized_train_dataset]\n", - " lengths2 = [len(x[\"text\"].split()) for x in tokenized_validation_dataset]\n", - " lengths = lengths1 + lengths2\n", - " \n", - " plt.figure(figsize=(10,6))\n", - " plt.hist(lengths, bins=20, alpha=0.7, color=\"blue\")\n", - " plt.xlabel(\"prompt lengths\")\n", - " plt.ylabel(\"Frequency\")\n", - " plt.title(\"Distribution of lengths of input_ids\")\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26cb0f5d-85df-4c18-a3a7-a62ecb010c3c", - "metadata": {}, - "outputs": [], - "source": [ - "plot_data_lengths(train_dataset, test_dataset)" + "plot_length_distribution(\n", + " train_dataset=train_dataset,\n", + " validation_dataset=test_dataset,\n", + " bins=20,\n", + " figsize=(10, 6)\n", + ")" ] }, { @@ -367,7 +351,7 @@ "source": [ "We are now ready to fine-tune our model. We will use the [Trainer](https://huggingface.co/docs/transformers/main_classes/trainer) from transfomers to fine-tune our model. We prepared a script [train.py](./scripts/train.py) which will loads the dataset from disk, prepare the model, tokenizer and start the training.\n", "\n", - "For configuration we use `TrlParser`, that allows us to provide hyperparameters in a `yaml` file. This yaml will be uploaded and provided to Amazon SageMaker similar to our datasets. Below is the config file for fine-tuning the model on `ml.g5.12xlarge`. We are saving the config file as `args.yaml` and upload it to S3." + "For configuration we use `TrlParser`, that allows us to provide hyperparameters in a `yaml` file. This yaml will be uploaded and provided to Amazon SageMaker similar to our datasets. Below is the config file for fine-tuning the model on `ml.g5.2xlarge`. We are saving the config file as `args.yaml` and upload it to S3." ] }, { @@ -377,7 +361,7 @@ "metadata": {}, "outputs": [], "source": [ - "model_id = \"deepseek-ai/DeepSeek-R1-Distill-Llama-8B\"\n", + "model_id = \"Qwen/Qwen3-4B-Instruct-2507\"\n", "model_id_filesafe = model_id.replace(\"/\",\"_\")\n", "\n", "use_local_model = True #set to false for the training job to download from HF, otherwise True will download locally" @@ -527,7 +511,7 @@ "metadata": {}, "outputs": [], "source": [ - "instance_type = \"ml.p3.2xlarge\" # Override the instance type if you want to get a different container version\n", + "instance_type = \"ml.g5.2xlarge\" # Override the instance type if you want to get a different container version\n", "\n", "instance_type" ] @@ -584,7 +568,7 @@ "compute_configs = Compute(\n", " instance_type=instance_type,\n", " instance_count=1,\n", - " keep_alive_period_in_seconds=0,\n", + " keep_alive_period_in_seconds=3600,\n", " volume_size_in_gb=50\n", ")\n", "\n", @@ -714,77 +698,11 @@ }, "outputs": [], "source": [ - "job_prefix = f\"train-{model_id.split('/')[-1].replace('.', '-')}-sft-script\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7ed118e7-1c80-4392-8ea5-147b63fc2f03", - "metadata": {}, - "outputs": [], - "source": [ - "def get_last_job_name(job_name_prefix):\n", - " sagemaker_client = boto3.client('sagemaker')\n", - "\n", - " matching_jobs = []\n", - " next_token = None\n", - "\n", - " while True:\n", - " # Prepare the search parameters\n", - " search_params = {\n", - " 'Resource': 'TrainingJob',\n", - " 'SearchExpression': {\n", - " 'Filters': [\n", - " {\n", - " 'Name': 'TrainingJobName',\n", - " 'Operator': 'Contains',\n", - " 'Value': job_name_prefix\n", - " },\n", - " {\n", - " 'Name': 'TrainingJobStatus',\n", - " 'Operator': 'Equals',\n", - " 'Value': \"Completed\"\n", - " }\n", - " ]\n", - " },\n", - " 'SortBy': 'CreationTime',\n", - " 'SortOrder': 'Descending',\n", - " 'MaxResults': 100\n", - " }\n", - "\n", - " # Add NextToken if we have one\n", - " if next_token:\n", - " search_params['NextToken'] = next_token\n", - "\n", - " # Make the search request\n", - " search_response = sagemaker_client.search(**search_params)\n", - "\n", - " # Filter and add matching jobs\n", - " matching_jobs.extend([\n", - " job['TrainingJob']['TrainingJobName'] \n", - " for job in search_response['Results']\n", - " if job['TrainingJob']['TrainingJobName'].startswith(job_name_prefix)\n", - " ])\n", - "\n", - " # Check if we have more results to fetch\n", - " next_token = search_response.get('NextToken')\n", - " if not next_token or matching_jobs: # Stop if we found at least one match or no more results\n", - " break\n", - "\n", - " if not matching_jobs:\n", - " raise ValueError(f\"No completed training jobs found starting with prefix '{job_name_prefix}'\")\n", - "\n", - " return matching_jobs[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18f4e9bd-de61-4806-b314-6bcf988a2c86", - "metadata": {}, - "outputs": [], - "source": [ + "import sys\n", + "from utils import get_last_job_name\n", + "\n", + "job_prefix = f\"train-{model_id.split('/')[-1].replace('.', '-')}-sft-script\"\n", + "\n", "job_name = get_last_job_name(job_prefix)\n", "\n", "job_name" @@ -832,13 +750,15 @@ "metadata": {}, "outputs": [], "source": [ - "image_uri = sagemaker.image_uris.retrieve(\n", - " framework=\"djl-lmi\",\n", - " region=sagemaker_session.boto_session.region_name,\n", - " version=\"latest\"\n", - ")\n", + "# commenting until LMI 0.33.0 is available via SageMaker SDK\n", + "# inference_image_uri = sagemaker.image_uris.retrieve(\n", + "# framework=\"djl-lmi\", \n", + "# region=session.boto_session.region_name, \n", + "# version=\"0.33.0\"\n", + "# )\n", "\n", - "image_uri" + "inference_image_uri = f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.33.0-lmi15.0.0-cu128\"\n", + "print(f\"using image to host: {inference_image_uri}\")" ] }, { @@ -866,7 +786,7 @@ " model_data=f\"s3://{bucket_name}/{job_prefix}/{job_name}/output/model.tar.gz\"\n", "\n", "model = Model(\n", - " image_uri=image_uri,\n", + " image_uri=inference_image_uri,\n", " model_data=model_data,\n", " role=sagemaker.get_execution_role(),\n", " env={\n", @@ -890,7 +810,12 @@ "metadata": {}, "outputs": [], "source": [ - "endpoint_name = f\"{model_id.split('/')[-1].replace('.', '-')}-sft-djl\"" + "from sagemaker.utils import name_from_base\n", + "\n", + "endpoint_name = f\"{model_id.split('/')[-1].replace('.', '-')}-sft\"\n", + "TUNED_ENDPOINT_NAME = name_from_base(endpoint_name)\n", + "\n", + "TUNED_ENDPOINT_NAME" ] }, { @@ -911,7 +836,7 @@ "outputs": [], "source": [ "predictor = model.deploy(\n", - " endpoint_name=endpoint_name,\n", + " endpoint_name=TUNED_ENDPOINT_NAME,\n", " initial_instance_count=instance_count,\n", " instance_type=instance_type,\n", " container_startup_health_check_timeout=health_check_timeout,\n", @@ -927,16 +852,6 @@ "#### Predict" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "a61094a1-1fa9-495f-8343-aa27d9c4ba2f", - "metadata": {}, - "outputs": [], - "source": [ - "endpoint_name = f\"{model_id.split('/')[-1].replace('.', '-')}-sft-djl\"" - ] - }, { "cell_type": "code", "execution_count": null, @@ -945,7 +860,7 @@ "outputs": [], "source": [ "predictor = sagemaker.Predictor(\n", - " endpoint_name=endpoint_name,\n", + " endpoint_name=TUNED_ENDPOINT_NAME,\n", " sagemaker_session=sagemaker_session,\n", " serializer=sagemaker.serializers.JSONSerializer(),\n", " deserializer=sagemaker.deserializers.JSONDeserializer(),\n", @@ -955,63 +870,69 @@ { "cell_type": "code", "execution_count": null, - "id": "1d5b03ad-a4aa-4107-be89-5d1160cd2d01", + "id": "57cdcea8-ecea-4a20-b178-a4e13feb24a3", "metadata": {}, "outputs": [], "source": [ - "base_prompt = f\"\"\"\n", - "<|begin_of_text|>\n", - "<|start_header_id|>system<|end_header_id|>\n", - "You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", + "SYSTEM_PROMPT = f\"\"\"You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", "Below is an instruction that describes a task, paired with an input that provides further context. \n", "Write a response that appropriately completes the request.\n", - "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", - "<|eot_id|><|start_header_id|>user<|end_header_id|>\n", - "{{question}}<|eot_id|>\n", - "<|start_header_id|>assistant<|end_header_id|>\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a19bb461-4995-465a-a5ae-0c13c067180d", - "metadata": {}, - "outputs": [], - "source": [ - "prompt = base_prompt.format(\n", - " question=\"A 3-week-old child has been diagnosed with late onset perinatal meningitis, and the CSF culture shows gram-positive bacilli. What characteristic of this bacterium can specifically differentiate it from other bacterial agents?\"\n", - ")\n", + "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\"\"\"\n", + "\n", + "USER_PROMPT = \"A 3-week-old child has been diagnosed with late onset perinatal meningitis, and the CSF culture shows gram-positive bacilli. What characteristic of this bacterium can specifically differentiate it from other bacterial agents?\"\n", "\n", - "print(prompt)" + "messages = [\n", + " {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n", + " {\"role\": \"user\", \"content\": USER_PROMPT},\n", + "]\n", + "\n", + "messages" ] }, { "cell_type": "code", "execution_count": null, - "id": "1586b889-6554-4d75-b295-e7dc99673cf0", + "id": "56c8adba-20ff-4a85-9446-0adf0f14157b", "metadata": {}, "outputs": [], "source": [ "response = predictor.predict({\n", - "\t\"inputs\": prompt,\n", + "\t\"messages\": messages,\n", " \"parameters\": {\n", " \"temperature\": 0.2,\n", " \"top_p\": 0.9,\n", " \"return_full_text\": False,\n", - " \"max_new_tokens\": 1024,\n", - " \"stop\": ['<|eot_id|>']\n", + " \"max_new_tokens\": 1024\n", " }\n", "})\n", "\n", - "response = response[\"generated_text\"].split(\"<|eot_id|>\")[0]\n", + "response[\"choices\"][0][\"message\"][\"content\"]" + ] + }, + { + "cell_type": "markdown", + "id": "d9250763-9b18-4601-b083-62bd3cd34724", + "metadata": {}, + "source": [ + "### Store variables\n", "\n", - "response" + "Save the endpoint name for use later" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed5e25cf-6421-4b04-9329-5bc53a2f24e8", + "metadata": {}, + "outputs": [], + "source": [ + "%store TUNED_ENDPOINT_NAME" ] }, { "cell_type": "code", "execution_count": null, - "id": "bb550225-a5b8-4695-bd89-f785ba547f90", + "id": "c10e2414-bd15-4555-b1a3-9a4d8c31f067", "metadata": {}, "outputs": [], "source": [] diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/scripts/requirements.txt b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/scripts/requirements.txt index 06e0d14..6d003dd 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/scripts/requirements.txt +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/scripts/requirements.txt @@ -1,20 +1,20 @@ -transformers==4.50.2 +transformers==4.52.2 peft==0.14.0 accelerate==1.3.0 bitsandbytes==0.45.1 datasets==3.2.0 evaluate==0.4.3 huggingface_hub[hf_transfer]==0.33.4 -mlflow +mlflow==2.22.2 safetensors>=0.5.2 -sagemaker==2.244.0 +sagemaker==2.252.0 sagemaker-mlflow==0.1.0 sentencepiece==0.2.0 scikit-learn==1.6.1 tokenizers>=0.21.0 -trl==0.9.6 -psutil -py7zr -pynvml -xtarfile -rouge-score \ No newline at end of file +trl==0.18.0 +psutil==7.1.0 +py7zr==1.0.0 +pynvml==13.0.1 +xtarfile==0.2.1 +rouge-score==0.1.2 \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/scripts/train.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/scripts/train.py index 8f1fcb9..120d584 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/scripts/train.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/scripts/train.py @@ -1,27 +1,21 @@ import os import datetime -from typing import Dict, Optional, Tuple +from typing import Dict, Optional from dataclasses import dataclass, field -from functools import partial -from itertools import chain from accelerate import Accelerator -import bitsandbytes as bnb from huggingface_hub import snapshot_download from datasets import load_dataset import mlflow -from mlflow.models import infer_signature import torch -import transformers from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TrainingArguments, set_seed -from peft import AutoPeftModelForCausalLM, LoraConfig, get_peft_model, prepare_model_for_kbit_training +from peft import AutoPeftModelForCausalLM, LoraConfig, prepare_model_for_kbit_training -from trl.commands.cli_utils import TrlParser -from trl import SFTTrainer +from trl import SFTTrainer, TrlParser from sagemaker.s3 import S3Downloader import subprocess @@ -110,7 +104,7 @@ def download_model(model_name): if model_name.startswith("s3://"): print(f"Downloading model from S3: {model_name}") - subprocess.run(['aws', 's3', 'cp', model_name, destination, '--recursive']) + subprocess.run(['aws', 's3', 'cp', model_name, destination, '--recursive', '--quiet']) else: print(f"Downloading model from HF: {model_name}") snapshot_download(repo_id=model_name, local_dir=destination) @@ -118,35 +112,6 @@ def download_model(model_name): print(f"Model {model_name} downloaded under {destination}") -def group_texts(examples, block_size=2048): - """ - Groups a list of tokenized text examples into fixed-size blocks for language model training. - - Args: - examples (dict): A dictionary where keys are feature names (e.g., "input_ids") and values - are lists of tokenized sequences. - block_size (int, optional): The size of each chunk. Defaults to 2048. - - Returns: - dict: A dictionary containing the grouped chunks for each feature. An additional "labels" key - is included, which is a copy of the "input_ids" key. - """ - # Concatenate all texts. - concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()} - total_length = len(concatenated_examples[list(examples.keys())[0]]) - # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can - # customize this part to your needs. - if total_length >= block_size: - total_length = (total_length // block_size) * block_size - # Split by chunks of max_len. - result = { - k: [t[i : i + block_size] for i in range(0, total_length, block_size)] - for k, t in concatenated_examples.items() - } - result["labels"] = result["input_ids"].copy() - return result - - def set_custom_env(env_vars: Dict[str, str]) -> None: """ Set custom environment variables. @@ -176,8 +141,26 @@ def set_custom_env(env_vars: Dict[str, str]) -> None: for key, value in env_vars.items(): print(f" {key}: {value}") +def load_data(training_data_location, test_data_location): + # Load datasets + train_ds = load_dataset( + "json", + data_files=os.path.join(training_data_location, "dataset.json"), + split="train" + ) -def train(script_args, training_args, train_ds, test_ds): + if script_args.test_dataset_path: + test_ds = load_dataset( + "json", + data_files=os.path.join(test_data_location, "dataset.json"), + split="train" + ) + else: + test_ds = None + + return train_ds, test_ds + +def train(script_args, training_args): set_seed(training_args.seed) mlflow_enabled = ( @@ -209,19 +192,21 @@ def train(script_args, training_args, train_ds, test_ds): # Set Tokenizer pad Token tokenizer.pad_token = tokenizer.eos_token - # tokenize and chunk dataset - lm_train_dataset = train_ds.map( - lambda sample: tokenizer(sample["text"]), remove_columns=list(train_ds.features) - ) + # # tokenize and chunk dataset + # lm_train_dataset = train_ds.map( + # lambda sample: tokenizer(sample["text"]), remove_columns=list(train_ds.features) + # ) - if test_ds is not None: - lm_test_dataset = test_ds.map( - lambda sample: tokenizer(sample["text"]), remove_columns=list(train_ds.features) - ) + # if test_ds is not None: + # lm_test_dataset = test_ds.map( + # lambda sample: tokenizer(sample["text"]), remove_columns=list(train_ds.features) + # ) - print(f"Total number of test samples: {len(lm_test_dataset)}") - else: - lm_test_dataset = None + # print(f"Total number of test samples: {len(lm_test_dataset)}") + # else: + # lm_test_dataset = None + + train_ds, test_ds = load_data(script_args.train_dataset_path, script_args.test_dataset_path) accelerator.wait_for_everyone() @@ -283,7 +268,7 @@ def train(script_args, training_args, train_ds, test_ds): ) if training_args.fsdp is None and training_args.fsdp_config is None: - model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=gradient_checkpointing) + model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=training_args.gradient_checkpointing) if training_args.gradient_checkpointing: model.gradient_checkpointing_enable() @@ -291,7 +276,7 @@ def train(script_args, training_args, train_ds, test_ds): if training_args.gradient_checkpointing: model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) - config = LoraConfig( + peft_config = LoraConfig( r=script_args.lora_r, lora_alpha=script_args.lora_alpha, target_modules="all-linear", @@ -300,33 +285,22 @@ def train(script_args, training_args, train_ds, test_ds): task_type="CAUSAL_LM" ) - model = get_peft_model(model, config) - print(f"max_seq_length: {script_args.max_seq_length}") + + print("Disabling checkpointing and setting up logging") + training_args.save_strategy="no" + training_args.logging_strategy="steps" + training_args.logging_steps=1 + training_args.log_on_each_node=False + training_args.ddp_find_unused_parameters=False trainer = SFTTrainer( model=model, - train_dataset=lm_train_dataset, - eval_dataset=lm_test_dataset if lm_test_dataset is not None else None, - max_seq_length=script_args.max_seq_length, - args=transformers.TrainingArguments( - per_device_train_batch_size=training_args.per_device_train_batch_size, - per_device_eval_batch_size=training_args.per_device_eval_batch_size, - gradient_accumulation_steps=training_args.gradient_accumulation_steps, - logging_strategy="steps", - logging_steps=1, - log_on_each_node=False, - num_train_epochs=training_args.num_train_epochs, - learning_rate=training_args.learning_rate, - bf16=training_args.bf16, - fp16=training_args.fp16, - ddp_find_unused_parameters=False, - save_strategy="no", - output_dir="outputs", - **trainer_configs - ), - callbacks=None, - data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False), + args=training_args, + train_dataset=train_ds, + eval_dataset=test_ds if test_ds is not None else None, + processing_class=tokenizer, + peft_config=peft_config ) if trainer.accelerator.is_main_process: @@ -334,15 +308,15 @@ def train(script_args, training_args, train_ds, test_ds): if mlflow_enabled: print("MLflow tracking under ", script_args.mlflow_experiment_name) - with mlflow.start_run(run_name=os.environ.get("MLFLOW_RUN_NAME", None)) as run: - train_dataset_mlflow = mlflow.data.from_pandas(train_ds.to_pandas(), name="train_dataset") - mlflow.log_input(train_dataset_mlflow, context="train") + mlflow.start_run(run_name=os.environ.get("MLFLOW_RUN_NAME", None)) + train_dataset_mlflow = mlflow.data.from_pandas(train_ds.to_pandas(), name="train_dataset") + mlflow.log_input(train_dataset_mlflow, context="train") - if test_ds is not None: - test_dataset_mlflow = mlflow.data.from_pandas(test_ds.to_pandas(), name="test_dataset") - mlflow.log_input(test_dataset_mlflow, context="test") + if test_ds is not None: + test_dataset_mlflow = mlflow.data.from_pandas(test_ds.to_pandas(), name="test_dataset") + mlflow.log_input(test_dataset_mlflow, context="test") - trainer.train() + trainer.train() else: trainer.train() @@ -355,7 +329,7 @@ def train(script_args, training_args, train_ds, test_ds): # merge adapter weights with base model and save # save int 4 model - trainer.model.save_pretrained(output_dir, safe_serialization=False) + trainer.save_model(output_dir) if accelerator.is_main_process: # clear memory @@ -381,35 +355,16 @@ def train(script_args, training_args, train_ds, test_ds): print("saving merged model...") model.save_pretrained( - training_args.output_dir, safe_serialization=True, max_shard_size="2GB" + training_args.output_dir, + safe_serialization=True ) else: print(f"merge adapter weights: {script_args.merge_weights}") - trainer.model.save_pretrained(training_args.output_dir, safe_serialization=True) + trainer.save_model(training_args.output_dir) if accelerator.is_main_process: tokenizer.save_pretrained(training_args.output_dir) - # if mlflow_enabled: - # # Model registration in MLFlow - # print("MLflow model registration under ", script_args.mlflow_experiment_name) - - # params = { - # "top_p": 0.9, - # "temperature": 0.2, - # "max_new_tokens": 2048, - # } - # signature = infer_signature("inputs", "generated_text", params=params) - - # mlflow.transformers.log_model( - # transformers_model={"model": model, "tokenizer": tokenizer}, - # signature=signature, - # artifact_path="model", # This is a relative path to save model files within MLflow run - # model_config=params, - # task="text-generation", - # registered_model_name=f"model-{os.environ.get('MLFLOW_RUN_NAME', '').split('Fine-tuning-')[-1]}" - # ) - accelerator.wait_for_everyone() @@ -438,22 +393,5 @@ def train(script_args, training_args, train_ds, test_ds): set_custom_env({"MLFLOW_RUN_NAME": f"Fine-tuning-{formatted_datetime}"}) set_custom_env({"MLFLOW_EXPERIMENT_NAME": script_args.mlflow_experiment_name}) - - # Load datasets - train_ds = load_dataset( - "json", - data_files=os.path.join(script_args.train_dataset_path, "dataset.json"), - split="train" - ) - - if script_args.test_dataset_path: - test_ds = load_dataset( - "json", - data_files=os.path.join(script_args.test_dataset_path, "dataset.json"), - split="train" - ) - else: - test_ds = None - # launch training - train(script_args, training_args, train_ds, test_ds) + train(script_args, training_args) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/03.01_foundation_model_evaluation_lighteval.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/03.01_foundation_model_evaluation_lighteval.ipynb index d26d622..a3bfd8b 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/03.01_foundation_model_evaluation_lighteval.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/03.01_foundation_model_evaluation_lighteval.ipynb @@ -8,7 +8,15 @@ "# Comparing Model Performance after Fine-Tuning\n", "In this example, we will take the pre-existing SageMaker endpoints that you deployed in previous exercises and use them to generate data that can be leveraged for quality comparison. This data can be used to take a quantitative approach to judge the efficacy of fine-tuning your models.\n", "\n", - "This example will run through samples of the medical-o1-reasoning dataset (paper here) on the Hugging Face data hub to generate summaries of earnings calls transcripts and use the [lighteval](https://huggingface.co/docs/lighteval/index) from Hugging Face for analysis on those summaries." + "This example will run through samples of the medical-o1-reasoning dataset (FreedomIntelligence/medical-o1-reasoning-SFT) on the Hugging Face data hub for medical Q&A and use the [lighteval](https://huggingface.co/docs/lighteval/index) from Hugging Face for analysis." + ] + }, + { + "cell_type": "markdown", + "id": "3d5d5ff2-dda1-450e-a098-976986747f62", + "metadata": {}, + "source": [ + "## Prerequisites" ] }, { @@ -20,8 +28,7 @@ }, "outputs": [], "source": [ - "# Install the required packages and restart the kernel\n", - "%pip install -Uq datasets==3.5.0 pandas==2.2.3 matplotlib==3.10.1 numpy==1.26.4 boto3==1.37.1 tqdm==4.67.1 lighteval[math]==0.9.2" + "%pip install -r ./scripts/requirements.txt" ] }, { @@ -55,14 +62,12 @@ "import json\n", "import time\n", "import boto3\n", + "import sagemaker\n", "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from tqdm.notebook import tqdm\n", "from datasets import load_dataset\n", - "import torch\n", - "import torchvision\n", - "import transformers\n", "\n", "# Import LightEval metrics\n", "from lighteval.metrics.metrics_sample import ROUGE, Doc" @@ -73,28 +78,37 @@ "id": "5b1341fb-a37e-4f9f-9d3f-32233d58427f", "metadata": {}, "source": [ - "#### Update the base model and fine-tuned model endpoints with the names of the endpoints you previously created. \n", - "You can find these in **SageMaker Studio > Deployments > Endpoints**" + "#### Fetch the saved endpoint names from previous sections, or set them manually by uncommenting the code below. " ] }, { "cell_type": "code", "execution_count": null, - "id": "821e1176-f2af-4e7f-9273-48b2d67e22a7", + "id": "64d1557b-06dc-4b9b-8a4e-bd543f37a868", "metadata": {}, "outputs": [], "source": [ - "# Initialize the SageMaker client\n", - "sm_client = boto3.client('sagemaker-runtime')\n", + "%store -r BASE_ENDPOINT_NAME\n", + "%store -r TUNED_ENDPOINT_NAME\n", "\n", - "# Configure the SageMaker endpoint names\n", - "BASE_MODEL_ENDPOINT = \"DeepSeek-R1-Distill-Llama-8B-endpoint\" # Update with Base model endpoint name\n", - "FINETUNED_MODEL_ENDPOINT = \"DeepSeek-R1-Distill-Llama-8B-sft-djl\" # Update with Fine-tuned model endpoint name\n", + "#BASE_ENDPOINT_NAME = \"\"\n", + "#TUNED_ENDPOINT_NAME = \"\"\n", "\n", + "print(f\"Base Endpoint: {BASE_ENDPOINT_NAME}\")\n", + "print(f\"Tuned Endpoint: {TUNED_ENDPOINT_NAME}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "821e1176-f2af-4e7f-9273-48b2d67e22a7", + "metadata": {}, + "outputs": [], + "source": [ "# Define the model to evaluate\n", "model_to_evaluate = {\n", - " \"name\": \"Fine-tuned DeepSeek-R1-Distill-Llama-8B\", \n", - " \"endpoint\": FINETUNED_MODEL_ENDPOINT\n", + " \"name\": \"Fine-tuned Model\", \n", + " \"endpoint\": TUNED_ENDPOINT_NAME\n", "}" ] }, @@ -132,41 +146,39 @@ "print(\"Response:\\n\", sample[\"Response\"], \"\\n\\n====\\n\")" ] }, - { - "cell_type": "markdown", - "id": "9a1a5e2c-39e9-4d51-a394-b666ffde44f2", - "metadata": {}, - "source": [ - "#### Next, we will create functions to interact with the SageMaker endpoints, define metrics we want to calculate (ROUGE), and define how to evaluate the models with the medical-o1-reasoning dataset. " - ] - }, { "cell_type": "code", "execution_count": null, - "id": "957d8b1e-0761-4f00-ae6d-dc0c9530e46b", + "id": "4f8dc243-5b54-454e-ab78-47455591a166", "metadata": {}, "outputs": [], "source": [ - "#This function allows you to interact with a deployed SageMaker endpoint to get predictions from the DeepSeek model\n", - "def invoke_sagemaker_endpoint(payload, endpoint_name):\n", - " \"\"\"\n", - " Invoke a SageMaker endpoint with the given payload.\n", + "SYSTEM_PROMPT = \"\"\"You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", + "Below is an instruction that describes a task, paired with an input that provides further context. \n", + "Write a response that appropriately completes the request.\n", + "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\"\"\"\n", + "\n", + "\n", + "# template dataset to add prompt to each sample\n", + "def convert_to_messages(sample, system_prompt=\"\", include_answer=True):\n", " \n", - " Args:\n", - " payload (dict): The input data to send to the endpoint\n", - " endpoint_name (str): The name of the SageMaker endpoint\n", - " \n", - " Returns:\n", - " dict: The response from the endpoint\n", - " \"\"\"\n", - " response = sm_client.invoke_endpoint(\n", - " EndpointName=endpoint_name,\n", - " ContentType='application/json',\n", - " Body=json.dumps(payload)\n", - " )\n", + " messages = [\n", + " {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n", + " {\"role\": \"user\", \"content\": sample[\"Question\"]},\n", + " ]\n", + "\n", + " if include_answer:\n", + " messages.append({\"role\": \"assistant\", \"content\": f\"{sample[\"Complex_CoT\"]}\\n\\n{sample[\"Response\"]}\"})\n", " \n", - " response_body = response['Body'].read().decode('utf-8')\n", - " return json.loads(response_body)" + " return messages" + ] + }, + { + "cell_type": "markdown", + "id": "9a1a5e2c-39e9-4d51-a394-b666ffde44f2", + "metadata": {}, + "source": [ + "#### Next, we will create functions to interact with the SageMaker endpoints, define metrics we want to calculate (ROUGE), and define how to evaluate the models with the medical-o1-reasoning dataset. " ] }, { @@ -231,7 +243,7 @@ "metadata": {}, "outputs": [], "source": [ - "def generate_summaries_with_model(endpoint_name, dataset):\n", + "def generate_summaries_with_model(predictor, dataset):\n", " \"\"\"\n", " Generate summaries using a model deployed on SageMaker.\n", " \n", @@ -245,23 +257,12 @@ " predictions = []\n", " \n", " for example in tqdm(dataset, desc=\"Generating Responses\"):\n", - " question = example[\"Question\"]\n", - " \n", - " # Prepare the prompt for the model\n", - " prompt = f\"\"\"\n", - " <|begin_of_text|>\n", - " <|start_header_id|>system<|end_header_id|>\n", - " You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \n", - " Below is an instruction that describes a task, paired with an input that provides further context. \n", - " Write a response that appropriately completes the request.\n", - " Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n", - " <|eot_id|><|start_header_id|>user<|end_header_id|>\n", - " {question}<|eot_id|>\n", - " <|start_header_id|>assistant<|end_header_id|>\"\"\"\n", + "\n", + " messages = convert_to_messages(example, system_prompt=SYSTEM_PROMPT, include_answer=False)\n", " \n", " # Payload for SageMaker endpoint\n", " payload = {\n", - " \"inputs\": prompt,\n", + " \"messages\": messages,\n", " \"parameters\": {\n", " \"max_new_tokens\": 512,\n", " \"top_p\": 0.9,\n", @@ -272,18 +273,18 @@ " \n", " # Call the model endpoint\n", " try:\n", - " response = invoke_sagemaker_endpoint(payload, endpoint_name)\n", + " response = predictor.predict(payload)\n", " \n", " # Extract the generated text\n", " if isinstance(response, list):\n", - " prediction = response[0].get('generated_text', '').strip()\n", + " prediction = response[\"choices\"][0][\"message\"][\"content\"].strip()\n", " elif isinstance(response, dict):\n", - " prediction = response.get('generated_text', '').strip()\n", + " prediction = response[\"choices\"][0][\"message\"][\"content\"].strip()\n", " else:\n", " prediction = str(response).strip\n", "\n", "\n", - " prediction = prediction.split(\"<|eot_id|>\")[0]\n", + " #prediction = prediction.split(\"<|eot_id|>\")[0]\n", " # Clean up the generated text\n", " #if \"Summary:\" in prediction:\n", " # prediction = prediction.split(\"Summary:\", 1)[1].strip()\n", @@ -317,6 +318,13 @@ " \"\"\"\n", " model_name = model_config[\"name\"]\n", " endpoint_name = model_config[\"endpoint\"]\n", + "\n", + " predictor = sagemaker.Predictor(\n", + " endpoint_name=endpoint_name,\n", + " sagemaker_session=sagemaker.Session(),\n", + " serializer=sagemaker.serializers.JSONSerializer(),\n", + " deserializer=sagemaker.deserializers.JSONDeserializer(),\n", + " )\n", " \n", " print(f\"\\nEvaluating model: {model_name} on endpoint: {endpoint_name}\")\n", " \n", @@ -325,7 +333,7 @@ " \n", " # Generate summaries\n", " print(\"\\nGenerating Responses...\")\n", - " predictions = generate_summaries_with_model(endpoint_name, dataset)\n", + " predictions = generate_summaries_with_model(predictor, dataset)\n", " \n", " # Calculate automated metrics using LightEval\n", " print(\"\\nCalculating evaluation metrics with LightEval...\")\n", @@ -355,7 +363,7 @@ "id": "080aa020-0aaf-438d-b0cb-dd503d248feb", "metadata": {}, "source": [ - "#### In this section, we evaluate the performance of both our base model (DeepSeek-R1-Distill-Llama-8B) and our fine-tuned model on the medical-o1-reasoning dataset using ROUGE metrics, which are standard for evaluating text summarization quality.\n", + "#### In this section, we evaluate the performance of both our base model (Qwen3-4B-Instruct-2507) and our fine-tuned model on the medical-o1-reasoning dataset using ROUGE metrics, which are standard for evaluating text summarization quality.\n", "\n", "The evaluation process:\n", "\n", @@ -382,8 +390,8 @@ "\n", "# Evaluate both models for comparison\n", "base_model_config = {\n", - " \"name\": \"Base DeepSeek-R1-Distill-Llama-8B\",\n", - " \"endpoint\": BASE_MODEL_ENDPOINT\n", + " \"name\": \"Base Model\",\n", + " \"endpoint\": BASE_ENDPOINT_NAME\n", "}\n", "\n", "# Evaluate base model\n", @@ -540,7 +548,7 @@ "source": [ "## Larger Training/Evaluation Results\n", "\n", - "If you were to train this model on 5000 samples and evaluate on 100 test items, you would see the following results:\n", + "If you were to train **Qwen3-4B-Instruct-2507** on **5000** samples and evaluate on **100** test items (total training time 32 mins on an ml.g5.12xlarge instance), you would see the following results:\n", "\n", "![](./images/sft_5000_train_100_test_scores.png)\n", "\n", @@ -549,6 +557,14 @@ "![](images/sft_5000_train_100_test_compare.png)\n" ] }, + { + "cell_type": "markdown", + "id": "b8213dea-cfac-42c2-acca-1e9bcb32dd86", + "metadata": {}, + "source": [ + "## Detailed Comparison Between Models" + ] + }, { "cell_type": "code", "execution_count": null, @@ -557,7 +573,7 @@ "outputs": [], "source": [ "# Display example predictions from both models\n", - "num_examples = min(3, len(dataset))\n", + "num_examples = min(2, len(dataset))\n", "\n", "for i in range(num_examples):\n", " print(f\"\\nExample {i+1}:\")\n", @@ -581,6 +597,16 @@ " print(\"\\n\" + \"=\"*80)" ] }, + { + "cell_type": "markdown", + "id": "636dd59d-2ac8-4260-9f38-b84dc06360eb", + "metadata": {}, + "source": [ + "# Clean Up Endpoints\n", + "\n", + "Run the following code to clean up your base endpoint. It is no longer needed." + ] + }, { "cell_type": "code", "execution_count": null, @@ -591,7 +617,7 @@ "sagemaker_client = boto3.client('sagemaker')\n", "\n", "delete_base_response = sagemaker_client.delete_endpoint(\n", - " EndpointName=BASE_MODEL_ENDPOINT\n", + " EndpointName=BASE_ENDPOINT_NAME\n", ")\n", "\n", "print(delete_base_response)" @@ -605,7 +631,7 @@ "outputs": [], "source": [ "delete_basecfg_response = sagemaker_client.delete_endpoint_config(\n", - " EndpointConfigName=BASE_MODEL_ENDPOINT\n", + " EndpointConfigName=BASE_ENDPOINT_NAME\n", ")\n", "print(delete_basecfg_response)" ] @@ -613,7 +639,7 @@ { "cell_type": "code", "execution_count": null, - "id": "70d31ca0-4755-4511-a814-a0a409265fe1", + "id": "9bc5ebf7-d70c-4537-808a-9478a3eb88c0", "metadata": {}, "outputs": [], "source": [] diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/images/sft_5000_train_100_test_bars.png b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/images/sft_5000_train_100_test_bars.png index 22ceaa88f01e1caa438898d821be6a4109553a21..9b17bfd5f5bc58aeadb7efd67a07491026e7c669 100644 GIT binary patch literal 37934 zcmeFZ2T&AS8wCicC<3COB1jYvi9;00NeK=?iIPMf=Z}&~#J>NO!_Nxa^XB_C&xWTa=Vw2DJi-YP+;?&j$4N4C9kvh}tUEq3xEI*iVRQ=T3M1+ynmt%*DhW3Z9uU=12kKnm7dJErA=Po96D!)1ARGt{KZGzhtK6*u( zpoe)oF6tcPt)?-0TtR)T)1z|F&hFZc`}axreCe*bcz4sX%(;_JH=o7t;11#?6K|Bi zk+>pgY{K(&`VKFDxL`9?q<2s@mDH4UQy|pN6|DlVgw{xlDfFz zcMN=PG3ibyPw+79u7@?;n7I>2e(6>w2UDN5m>>Nr;XV4$5}V!8rS)~B{1!dOdR57K zbZCK={1dTI)Z>o9E^P0Now1C(`J|1g$c$kF)2cVSQ>(6b-(z_ZfqtEduJ#KNZ#$&) zJ9i5{hKec}@8WlHubfS;$G?Lkv?7yNsx>l_Gk1@x=ZuS#{!^8!5rOf1oDr=iaWJ&D z)}tXjE|ist!J9HClTHS-Z{l|D0U1sl(Wln=6eT;Ld1?NnX_W^7ckKr+XjwiJZo>e{BjyK&!{P!Qv*W=665ucO1@aW8d&y5Fo zrQYw7Nc%idNioYS+xPsl0-YMD8)%l6*DbI7!0`ISPs}yxC3jylmndaP&1dwfXVPZ|F}j2jypq(G zAq}}5b|JJgY$;?kwAJWT;dMd3_v=kZ*wrI2_L{f*p3r=a#YCF@K+8WVD9a?pBTY-kz znr@k;D+Crk#_aLXJ=**Itje{^w-Kih;#eNUUEJ))8TD#%8n+(adEcGFm3J%OH04@y zboQfEXPJjNzA8Kqp0cL0T0RuY<4f8w^J>D|!&~xJB#gbf!4a*dq3WnQ%}UNDz_!ii zp^}{CoQ2M?P&HF`LBH1Ef2j99Hli_Z(%dk*V3=JB^BU4?W^M zFvc*} zu(-4jneSM-?1RjfrP`(ZUmUz}AUxY4XKY_&Q6y7DZ^vZkv5vQ{v99@M1T} zzGxEayit2|c$I`5%Yot`WFO`jdld7iAU7*#=}}m2P0pZpnO0pXcgc@$izOA$;044V zzMBr4?SJkZRWcnk5-`p;p*1nE6qi#?I@5dKOp}`H!qSCj)IwBZLf-_91!yb4Z5Zu6 z*<*7ebV}Z5*_QWXFyx=CKHJtfobolLS#^|mq^M`TASzJvy=KZ_d0FoE{pEGBYqa8t zmSR2%#5d!H>Z!I%w)K9xwgyd+&uh+Ye;WH_Fgr$rb9M*cg2ee+25k$81u>pvd(uzI zAa7zHUT+UQZ#~j#w0UdqMNF_dyb^3D;(D!S(S@8?Ur20VpGwK ze--fx@j2mTl;?1|^*;WTWQy1nIDx6KWM8xOxZ&@Nv{%H*?)Vq&afvt8YCK*Bk*%{qa<3@t>)8vboG=S zOV!4na-_KW!G@(>!*l|Nqt2GqXDh5V_1Edj+**Y^_Bsn^*ZQqU>OShv!lOUmes{?o zNvk#{e;#ETRo+nd-m+|TDYr^o^QQWiW`!odX11nvR>DZ*P;c3wrL}5+jG}x;dZLT? zSng(?WhQS1+nE3NsG8-uqnC$zb=~e}cao4vQ}t0k2R;*f!FzDxEn;H-J`zha-|P1R z4E>|1YaKpK%ISn@sah7P*tI^iQM2FJEhslU)qEJ}l#uk!R4CJQ zk>a``N6Zfm*X4=PB2*rV+{$&N*4E!;M#kf@$Fjsu{r8UupNXpYs;;HfQCF1s{!yQg z0|__D8`@SeR*#&c@+GIqvAEBU6TH$i(%+S(5=E{Vlho}ntOs>5yijgc#>ke)Tu;yv zUAVSRS+(glw;n?7O|A-ia}>T`v={LuJItfvuwRshUu;c#XKy&Jr;OE_q_)UqQ1Ft- z>JIALerD?g2`lZqc#oJP>h%GiP{caLcE!Y-)u^p}cLQ2?`2Ln+J!zq_Htyy3Hd1&v z))(+30`Wa<32`QRhH8;ANki6N6nMw4Ok9^K3QmoGb-_s+7KRDn%wM`K@d!zc^RWdL zecdxRmq}>wRM*Br?}RnJMAH*O88y#qdS{PVifRs=9FBYO}Az zLP=F=d3o@!s-Z0cVP$7(Z6EAIUI~gCG*f$I|430m$k5u7^NEr5Qv|28r44ioj)=1m zIJ89AKcREBw6L-hau&UDa)l5$hJJ?KpgXz5{+Z~FM~V;Vq^)fcbhkOVIJs_!ou{Lt z6R|Zi7E-w@^Y`uG|3q(?+S}U*!C+2KPMl7>oYuA`Fm6FXK^PYgjE9Ed#x%?GU!o)|TLz_G15b&EGfwdh+iZMPN|Xe`$)p+WF*H(9mM%MPUEwG_mu9 zN!e>SI1)JWcO})FaTmXzbwj9oiLdaYqMN#@=cwltuSu}n=y;uTSyMyr%=dnAI=YvS zKl&9ep7(xv`I6+ts}H&zuH2QlDBJB z?YeZTYx$ksQqnGApkatEdI{Q1hiB*4RxVb|Bp^2X zHDPSI#VSMQ*|}{${Au-Rkh96|`CO*mNj%OrT}LxpA+j_qV@F$M7|XKn8q=NchMz{; z5mWG>o4W>Wy>ELkwzmpKX`5&A7`I2SRSwqVRbx4A?OO=$ zFBV}$w?|Y1Mb=tuCtfjYe3Tsw;`^}m@^HKE_^=SeS*7MsIPsb-@@MIA-kR&)Tsr1p zcYU8&ZfSeO=jut9b|=(#Z)s@Do^%BA{qrn%Z=_}>#%$Ma*I-4LU6ilf-B80#be?#{s&-A#)@aDSS5YUh zrr|tWxWVz>AVT(u6;_^YLQ3H zh{K>>zrc*3Nn5mCUde#A`YGQ1>FA-O0Uc`&v+z>A9^t1on*-TM3!{Bs$=-D3{DwCh zuUn4@-V|B~CR(o^uSL~`FXY$NYS$fZ4QtL+f@)%X3CONiZh?-?v6YXR2)E@ZYUfeE z^ptewG;WG)P+lqg``haRd0AzAk>q8I#l1?1qK@0yw}+p;;b?m(ORY}K>%2carXRsD zNP1k^samV9p_Q6?WG6}>Frer3JxgD8#;`ds7x@7#M(c5~Ixl2c)kOJBZ7@!s^LbW2 z%QDkS=Ba?H<=U#O!F~;$+^k`XC`;pxSasXF)CaOCuqyP}UM;Y@dEe!i8&brpc;dsP-|0e>&QI=y-WGPiPvdp5|7CV_+ceEE_ zohLXL?#kWVx~sa>r!Gdm8R|ifJ;==0HE4_Idc2g{=|(d@xuXb8MidN0&3&Nq zJoAb&aIavmS}asgN#1-h5RjDvpJW)5IbQJ>XW#!>W`Hp!kniz4&Igkzi#wkFax;~k zg5b~M-Yk3@%G+8OL{{pvzO215ICy^^x9}*y+Rac98?Kr)egko5@-1&0wbRH${&3Pr z^rZ;=FPgP$5m^@GdiD?DDQ;t^Z4TPQSy_xNhC`G3xN7Xs>PibbIj+gp&t?&XNbKRV zhX>|17*ZDf&Y0jpzJ{K1;i+KhEc~B>Gx4f{;zv87kLj>-BvDBDM)q};srZV#aM2kI zS73EilMUrde`?1^Ltvz5<8Sj1>Z(Q@zh=w31sMmAD_4v_6_d&&>kdJrWbK;Yl*>GT=6`^QJ@UA<1OoBjeb46cwYuVkEOvo(?{?R zA$ITF)>;wG)K2ZmZJ@z5UvTbHrHK_OiuqUgi?7FYscq}$Eoj#5smoK4zGn1nJq!`F z=J930)g8dvC3cwvzMGsnYIm5XRH@xCZ>tIKvY*c1{l4t3NzI(N?5454?Bz>Su?;%p zxNK@Y`Z443jF272{&JlMobHN`Z!v)lsnPV9GQn(*o!P{vS8vKS$QNl(*3n?Fj{4wYRW*5_<8j}5Kw10+KPpuQXFJNa=g7PRmwW9DJ`FWxX zqV(C!;!}S_>yrvPQGPMAZ-OP?MLtA-+3$3!Ejq7$G?#dkX-<{KohV~V`7Jlfmw4_; z*f_nHd|M&`P46QG!!^~!JvGcUflp84)bU~6v1SB4y=rI0{!&$$+wf@N^EmORp)!_I zfo{SH63n<7Z4rhCyImffXKCGRuGphBWhnDwd>98q_vztyQNEWl<2GpI=pwCG?Z%9b zUXgh0G^+cE46aplq4LZ(ZXUGpGx?T-_?F%DY;nPQL6x=wHoqsA5p2iE)p|N(6Sabc zc~n%A^t5!W#EhI3Kb%v}-lrqp;>#*3t|d_kY*!3>a-)KT)UzMn8JB=dP+iE_f>91E zX}uYzMnh1qVq*4nMoj+uXRd%1z7J%i*lRH#__x)-@SpCrV~~`i{?IqmzFa=z4won> zPrggQrsCIUO}U@h7|xxngS0_EF@OBJpr%&`h4JMC1MP?!u+vdfvKEs-+ zqVOaJhfqt~F2=`jy}oVfz>vpmgHfJ5GU9t*f>;NAZDQ6Lf2eI?3dmkFO}#uUtKG~E z^?dF_eA{%emeYUiWE4IEqp&H9Bo$y?i8?YlX_m3Yv?tG3Ka*<)RPp8XMz@wN&)JSf zFwx6P1@LVZP3ipPv+enycrH}NsMhPK>-eB+UT1p1OO~>E_qM>;<(WHXQ|YVmGKgAK zmx@5;n&tC`2#csDM@lJ|ch;$@(;FhKB8+BqgJbBKL*H9wICbAb@l|WkUt(yk5xh{z zJAJx9ev)fOv5_sH_295o*}ZUnGo)}vi1ZO7m)xDuQO2JfW+Uk#QTk`if@gNjT8vSY z)^lebz*y`kUlUN+*6Nx}kIMxG@f`?hlivR%3cA;Zm+uCy3X#Vy#vroG<7l13O9C%l zS5+$NwCb}1hLx5|T|;WlDYuFvJ`KAX2MW^LpE&|1`kW&9e0#gQ!fbk+I|U2-HXkN3 zB;Sub%{o^Tg72EgkCn5I)b&8o8dXujM6Krc>1aU2Bie5j4d3M^mFnM?lzKCwId>&^ zpGEYNt%1>3HM}5ST>^em){+ZXZ9Cby;W1*xNdoXdok45LO@d6@<^CrlW7|9X;=MXd z2t^x#7{PferAP9Q-22}{G_X@3K_=KRghSJZ!uss-n$J;IS zDT|R2h^UYVy4Nt?Qmj~C_KW9*&)@o}c|;vrnQ@E?cV(o1(iykUwHbo!6#PaQH*fdX z`W(PSw_=*J_ka>wpM|~nZugEWp5m20$I~oHNwa&iY)P^^;*HWN{(u;yd>$xt|5~elihlzd)fgw!!>rM;wc!K&!ZS>$hN((ayCcc)kvE2cInrmxe^vncj zd*rhHXV5>sKK%NnlHJUWwr+fQDIP|g&Bs`Te&~AW?IOq{!c6>igHbLcY<((jI*)xw zd@$5DwIG)1w%O^lGKtar>(;wXB4hD`w>RBxm_@^^>W2Ycj@HY1b~a;#{&P_aOXYYVAN}r}mhp z*-o0i&e++x=OLrSmdifzRhy{b(eo9@h3GHuo`}sU1{)$;`O(SL>sygT!!4qkB%7IK z0H3~Jb9aSPmrvE|jf8gJ!s<@ada1Ij%&=%m<@suEhK^p{JS2+XiM!!dM;22KdHjvf<}K$B4{w6p z)~o62MQ1+8!&}rKMc8{GK}ogSdS_ojC3^R2Pl-rCs4uZYW6|d4>FZ-|&xS`;1d`VD zDJ9Gn(FNZtxx*bSuQQw38>4FaJjz7%!3tH4Jus8-IGCi(pXO}g3gGkZ;lf<>A^|R0n3+rIl~(O4~Q6cUE(Vv$qfVFCnEb4rD6sWYrZjI zvoW!gXsdZ9gpw$>sJF6qt@7t#=(7xQ!_6Z>jZ(OCaQ)BgV>>eahRk-QXlD6R)W~Mh=?YRQ8~FD>ggjG<(V=CY!8pu}!C5pTql;_F3pw{cqepKp zu%CqL9X!Hg@uVhLJ}D_6BqYcP(a%GET-d!SI+m6GF|kSgyR)LQA5tFndhhxwQct!{ zM_@$gTVBn!&^Lib%AU{SdZWG1=mRE+hUI)%0}=;_lQf^zmx`KH_A!5LsV3XJ7-|vb zXGQtU?4@yrB=W{ow4ev2m7d;2tcFDrV2N3T=PlccR96ep^KI3;D}FB1`mz3=-iPid6~R$Cy3+H-4tIL@uFbs&#&`1DHk!RZY+K^^ti}MTz(^-gVRy`^hAd$c z)zu~$Y;`2CCY}DwA89G=KhS(X=38X;=Kij5qxd&>KUHf2k{*UzA7u`B;uoX|{g$WV z1&#e`OZ0^7YNN4Yn3UM{<4rAD1E>{YwUvdzX*KZeMkU0_-_9B~}B z+{B3OzmqX&OT^dqOGH93X5JnSU`H#5claYo52Ml(% z(Xt9RZ*}lvwdWK0ZVC6gqAd%0bQqi4$Ry{Ph-cj567fAmMN-}K)7@vpn1FpEX3t`w zu)J{U&ZcNWH`}IKSL?&B@1Zy>q7LpL5R_4I;u_w(-GRDdLI_&-+y)F+RAf&RRSB&j zeDM+qQXvV1G10tZS;mX*NI051+(mR=AAk4eZE3-^T(??W<;k2)Dl(MA6SMfV=UaJB zHQBL!Jcu?10kR%@hgN=&b&n->hTwZ7)T+SDa?-ydq55OAU8z3Lj0Dna zDsF-AB^m1!0i|V>f=z6?4O&)yI-sTOI0nwJ8IrDSUvl>D@FZ->p)GpdXh$iJBj~zf zib$Av+80#cL)nty^^}y!v?xWIsyNwzG)>uQ%X0UR@yupwCZq%LcJl0CWOif4S}rDx zG@G%P4HhAu&vc;@ljwt@JIEgNFwi^1%sAi#C$_Dp0rE(LX|N z=~F&1GrKqzvwfpkf#h|cYD4YD4SCO~ees03&*98TM%R?Q<4^PKS~!&W-&?fp5R|P} zfmIwS#K&E{)r^U!ArM`Yo4m$!t@T6Yfh)_lG4(Bntv!nqLM#B};HC#cDpFAYlt?tq zre6vrIMpkmK=O*nOg@D>?CbzoiQo3l-ao}fFP9JF8A+shSW0!-RKax z;m{JAw@dJ{Q)nS)ezklK6^yTw$3P}LK(Au4whYj(|=jt^8A_ zexca#Kx4rCX0AuN2mwOK{8riznnqvYn0TJ%fG2+3=(7Or-Dh!HST{ih#ub(&Q1LGC ziqJQ=e@LR@ZEiJuI9B*rsvRkhD*h=6@{_Rxz^RR$tq2@Bo133fOJcjUC)P5RT|u%Y zH?b!sXZ`wGNiE5@3uiKe)RjNQtQ+-w_0@4XZHt%GCz-$%W@%e1LUC)*{l^P(4F~eF zkr@|@l?kbLd@H@PUERSOH(EEF1aR*sT zs}7_Fl3x5&=zzi&1@EVBny?IpGSeYuwY^_ZF()SbvTBjmV>7v2PFo*+53ZEI_E0=a zaQG8{;^*+p&4P7(F;x}=_nBDR4ieZbDncJYu-P-r+??lsCN+UHE3NrqXH=IV-baxp zfzr*;74F1>0Zev8^5*TMzpw+@wc;eq_?U^Qd37PnC1`#_^tS}rdB)54qiqkoX%ur4 zGkSfa-nX1AW1sM*S4=0`RPF!1@E^WA2?c1%ea-=Bar@`Dli6FDP0J)bfZ?+&PNm&rzb{D$vP)B0?OWnd?|E1qb@WN$zEQj&TaU?&Xm;QP?=mC3W z^~a(fd5L-^G-AAfXXSBP8ksLJKU`eXz3@*h{(6yX5Ae*!W(p=;`dnxxYs*u|@M%=j zl{g^s2cn{d&lKFbwfxWP{X^EH*ggREIg<8xr&)crpgeSAxZd#VhbM@ZVETUE@v-Bh z?Wknw{hz6vd4V;`zt#A3tTtWBoh6VV+5%1|B7{NKabA1R^85oQYl2T-3k=WDysM

`l9KJ+|MT#3L=QUk4cVI5@AaVTwg>56S~kxasONq?ca^!@+P|26 zso{@T`A!xsV6ER8^g=@UyirEm_(vbt#g7blK?yn3&49T0PZ<*8S6f8R^BT9`3cIOD z>bxOW!%IC%W{T>CrWvg)0QH;B%7K}35r z_o>PEdWd|2siRmS2lE*IXBt}WyV&fN$&z^8@m1}OMfktZ#PPyq4ECD^Tr=i8TTb`d zr?X2k&)*Ao%_|B@vdpJi>fTbzVfdx0I9`rtdT>x20o{%y=1g85=lZra z8v+L_LG`ke4hep(pQi+jkny53nAmNLLq+7S(;^Q|)}~sS8t>xwJe9*pWLXa0|Fu3e zSDp7#=MZggKMPbEV9fkUsC6}TlI_CK(=0s7BCB`#cM+X2$pUbW5;ZNe)CeT%eZMbJ zkm%MU3LYalEsw)ZopU_oZMW8S_=X%zsF!rTGgD-))^(#b_UTNi0 zls+gXX3fNZhjR65UoCB=>nm#$4*lPG~72p?^*UU+!L&!mlBZ=(3lBTg(&oxrf}y zjoHFB*6d=DT?TxL3xCno_&n89KvQF@TB>|DdGlW&H<iO;stqWOy))XCl-K%L*2(cXnCfOcacvS)piyPje8w7vWBFm~%W3y94zc~TL3_Cn_7 z)Vg|0NH_gQnFenK+t}87ysu0neJM0`pWnM;Y5`;=1Q@G%4ZgnTc3Xp^d>OS!RYZ<< zCb7FSiPjYEDm7o`X({wiBuLtW@w5G(lIJ!Hu#q~Pmz{N;zB^!lT6?$-nWh9}i9(#Y zEf8EaYb1Gym}e9+LgkfueXxngyNQ@H>S{)A?8$XSn5RZ6VDxMGXB%eRbh4`4Ug zZPUymxMvM?HrusUW>A^LqmA@z3C0>Jp4M3?B)^ZQXPo2j`4U=)rXrx9PhiX~O9WxP zpEpH6N}np`*l@Z%re9equ)1@Zoj`Iv)qTVwunH{fgW1{id@ei~3BA)ar%I;v1b$tr z_A(`;yf)V4B!Zc0ZmbE_+yRBCx^GvFsng3OOovw7;USSl-tEJHQMbn${}t%A*)*gS zg+RwC3*XCx&24^h?CBOENJ~L{cNpY=1TcA$w=mQkyq}6Xx2(&Jo`Ode7GuKrLk{VU z_LpmWH_AwWu+1YH%#3QE;|9WXsO>Tsw}gsGv7?Mb)^wa;{6AXAi}`K*KccG<;t@3U zL!xIsF-7UB(6~&6gyJyZN+3-}oW@Z7VUI}djWXmLg4tu%8G_z+h)@$;T$>6fVs53P ziL)%#x)K;~(?GyvhhW5MeGnt!z_f2)Y_!%#6ej;T#1NsX&$A!XMo zIHM+dpNEP`*}Ju39UVi(Oy2=Cqi%a7aw%$aB-VUB@m%nFUCQl7Pf^U$YSnF zPhlg@E*RYstVlZD>4P~KBI%n%c^^$xlVqc86p#JgAnGo!|Evc13H7rfjcq)Jk&DOa zR08IY(x32NHu_dEb0=|*?g+yIE2Wi5fqeNs%y^-6@0`K^US0n?xU#Qo#x+A1ytHb z30MO2pVjaFjv{cpXvi-*Z`{c|^=k}*fCGvHXr&93HtxSJEC7KdHJ>?6BdRp-`P*U& zK)C_oIg!0e{P@IQM=Q8!_BjCGL1994|5WPl^WB4x+anKmhWS?n1TI4A0qv9M&7kl`T62kzB(OOw2WiH))R(unsXyw}3n!F2FB<8L6-%M;?P%nLoN_fYmdW88m1kjP zjueNv{fM7w4Tb?>KgfgK;AkxhQ+Kpb z7wJ^9peO1wOCvl0$qKeWtBoiFZk3bCpanv8wxM#|?OcWUI|TsMro@9eiU6=+vA}K% z>vft2mAO3fTi3BckZB9XSX=ErW$L zOSUz)Y*ZVcghu!FsPiO`?Np2TF#u-;-w6ER{Grg)#KP$5yQ=6pnt4nrx6Njrch z3bwg(t09J|O18BCz4phkua4mbF)1yc{=LEC@XS61QSzH7)6fVWlh>}iU^U(OHdw2% z1dxR7*C+lF9AzVg=kFu{^*4(N*+Qjq(WxQY{pzSeLrdxxs5$Itd_3;M=s(3+= zLqlZs6B%Tf%E5a9k`}e(`b)A@4{M^<7#Yg}Ov+9g_a@}b2t1@cUZq68mUd<%iKL!G`k$bBPaEpF@}?K~Ojt+y1sX>QZJi7W)7q;f{Xz845C+ji>8%EJ9Z z8QFIM>d+nCNE1+h@)=Ug=PNr!*4rcXCd2f(1@*lneZ?Vv!|wQ5@kcyv%!4jX6eent zO2{6C1d)pKw)3YLZwWy`^I2zQyd5NdFM~C3K{0a2+N+tA*%5M2{F{o*DthsCLdGKv z+`FAs0BVMV(QOiPL*boJ$q7PYpxu@0e|-G4Sxkd}syA#B1aBCGS#~GL>mHpU4H7?c z7Mw$ewdK2SX3sl2DEm|K)yQm(x-PTc60VARO*jt&O~AgheFS^!$qaa$ly&5N4^Dm-_wAJ5XVGbcDdz2dA3@+86+7c9FX z1_@xLqsRqTdwGelgllv3wrwow;X*5N&X?*ozP0R<<}o_A9x#LMa=l^;Wi%0ukcj<) zl0d&lI#pjuY=xDd!xV z$iBH;ifvg5guD`zS)tnlj|>B{RueCdgCzygY7@e@u^u*hC5D_4R~`vtB$Bs&d-@#h zPW{=eu{nZJ0OntO8DbhscCN^-+=fPK;ed2qntyqz)jy&7T0zNPDhY>r*1P;(ibJ_L6k; zQk2J#zc0In^sZ64>~N~(osdC_giw&1@)$H5$5{>(>Ky4?*KB5uUZ(I1rMW?Hs;6=C z1-ZWIP~H*4NBD5+&)$bJK{R?V7&lAj1Veck`Q(y9lUZYy84#GlxwJS9Q7O`TwCufw zS6aMLGBtX`>!t4RXqYH?)*&~}yrk`yd*mhVRYFdFE1$RrqnPZhKXD=M(}mNE=gv&N zl=O~apQ9&@Yu^nS<=HG8)OS^159`a1+OQTCi7fsw`D-HYDUpAkQABQ1yQg zWA3q%wCK96(OT~Sy;MWk#a2A7{vdfdZt=82o4?^pKa2Y5f(t?4Dmt-(V<; z%zo;C07Y6mwWqp7R0o(&ShTKnUhZZIT0lW6kextWrF28P^>v6*9 zoiMw9rzONwdELD96-w_`#&;%z^8Ku*Br1HXP*D?0Nn?jgPW>Z{6s)~EZ|YOZ(C_?z8(y(8 zbjn7pQ^~+lY5=EUtLjw}jsQzn@jO9@K85|ug)FJWff|JNZG9;ZBJFNV>@iJvm9J6{xqpXqI^zQh-i>4u zR0d+?p+Wd(L08$wZF6Lp1B(2O&epwFEtw@3DT>lBqlZ)Gy7>RVsQS>$a=qLEqwVT| zBB!jqiYLkuOv6F37D~e&^9-F}b9>t$L3{ng=-|?`$0C@ofiEheuh@kAEjj@}X9fVB zqK?CPrC;zC^pNUU&-%%JqntnSS*!-YXT|F2|KtIKXOQzltkSME zQ%b*LIv{>A2g^oHKTmaeEt4?71E`9$D*Q867G98@GgCD^l7iP*ZCi7=8S7ZGfftnb zdMM%7(Qr!b@u!!U5Q|?6pcEGdtKFHvQMVa_tI*Lmmp`98O#>$#D)X3t5)LlFF6O`5eWNTpw>PlqB;0xGTz-d$*=M9`>%I$1)nBGPBlvB%*I8YG~3ZF*e)^ygnyn?1%u z-8P#8WRlt%uj|%g;cC%ojJL#iv_Ynz!s@9uZ@-yc* zk{f!zK0Fi%(IB^BW6LXlDhQ=AAD%R}-6Pji%d4EvDgUkWcU*C^7PX=JYVrwq2(y|AEmXnS> zAthXwYcM;jljId?F}x;`RO0ZXPMetB^%+g^_2~{BBUiS)sQT*NRQ8?3R6@MjytL=BdrIn8&9T@bV0U12YN?W6$q;M?wNz9;Y0_e$X=ZEn=w@?H%R z~(B4O55~JBM>3=HX2@xq6PWuwQ4%) z1?{uxM?b&BWM3C)F-hT7D*a_!Lz49wQs=&ROhP)L`pURG@^g7rOWJ_(Nem~~cTT2k zI?3P9i}18#V@wiQf<)Hj_>W~cf#E*k7BG?CkKs0y;pf99xGu3s`w^LKUn+f9V>HwR%l)-?{8?i2P`A(k(51T47y3>NFo-@weue@n@qHB4?P`^!*-9+2cgxFJsx z6kH3oAY{6KTZ8fn!rc4ULTOJl>|er!L{bqn^-_vn6hQ`>f8P!&MHnQ|OBqAIFOlai znwo-yTUs-y;m_9ke(9pKy=;fr@5OjZNP`eXS&6(H`0D@#7tIC&&Oq3F>z7_Vna`Bk zU;*gvPqF#4ASV#hzCSD?`QuHyyTOb4<vTd4}vaZ&CeO z5H+ZoQ{STWA7O>B7`$k5w(O5T3tBpNPNXu$nD)<>O9o3Gv7YGp|LSvDP%~=&{a?w?Vo5x z-=5`j6q$5!g57(F6NJKpVPDVkPvAk^-~9}8e+VsTh1;X9Q2@tosFzu83EWhCr`~yp zMRmAS)aW0Pw8!emZ0u>h3;u1%p?sL6j28rL38Vtx76`45EQU>QS&o73tC!$$w7V7( zj+DyY708YUM3NPJCf9I$TP@&Ql$BmhK!?ukfY|ofj-^*>dbkrP*t&@#mL(6}gh!9N%d!EKA`)uuPP`w-F22{xfX6kG&^ zq(ioI{|g-#U!C2oXAtEB1eU>xGRO`xXmzdILq=id&KPMD(PFvr5PBFuh*nz2xm1k4 z>rZuvkQ=}n%r1kM*N`?UEIe{OI@o18LA0jBGVa_SN7XzT?bcH_ZW|4$GQfjOZKsI1>qI=`gIYKn?NVtu5;CN0o4t=k|r z833>YGz3yM=kK|9#qwuU*|l?@h@2Q#0!<~MIW!<-AjG9LHtSI>;$p1jFeC#*Oh#Du zWT>~bOY&aNe@v|V{*vlUBt4-R?Oz)n|5*bnCH_XV5N*q9uwaySROOyu>j`bNa$7W{ zWgFXR63C*Vba|I>J&akI6Kt`AF~|n8Z;Anp83yfX#6;C@q?&J^rb|3rG{FEv3I=Rm z`l{yDyhyK#G1SX$jwTYtGXH2w;w>{f%B_O|Paevb1`+$}OuPEsTKdGfZSL zG&c3P>a!1*!wHlZO_W2a+C7sD>!_R$4d+uWt%H%* z>we92PYOnnI!~<@LZG(-q@UTz?0|cJyeapMNOlP5%xz%HV-_9SauZUd5rFvT1bZUg zhs&z}bpD(xKDQx19#XUn;Ca6?T>faa8RVWlj_M#c96}r)Zd<$jB>5*q{%84g#hZ4v zQ2s3R$BCp&h^Ej(o&Uqobgmd|g*N@)_5W+1zvsmNqw!GZqOJGOw*dZ~WB+U0|2Nw9 zf9?6d*!BNetoCzE-&mjNf&m|Ch9fojm;wc#=>w+=hJR86p7o@cW>v;Rk|yWU6FH&1 zoF`v{a0DBQAgL%~*s3bGJzlWrWGg@L+rDF+4GhqeLZ z#X4xX6^hDjn0_0-pbPO34r>ZBL0@w(23%^73>`lek zzD&EJ-<=BQE?#rS)0_Vh68xjA?$8qQCS-ju1oU6TSD>Xng75<5SXLrPK+b1kH}gGE zyM5_m>YLwofUXVrbFks@XW3Y-Me!FILTW>>_ZSOpf7de4OwF=k{CM9x61-VT@gRdy z^1GTp(K0?-A`0mYWD_8us=Od2pqodmjDPG8j@)yQt(!R^hEG(M1>pTG9I?ejbO?-9 ze}kA|ZKFX%q*fXPt`QrvUzqmA*4uge$AIro&q#;u59$XO4|F3*5K!8~PhSMMH<7Id zS=f`ESD6Kw8<{>fPeitV=7#(Td-K_47O`B0;FBEc(LIH0v>ry+Fp%A>5CC=!Y5ktb zNk~3JzAXNJn=I4#r9z}H?{4eSZVYfQ4|iPO+615fXiMU$dx9CvI0GjZQ05Tuo2b6I zZ^7pmtMUtx)IE&nFkH(v^An7&gsTKJfxuT3N^_Oq4ylZhQXrA4JYCOF>IHZq>d=WZ z=_inf0kdd68Z1lWf{taII#}<5fflpvmqw3@{b+jLOs&}ozB6F19vDGG52@TPKy}ZS zNF6Qx;!$_d_O@`ICqTcgiERv%qQO#x-(gYh&2RI{PdQG;_}bU|+JB=;9hC$@@}42aGLufT1!g{gaWLLJbkEST$ri!vcNwX$dQCi;4LD z5NmWRnV!)8M)aODYS(pOJQG_GwO4q$!(}PeUsS(|C}A}I#d3;D5EVa~Nax85F`=s7 zUp-1m>v-T>EEY2Y`X3NcctIlE;NuHQEYLR^poFy2{ylP%Nbls!CV9{|2_6=^020_q zQ&Pq|9P*q%&X61+UGxK1TfHZa(MyH~6}QbRXs%zTNBXu8*f^aTlfY(~1AF6g#o@`# z{)N9lKTk%!(pd3KhiBGjk}UIY7@ZquXh)0^1DL$WlYU9d_^Ks@#QRdDg>=4ho}r03 zI|-Q)V*D9uZdO)N0VfQ0rI7@7k`0a007s7|^t}cQa1HAa%eOLE_o_0?+*z((&$`I4 z?q<}`=gR;k+FB;>94cLr`+w59hq_5{FIm6$3ejB2lntTH4aRri3khVv-a03hZ_vRh zt;KUN1=Pv;oyuIi-fc}vnC(aoGpN4__y&Sl*m62}8`(vv7I2FKD)ArYw3VN_n2DOXeMNU4i!Qb3O%9`Hz>wr4R z*n~K`Q|k?ZZ4CQ~Oqm@D-b28InkCyT2COL9K2>FoE|GPG-F6c(RnhaMy(VS!|Ku(I z5E+jN&e^l9v`m^>1AE4ac~zihPLbm?ki;#sK1>38)zZ1(@1~1Odl@uI!?8OsCPsA z>!UUYbn=P7r;ma-8_oVJxjSZtst8nXHt16lg2xBz7!b_$*P_3*FF*lt?Ul0IdoW`U zjsq8Ae-gk#%*@u6JBosm^yNpDL zP?>jxN@iAOg>16-YS1#u%)UkT-ei=bjNF;oO4e=fdGnr^p6B;G@xK4P|GdZhbR2bb zG_LD=edl?epYe^>?0_x<0bUAaOxUg#Yih#i)ge4RsEDYt--u8m6JH$+V}Y~6VA$Gg zLt8KdP68fz(iYo8Ub}~Wzw7mQx$%$Bc`G#$bRT=9Ad_?!h9lJ4V?vwF=gA+~8nO<7 zT?q5)twM|z$N>V(eztTF>O;=mfB>riKeoCS{%_LWQ_wRkY|*f>3xfWqr4nGl_8~Lv z?1evA!pO!MpN&j1goL%jMMLayzRZT5-Eojw`xV?ELuaY#OE%RK!tCMD^!4A$5@fzC z+WGLXQ6r?hON$t(bXg4hU<#95RC1A_6>?%N+VO<%MkWLvqLK*$$6J%b0r(6@PTw8! z5ztZ{mJGuXGV3IKr%`5M2j{~Bhs}J=e;YLY>-3F}Bei$E+7POtcPk5gbDkL>0taLZ zT?4-v{lnImem-HTM*!xgnQzS%I*hv}HfBG544J5c1a6VvZln?Wir=~pq8R1Frk7y+ z8aUH4RG_4FBQVMG0g1+yKc~$(iv7Z3i938O*py6-5NSl%wP5y80UCEBK)ISYJUOfGS&+u~)Kj7hJjDg8 z(B{`m*)Te*5;VR0?E4+Nt{KH1$ZZ)I$1k56V4e?xWEBgU{yYM}&%` zX6N*am>V%KlM4(WAQ#O+`28=A^Ce_w*X^d`;^8N?`qW|2QVrPjY`wz<5T#-y?Qab@ z_DRw?|Es9_2F)2H2PyU(X->C}_uFM9{g)se;=5pmh8g&$E^qu z(c&o^OF+&aX|v(^23}r7DvQW$P6%0OURvq3Lr%A@>7wV87KFQ2<~e4<^+dlqMBtSN z!m1sQDdrdPOZo1AvC%&zPcw%pjXHyQ`*WWmwUwC{5oDbB1yC~pcO z(kO^6wV81=LQz74kTsK6Kuoq0w2a}%cei8d?GSPmLX1JY+fMZq_ZNTLibWFA`BfP% zwW09Q&?G(md*oX7j1jfGE(4+WR}0pR#ul1abn&7Qp_3TvxzYC%^%)YIxNU+(%*hSh z9hs?(hn`;kJVE<>d`_P8`2Ngj>vNXiBO>3%bp7F+6A$Zi*v*L&{o}}}cYv_t5}xiS zL>w&y-Ho6$p>~Vv2IzQ`4V~qB*k~Le0SZUlG1Ght=sgy{89^LuUVcjuJ&w@)cXT~7 zM)++*0U}vQa7RlS%*fi5*GMmrDax06>w3?{VfqW3Ul8!A_KXr$h~cl4_eS3zoHRgi zFoE*w$>amv5fc-0_o`m#ac^salct*WX%^J&FT^)ua2BsetMGL38B3X&&1xO10Wxh( znwW$xwsQau7UM7&rrJQ<8T-x_*!dcEwuY4fNb>K>*#PzfHxcp|Qka?FF?bk45{a|K zH(H`t>P&lBSbIiK%LPQmSp>(G8D9&xUpE;Fae9B2&BK0qM9gxXY-v4Fps7JE)ct<2 zhfrj+_8c*V<^%lps^+l4xs4%o)c7{tlB`$f6>n5C@u3cTEdRay!k2bF2vYF-Zuk}bH7t+6gKn+?^powjAbZuQ5EKV)au|{~pHI;FH|Xxk6Ui6k{xWbv+b(!sd_V3@np;duX zKd3!HX;)lPZGL1dKe^ytm}JkN+E(S;Ba@}WXk+%3IyvpTodqq?F48rQPv?Y|eEMj1 zfnO*uO6U1cro;MGE>d1)iEN(S1NwJlUY|338dpmksZ<>l#-1S{y&IDg25Oc4WzfU&iD@pBgaD{26~EzBmx7eYk$paCbuuAo8jc@88|kkxKC1N1<Jjb*9 zyB@)N{~G{^q5uCIfZHZ@)^2}IpfO7y8usS1=Z%}ErdoxT)_mn-ZpjEO9pw=HJz+4d zFi&ESIf(Ae^I2NLO2_e-;bJeB5kW?y+f!Rb$}3@t0d$(3lV-nXs!ssnA=B~yR^)#x za%X==YEl2kE%COAe#jaUl}l{R1@3Tzz==Y683-tOE!VlYRYCwbNZ5&h5&La~gM&B& zG=Nuk{k){lm>92tF?u2FB~bv8G$QGW`i0I*K5}t>c`|+ zg7BX=*E_3t6d`6j)uXv5$~WOc;#@>Pz2oCuLoV?Ssg59(`JH-x@qnjW7nFvNaSqS{ zJAWQbTq{L!i><@pFs(Hvw)a@iX2+z9i6R06D+uin2nX>g2~7&qO4|v)k-8`7akozy z(evHC(oSUv5?Uin{N?BGtaqmjsCY3Sy{R8_kZ6R15+#I~ioGC&4PE>BC5v3Y8#iM6 zo^F819EBT^s%8jSQonOqsD~166z4fWhpbNUn{1i?I3Bd|l|i7Sk-H-DhhW7*FJWBZ z``^qdTDjXw5EBygrBH4k9;4OT+J5;M3=^q78{gE9H($z&FHB~@23!d^nU7F zw_hk&YptBAZ%Oy58+`vTeGt9mb^GSnCX&c-4eJ^9EHWYn)%E0_vk$Qq^ylroEy~8< z4nyMP+El0i>yh03Pa$8<*IyqB8zf#IM<>6~%xz>4&5zUTdsUh-etskz&0ASwNH(1U z)gJBJjX7OgA{gn4B$*eSyn*hzG%K{|Z6C1Db$lDBgMKG! z=)Cg}$m0Z4&h+3R9R6_5EDeIdLk1@xAa|L4Poz!q`pflJDwY~8y@pq7C~*K+!&w4F z^Mo4_qn&N*8HM<)w0emEiN+zW;6qRG-*S9OWadQ6O%lrUhxy(#x=zML7}~~m=}IPj zLO~;mU{lyx4hrSVSE=a=s*NJ0PDy#6;v;|X;(>Sc_STfwM%IR2fA~tTrcS@dj8nEl zX{-3?^ui_Hjmd@RiWP)fl?gRS&pskrrAtA1!{w6-8z0q~H2uQez4AEb!v!@B?PdhBs=h{cb_I>9B*j$RdH4}4?M!ddk#S2;Br6_|oRFcup#dZ$ zrjAh7ju;ui=xPfzCYg6;Z3UdA6SBw|3Y6&gN}lGV#U}W<-sd=%&>6WQRh-%+zkb$T zuh>erRm#q54BgL`%NlyYrZ4a68nFp^U4HmNPtn6rk)mxnq6VL(Q%Vc_FOvPLQtZtY z-+p}E^=U-63|qyU@2jt94QG(`S5NBItxmcP+1w%RJq$Y>8j_U4&AP!^L^ zxsO=TlUTdR?8Dnnf%@YY72J|o{0cm1<}w{@rKsrwH)C*D0p zwfjR3eq=>ev%Ia4zIEi!m%!6P_u&Pq-Y}H>-P4bFcn=f9Qn-?sLD@yx+8rhQO|5pIw5SiaR$XdWJ8Q+vV@$O76E9hoa< zz8c4Q+&gdlwE#-YJzrx)rhpe;d{W9+v)plN`AY-Qo-@STdp{XS7!emcA=_V9C%W>J zL&L*+pz3W1jvgiCCcmb`z<;6@(08(l-qQS~nSlW!iCTX;V>r*7n(E#5Z2hQeQu+ir}ju zx^BCmt{HS;PcxPB@Cj#r*&Gld=2jE~ zw-LR5r(0&9Tj93S-CbzD0^gm)9Tc)WlE3uNgQb~+Fey!LypmxTZhbkcG|R{c7Qwlu zdI3U~-G)I&SCte4r+SKw#YuTPvh}SY337He_outCj0dJWj)vl`&p);wB6(nfyBUivv$rX39rib)SGo)j5QMG7p8Jz=)ylKwb828N*A@#c6O`1saI_!Z5_lJsdtXF zFWyxV-+x`<_D~wz^u-+Xuc8fZR23wo&=ALW%YFR%tg|0+ppn%Ve=jNQwQ5j z@B;A0odUm-`<3j=lEH6F*WkrHNQzw;KBIsNEO`~M)>r0`PQmhh;28Dhb_=9}_cFB# z^FBSN(XR5uxpx&=bp3QQEHL?YH!jbtq~R5t9DiJTr!PK>k&)5s=0!=bUKW`bvkMO% z@!D(9EKLrQOFW^T$#*3(z1ssifShkKZl+Nn)z z8w>03=}=$qtyu?YMZ5Po19Ap&`JpiJP!vklQgvRsgBTm;a&C(G*%wGar4)@{_A`57 zUkt&0My2SDRa8;hARhTqM=&LI1pJ@}?l2poJ6$I1^`4MMvo3&P3M%cPxQ&7B<@e-* zTA;(=*Dkq_?gMST_{D^S{hgJ|h1UrfiSXa&{WI79`kj&}z8~u#dQOeR@g0b|Dn;r+ zFEcf&hY5W>KX2=0-i?u#9>9T{P!3$#w#t(pBkYhj-B-R*UPRq&bz6>16(#2PRcuA4 zjE{+fwSfhq&Lbpa^|8LyLe~AgPn_U7vq>FsSjz z5zpw+i-c`Pn!0t8Efkbu0=~i`X95}Tj2L>v#VE#^-^2Cx9Pl%;UBEik9L&h!8CE8l zsNeFa=vXNev@tqU>|P&Uez>b)*s8d)m4E#(&Xc29ywezM;i`zF2hXIL%ct9fhCAx+ zh)7kWY=#b#4)_h%*t&M)o-AiP?6Q>M7e?GLek(OkSS!xl!s+vym6N7oU3HM!4Dwl|LFf-pG!@K0^}LqB}LdjrTmkX%rwVTvJx%x|s;LPfe2Q zuRe%Vnudv8s5F|bSJCa2?J7V|^MORQSpo=MTbY$^AlI4%Qf?t6s_PCObI`HqS3ujpr>UiHjtbKN8u|{{ zoi4BKd;mu80shf#E{&{37cgi$%*Dl(-)rZ?v^DEj1T<=riB7s0Ch&Y~1E(>jsRi9> zBH;iXa97)iCfWo)-Gzx*IH`02l-^>ehi$omKFYSUWd^i|HKi_CKDi?1I7`C+~XutST%wWkRxAO>b*SRB+Mr)E9|AGkbGw!O7s z31%*iL;e@KyNu&3xwQ&%U{S9vO{{^RaD9duIQoe{p%gdyo`X((7n%egy;eUCCBsF$ z;QDz(uea%s%x+I)T|POkH-JSam@I+z?FWJA&9^s*^%symlRC*8zdWs3Z5yjTPhv4f zcE+jK#&3pj4oGzEh%?3N?G@+>qJgWbHR{C?8Wl0Rfz#wy-JT3FjY^`To684!?+UvvZ&=_<;fT|D#b8Ppc=_6;jr zY|y5yZW$9` zAxLw^bsE*9s|T-*;=9tDs6^@43CCHcVtH%r7irOA;fMS};Kcn=UmZxV`z+&cQ!wfGl%sH~VKcN;l^w-L>)nNw{YiT{ISGp%1^NVWMoRvw-1N}|VsiH|+r&erE5 zU3Z9(RmPuoIGfVfh}s>a_GvQFAUJ@aUSGEiT=cjMmC26~8n-!TaSoyl{3=QNa%)KJ zMO76qCqb$ElzSr_4a`af_&;<|?+JBh-GRaEo^4?{3+A4@Ub!k_Ly}^d20mZuAozq^ zIS8M&CV~}F*F$pgF}DD&6oyyb{kZidQtR{0B(}+yXg;3}nwP$~ojNIs>dE$V-NeC` zB3~+#cND3wpc9{uIZ3YEl{RfBl-DR%Elr`ksnSd)DAXR`qyW%#boL=?dgaanfoozy z_K$h@D>zQ~jz8o|Qw?nsQ>f9ARFOKrIHvSaM>o`MWtyQS&2;8iy&4_UjuN z38dh|!R7-Ya`(rlR33xcQtQXKwL->C(JM^uvp(ee4D}s?6_{ShxcHj3Gi~3JmQP4fo0^txLkP9~w~OL} zN@_>a@v;^}v)w-tUEhon-6HPEv{YANOS5u>=oJ;;wx8NUY4EZ`&Ind%@Rr!ckhbG$ zeJQoK4~p(*YLCL=XBb?KOdR-j(VUyg)zSo)HgVKmS&?#SnN*CwK+nOBj0*EO-5u+} zZ`#7T9W2aaa{R@}(7|i|3x15QpDQ=|W~Phh(hgWJ^P$9at|E&el{iH{V{P4@C*BWh zy7W;UqX{dQ#ChyuP|Sx?@9tlBK42zixZE;JQF+Z=A4;n~JSLq2y3-tuvJ^_EfoqI0 zfW}qb8pV+QRx!?swl*sfeVU~5SMUcRKLhUo-e{s zZq~{wnKrf#j}o6^Zk=B2ZAXv#wki5}`M0(W9-{kTkV z-Nz`Ag=dD3Ek&c8{Ky#G70KkkryfXpK%8cM)LESz)jleG?@{8qj@V|~d4)PWvBlTe z`APS*Vuu$PSJh^-6T>%gndoyD`s{@ssKDK{ew@& z$^oeP4p$YDmV;v{?w!JVMp`18tE9(O`;sOpb5g2DVvFryw$o|W?SNEHf)D{_7gG-Y-LqNh3ZKu3fRPO za;r(2{Tfl|PhGpXh&e~CcEyZ?%V>Z>Iy5QagH|!;k*0if{WS2EE1J^>$dvmr@k{!f zD@>Uh{#s&-TA|T;=FUyv%R0dXL0Xd@Yj)>))8vw0t-T)CNn~fEDD}n~>6W(Mml1Ro z{B=c)RdprZs2Tr#b(**$%S7D1#T6Na;c}7eIiz33u zUYS+y6B9)`caE*?sGzPN_|A#Q};yldE)@4@???yDL+ZC_4T(6l3hC*>?zw^ zdhO-iP&sn7EbPeJ+Vq5$(BWoX@r?>jbBD}U;Q}6c*JM;c*^Tr(Hl&39i8EeUvv?R$ z3*cst^#$k@o45Z;j4YMttye5QyE(4tg>_RNQ!g}ogbjmRcM)OX@_XC%24TmsiwVrn zuy@-x%9cu|CllfyO;fz_U4%;(hOXrkG}BHe_}V(6L>Jk=9KM%g>{tKhP5|~&ig$Sy zzV88B@eR4l^nLy}hROXM1hIu#y)HNAZ}GYyInnX`0xl_5Y*}|$NW4>GJ1A%h=hS4= zjURQ_0-bTr(yhHbE!e^4Ce3vR4c@+Cx%VcA`BZVKqQd0RMP&qG0cV71E&?=zyY#^7 zRlIxYd(tJL8+JVLdaP;X4%6{Vj}jLS#E=>?$$oHB6Ps8OxFK7^?Lhx6NU=f#6T9G@ z)<&{gDx%DCGDxf1KUo28hO}nO647{Uq*HU==3Dni!c&wQF05Ja!%KSj*?5k>D(Tvw zx^P+fXPLz5EZgh7n+&gwbP4!u@>fr1*+ki6+s@edfBm>La<#X;@&;q4x4aID&88TA zz4t}q({k)+=Y$noM2XGAt(9I&QYsBJYu25pk|*~^ew1;X=81oKOT+m`Sryy#^x3DR zX{=cv=bbs#@^s=cWu8&2u2Y2}73yR&V)7%Q z8l}?`#%tyQ{Zn&_(>x|IsoOQVs5-y;SEL!Ic{t)q((PXx+0)xn*=U4`*-*8w4Mhjo zlztKXK;vl$)?dT}YU{o^Sm?Y^a-3cb04uUs&%QC>WDiVSd>LXS!oVl>(!+{KZ?(@?^Vg*Q;G0{F7~~c+z;~ZF_&mCe4lhF zRs+(m9XI7`|CmP)ADllx0UKKnSN0NZ$8Y)H7Sak7=KuiWmzCu1?5Az+- z&-G{(S(qYph{w%bZr}`QG0~Fr9CGT+ylgIaAxL(ij;sB++4=AC#wT8A^6Cno_LH60 zqx_G2FL%mDyqVut`?=~)LaH8a33VZ%J0JFg8RYBnLX}snMdqD6J*9TZUT~*Oo!8su zV(S5`C?U&vm!*o1RKI!inDw*!r2>Sp9OjUN=_hX~S1L;2+IZ8AD>PKPK8$MVO?&C(hSOTrj~@11rR$e7V~^6Ux>I7QF$op~+vtibY5~3ha(!O#5F8Sw=zx{I%a> zMc2Ibt-ly3g10uHmfeAfk)YBXBOiV7Tde20DOi{T8KoZ5BopvTj}}`Ud?>C67YbQ` zhT9P;7DyS~An9%I?->$;Oqf^wI1AC=gW@L>514!<(I+5xOhp(3|I<|Et^9_#jW;_f9(>iMoqX6D@msm*Vy5$3xY;-O>NSJu9=tQ7zF zZ%EaK1b|e#DrK|ZYbfE^*!SEwrRY_>IK%%smVnJ+?Wser*Lp7bEe~&S+HAPSL+@Yq~S%W@zDU@>1vVn}18osUu zP&qz*-u{Z68IWaaVZxeZS8bI&);mUWKN_T1;!4V)J`bqTbuw0rO$}%?fBl<*W_b{( zE>=i@RExMOz!SIL1;s)4`)n6@Xa0iQ<40hPrL1i{q5O?LD}~^Txh?O27$6I{2ix*v zrm%#=YPY2-gaW`~5-1fBU>$WS9~W6Ag?CttYgmC(3uqMSy5;kix&&=f8A%d3X3_7u zOhJ?G8PSj+{d|+bP+(Kq=pxXO^77|s|Jqzm9Z7j7O>ynV8TP++gpXYDKR8J2UoeVw zlj{=W9V9#t?o=0nJ^{R-w`h(suIIj0CZ}%piAh%oP#3_>SFu%eawRCGR4X5 zImi1bHEK)*N>lOa%gUdR?EBWsK$2tf+28e%$rIAQ(fyy)DbA)WUoGR?{@TeX(B7`L zCLD4z{T1iMgbGfq>)`o;R^eN1Ad@wyS1Sg}U}|;2r7eSQ4r3c2VeSHjA9kkR7IX}1 zT=62=CTmSm7dFasIj*+6bFnK2FZIotaRole^q26CE}i?few6v-Ut29I8oy9SL&@qK z(`e- literal 42769 zcmb4~bzD^K_O}t~MmmQOPBx z&N=_Q@EO>%_uPBmx$d>D^_@3rDzcbpq-bz(aG3IPQW|h@2+eSC@DLOv;0%aZbRP~5 zt=L9VQcYe`l1k0h$n21A|_u-=!2le+`PS5n#GueX9 zrv0+s`%TO|di8Ob@XeShA{<93|FW|vo1VRa5GF7Ds2E;6975@Ix;-u}7Z(Z@59j^Q z`E%O9h@aKz(Wm}sx3|2(uqu$Pv`ij2!}`!IZ;ZnQ z>w`B=iK7Z9;Dy_gmo5Obz_}7ZBoY*i)4CX*RcIv*^vC3lcbEteszJF@nAAeL2Yfar1f3S zZ9KYTqE+uc*EeI57I)3=qA8gfWYqd;ubb09+eunjCHG^YWSHV0(7Y0MM#5MO7ou?W z_Thh@W?~tc_y+Oj13BsAMjIhg#jjPVil1xiJm5@@=2hAVo%WS}%E}RlmayF8!Cf;; zy&SSte+j3;Ca&RX#hn`t_f`L~vnjVM&2YxV=zDAoDtMo446avL>~Q(*Pp05y7tw@* zu~Fctsh)HNcfWe9A%=wWI?n`A7adI;JcZ^N;#vtJN8nyW;lt8vw{d=C@Q7E8cjNJS zFh0g>A`|>(WIP;%B`P$jFfJ;?uPD{8?5PEQVO@%etMTC@(@FTtp~z_{gY^S;zxH-Xc_p z^nG&`(ets$nsnyLZseJ)Y!3eh^9kb#y*vLGq@U5Pp{-$*QgI*K%$^amL?iYf!aBRm zE|T+-C6u1$qkRu;jNXJ{n9rM6>1vgtPM~hb#lZ5-lFWHc&&|lH)~kf4sCS65!oRn_ zSe$dF=Hp>S)`^&n5sVfL-|og-5m}V?Hf&~@Cj!M0El(~f9(r7SzA(Dcyg?8THj#3s z4#jB*wFqSil?>(SRPJOq4Jnl4qW*$Q`=+R?wu{5;@gc|IqeGTAQezDK?>pt-$4E6u zHlJ-W+Qgr{EtTS7;Os+viR#1InChSDpLCNbkwdI0^!)GxdB3Fs%PP$(?&@oDtzm^& zy50nR79HwI=1Hb=g-j}^rz7#k%vo&4j6o82`8SG)At6=pn!#n23ON!>A39%kCevtr zlL1SCC0*W|Wh`@dTPg6-nbU#j$~A(=#PgyQBq8|Yulhf^7H$`X7bX?K`gJYvED~Ld zU8|08S6Nrp3Frv?608%163`Mj88x&%6jw1SavOR?ZMlpGj;q?v*kW*J&dAgv)f&~h zobqpFZV8>zoXVVvoT`#?VS-|~VjM_WNU^!UrLLu#rYdru+Zavw^UM0gZj%qEf8_67 zpZgwCJ6PLUn`lSzW9tX$_la+3-wAE@Ct3?{HDtAVH5s(5!iQgE;2IW}H&9yS^-$xsyx za(8j>bK&1RN-D{-OSQ|`{(MP)>~oAg1akfEW!^;L{Qk@LIlP}io=LtNK9$0Z!jyib z-mbzWLRlPx#6{7tXdR;oBeM$n3YUCzH@f3y-u~HgTiGmed#31n@mSQyE%V9;MFk}0ZzodMMWs44d{Wzo^Syb1g zMyw&54k5HsqApwrgBx`S?UoyZjF40t_w4B$y}z`#ECRQgyp&n`BX8PJ9XLgpf2mJxEy{v)H*ar4E#SvlZ|0n|J}b{CA+IuN(Io2^v9(GHk+c z3z8WIw+$>??N_P`E?Y0#m}S}4nVpPt>dkf|I})N3khR52-f7oX@aP&=`g+GKq^8z( z)!Ntv>07mwzx>&B3f`VYt3jW@$GBZQmkh`;$Z2pkdA$w275x~5e=}A4&2eQf#yF}$ zdNQgkN-;{*Y1#hE-TNU`3(PK5xFG2OsZinEcK{g+AC`kO)B_suH~)8wfAl2v&+bZ(cb)gK;|{+ zncY6y{Qag?$A}l!1%-7vS2?qAgmALoL0`^>Bt_e3P*K@fDP#GS21{w;MChbJu}5)P z+4tgE4eUw9Em9wC%dMTB74$VX$N3p5C``U!=x5#5CC<0jmZ+D0C>q(tT z7N*jhd-EP%6X6a|^?Y7)4W+&67*46qt2Gnu2bFzv zi6@n&z$+7LIP#PVVHHtj@r#CT$?P?z+lK0_IK*tfnuW%q&>F z9e@kKkC3+jaOhy+Zc63tVDIQA;4Mt^`x64d@xy608miwPakmqu(N$KXl5}#lpyFX= zXJw}mL8GFg5^{ZMDWD-G^H*`;oiL5HySuXh8=IGx7poU1tCOo08I zTS?u2mjrWwIsRVsZuYy8s4^97DSN!Sc-)Di2 z7C{qY``4g}puJ<5Q`H9$IF#fadFad{}8Wkr*KSVLRfuNocO**p9 zI(YGe#9ImdIm5^r{BZnw3G_UCrt zxUN4xXgxk@o7mI-s#p0u`xQLev;X)rDv$8R>)px$JHDFOBgFsw69JE8WQ6ixg@Aix zD73eiXlAHj%lvmKwf6P_yZ>=FwQNmPVsDS)@0%U}S&fHU5qtY!{;%`yjE^2ysCWnn z6aQEJfI2^BtN0%^5DQkJGg%xjQYk>r{U5D1SzP7*kB<2LcC-iqqgh~l^Tgyod*Y#Y zFq)f@{?~bdgjc~j1p*nA(f{4Q*<#qJf68oZ!Q;^@q2$WOVc{{TU@QdP2PB(9NhC%x zg(b5@0y@dv2ESSO$i$M9^u&Qcght*=k;ML|<2pHh=X<7WDOQe^w|(y!!oB8w7zTyU z=f3+-m~@~uUo1zl!3Pz2Y)`P9Y<#0V*`8#>Amw^yM7H^PGa#MALg8ZP^GIid%L@F+ zbg%#M5N}J`?$LJ8^=|#rc12t4f|WsIkMH>&g=LCG;%HmYeLMf|V9dVCLTkXvc>yP= zpf~?__lYS2Gl2#5vhksXVg-bYy@x;@@XDL^z8v&1WOaf1Z*-r|QEglvXWh>g#kDoQ z7ThlV+_@4bN_hvI;;Pr3&N6iWCJp{|IbIPsZbB9+CVF=m2TQTaB&nOWOhd}jy6O%i zG9r!@+-qRJzd7#B4y_EOvTa;VL=UI%)YS#*H&cd=fY)|+S^+}_`QtQT@y>g%92AJ& z7X7SSAmEjf^8*^yyMJ=M-v)X-OJbD{W^PyvF@<6BVhHZmXck~sc}-g+RbFkC^zN(b zG$RRF{+UkifWdorp#Ct4vK7(hJZBwDD_YLb1a~SWr~LL?U_k#-iU2h_I2vFJZ`yBd zQcP9K5a5OiBU8AmO_|3DGg`9yUmOrKCn6a^#(%o>Qh?oyr-t>X4!9=`@R&jhTYWcj z!aHpWG6?Uk7w)0GlzzQy8?Hv7pMl7s`<3&*7|Mm1>iJA-o@rusLV!Ic;&s zC%6LkRr<4HYoviS%A&_w@-T#>n+U8eCu6#|he>TWc^%HR%Dg|hZq|6)Y@C}b+U|TY ztPAh@>&K|2%zL6yA*ncK50fPrGu?(!82Ih1@6c-Mz>PT?a`&zLI6?t09fk%?Fi4TJ$O)I;G61| zEaG6HdM-RnaC+^DA~Wv!_Y$3o8f)^cB+#DD4V7PKr4*EEOc$DMJ@!ey>fYXb;)Rt1 z&W4H10x`!MrWg?ot?g{wOc`EMxJ~h>X0c&uqR) z6U7)Rz^q81;neG4>nAC8q1&v6w56}e%I&jSE0v+UE*VNAr@eLhZdfl^DL}W2bDvgL znm=G^t&*zDDociM+TArf@8+4S4mkgOB4s`YS@Cnd+AQdezXG}t6_}vLucasTomwi_ zGyQsOznqBcXB6SDkh`{m3l>g5e$e%5;l!n;2^G&P3Hown>!r;*&PIzJkVm ztXtK+%k4f#2v(4N;7Kni0>m3Lo?av!SZIiqN zZARv%o$4@E(c7B%Ft)HAWUu#x)A!=6fYkwi_s6d&=fplIiP%6R=SX|&q?uL;; z!^^^FgjD$h$B*HWuMI))xv-?Hbn;{s~d$g9b3hoNYMQ*6C`r(e%EyYN(*org6B zBtCZL#);^CWqiJ}o@@b=%6-A}fsZv2Q6?e%h^-?-0pZ_|n7?bUe;yGLG;%wM+8+mO0{(r|Tg+gV(l1-Zh~@I-OMtCq-=!BR z|2!;k=r>#o^CF;aJvTt!ocfOD;KJ5Y&R*c6(!i>*qBmUpEEn^7P0!G-tQKuLezitU zNXLLJ;VzKCYXk3(sV$}&{c@ked3|4K|HneP%LMNrTbo#uj2ommwfbaNheuN16$V{cWEpLo5C;}`FeEb`(Yk7ZRwg=PcBnJS`Tq8t=z1Vs?L|9bb^BB895r0m4n!h^ zt4M!h|3rTYL9+5(bO(}Tiy+X>5!r?E#oP@>j&J$Owv9?e!brGxZj0hcf zDHJkCqXXvGwbU`^3ffc2)Lg`o^CdR1BJz#XuRGIBZm5o9bszoG0Sr5CfCE2%CI23o z(%Zyw6Q>&I8L5@HW~pa?C(OD15ppQAyNySJ&e?rd139v=%30Ta7v19l#2QT>X~NYMKp*wO9P zhLX-k%n6OUk2~j_E*TqcI4;*6Jnqih?kTQQT5|`FWx9#iBW+or%^GjLx^QK?+!RkK z*ZH{_1&YL0e3u0Kybhu{ha(8GU&!f~ezlGA+w_pGePJH6FYJap*2|0&J-G~Qwq)07 zi51=FOuMtPf2I&4y;_*skl`OcQLP?+TJ2LOTi{`ou6P1w8skf9ozl>+fggyf<8$LN zOBm0DuSI*RmWwZX?uDP170PMJK&)5w&&)-%Q7t4_j+}@J7eBtEtA#8{A>wJV6#Jvmi`=(@7^TS_i7=#VvGV#Y?N)nSdSIC zzBvoo7wxZWi;+f-3jyGq#7GdjcLi9P%2A`&qd~5K0gOT{HdfFV2qd>5 zR$G-qRBpA6aC@yz$VyN>`jjpTH{^MBq-YX|pl~{>ob%Q5|1<01Sliy=Upm z6(I|}YNCa851!~6@h~zF{cG0>(YkLlZeI^wj6N?WjGMtC+>Qo}?>JhtzJ|2x21|#T zYsPHlGuOO{?fW3h%r^n7%*G-u^HqugwZqBO32m$5FNJW*F_Cg-y{EDuO~r+8fbrsK zA*gT}emPp`HsM2D{LFM35un|slVi;f5^hss8dAb#qm?{DGaytD@;GIgmXS5dXt z28E?^ZJ`QrKc>vkO4LoI35bsI^JFYAwIp9SIm4cPiqnamv-aD`RGDr(H*NLlB&WGD4l;P6 zlKR3qMHaGt;`MWdisQIdqaX+2aB}0+wwPN@VL1Z4i6V7sX$u-Hdu48elyj`!W+t~Z z7fR>b>5;&RjV;K+w=wSd4T(nH*{$@jj4BNd2but)%h-IotdX&6azCD5d@PjfoJ_a# zPJqD(7R)(XAS)knozr(S{3c6(RDc%mCMHee_HjvpK#!%>SeiKNujM5U1YT&%Uu+(h zUT`dJ{Lp8Di(K3d_M2xVe5gS;E1<_6xG|u*>3dRwto3`-ix=i)?qho=xy@DQDLYLy z+;!vwT&ICENI6d=x05k4ax8A1t%b9|NH54de7@=0zZ_A>!pjF@3Ae-H#8AtN zPb+3Gpx?nim`YNk#POF1v#gGZ!t5z4Y|ae7-1Zn*X|a717VYrVV@;D}7NJuc;0-D8 zSA@|i#tFA6-SNZdrjbmEdgB);J17Fo`HbftRS1(2@G2v6K_Xn{F4op<6s|`` zs=ECy@A55DeY+ACAG7TZCX@E6RQ9i5P9!?4PulJI=ntE^&GrRBNqXD;?ec<6^kOae zC_8)nBT_}2m=Bb)EF?oDd3M-e!ayqqE-_)^7ga%PUu;Xg#F<}%(Y6#G|3BgZI+ z>*@d0_uE5-_ZnCeTZ!9_RN!CRS@BE%2}g_MG=3M@{rC*@myKs0J-N+bYBuMs_!E2z zqo|dg9QJ~Gmpnstp11n~zOlJ9=6h&C;!iEGHm|yPx9jwcF?E_|kvN^AUR`1w#O;7K zi7NJ0ENBM9YjAIl&+w;NvNjK!rjQXlOR+w^QZml{m>Ne2RbT|CJZ}Q+O?w~^?U;MW zZw|jm><1<0Y<;R>q-RkT>o^dC22_{1D^2?wfZhA|(1@9|$Bq&LZ#GqfNW+J53Ds_; zJ#bCYgE1u)($M<#zOz9&o(!@NCLV`VPm;aA!ApGUC1pQknQxa;mkyiQi^wTZY8`pG z>+f`N=S=z8C(jI}+q1eUvS-o&J5;RUs8{Md)UYVEqAK8tuPf(kwzeCNbmvagL%|`> zDeOtnN6tK6hKuyLHMW9o0eHV$cG5LlLFwkP&zr7ocQondrrGZUXq{nX1>@TKELh*V zq37GX*971j+24pH*b~S4Le!=nhV(P^PiR0P|Ej4N_vry`{N-yDvO-d`J63pB#uklS ziDh8_2)Gd+x)DMU8x;HirO#Wnz5S{tc{QA_dGa^>@dn>SnLz^V1wjtI3cN_HMq$vln-uJVDWKaGMAG5Hj8T z^qzRlHITUx*PfnSe9w)=G$#DAOM&yJbccFRieYiP5mUSRz6_ESS=+3Dvo|oql!2&r zy0kgr`RVwj`gHxhu7>-&>)ys`K2OFgl$!>0S}%>EktnS%PJ6!c;}B0b=!KkIzmV2W z__4$((Ijr!%y4LS@)LIO0*PA$Sd{8`G4FDOacQ`UqnDzC*{kJOn^vg9G$!0~(DB9m z*w*GXf#5IEzXI^r09z2G`hxn$fHH|`p@<4yf`{YOp+aEt$jA>sG|!#l`#uNTp_&aFgNL3ah=A@Ffck_CK=|MN5BGcJQPWwy43z6 z8u&wF$YaK)pu3-H!o1Ty@Q16xrBArRdBy^3)+z%@FZ33V{2<8*0R_PbAsh&DP3PFe z@g*@)!}$o$vOvW$D4os92>1;enhmrVe5T4_xodJ_*lM{R5x9P z0d6A;>65`H2&-hh3sY*t`537Jbz^qfEbEm;D=Sk+F@aCiTVsoE7L`4eOy z;ucum&sm~snzvi=|xsHPpZS6w%M2Qzo8CJ?L|mpaqF%4hR8 zKA&}JTXx>F4QSKI@`wD!mD!~5KdbLo?PsRFN~)~RbADT&KPHHQh$=o$VRtpjc<-0& z*3uuAs%8v7yrgBk4x9g9+)}CFl_Y5a-sm{NztO6-LUvV7i89Of^`9B?Y%~xa%f?<* zvHsJ)ANH>!HDuso>Ff6iX~v-Zd-^cIbrCw*o#l1ko+y)PajyTzjsNt< zI{q_!G}D`lLsI}yVz>C77WavJHp*|yMib+%Wq8e3ex-@hUDbN8IGn-wpPW_9iZJ`j z_HO+GR+j&fe6m%hkF9H=VR@%reu`z}D7Ns48ZoZ!uT?Vl*A{Gnw-TKuDr@oX~eW%rT|J|Bb@DyS|Pd-k6 z+EnnzE5WGg-{X=>RJf|vz2?g3}hRu4U#Rbx*~Gtr2wz(B6r5C^76mejY6L)%n$%OvdJzzBzC?! zRk18K`vhv)OepLpe#>2NOn(%r9|kceJGEYJ9aUmBZynW4J^y>93!!k5#J#Z;WNWE* zHX**Ji|iF=G-oE3OwLyl$?CrMT|xqFb5g8c9P0*+SEKJhnQ99n8V ztSajkv8#Q{t9|S3$NJir>o@{>~1hrmxilta{k3p+RfWV@o{kRU|#cckvD7{I| z^pCru)nvi(ig@3iPOdb|?ADKOIcx$`4HW?DtbXAS)DmcprOPb~*$b6cRtPN>mQzLO3X*L z*2-T0dav5~PLY))GwANJa?W+2*7tqr%OaQX(M(}KnF4@#^S2nLGrW=SWKd4UXVK@B z=gYwh;$n1}5}A}~3%ujaxL4x(64-2NaE2zW_VL35`SH3{4LUosS0>Dhk~ zH_j)I6bIeknF6zmO<;1{tL`0r?R;IZPT9z-Yn{LM$rrq-rD){hnS$Pt3!Ahv*;meh zhZNqpo+(%^8`05q{XjIP+`lW&)|8zYTmU;WdME=rON|(n zcc`x{1#(^0ONmZo=&tS!*kBltk1$*01jwTE&0%jiU83$sVeGb{S(>LSs8 z7H=rrN5v0?c`kl>ErjL93NYeuwE)|WDd0`m@$e~RP`L}7{?LTo9FKrT5U)*)nMx}q zgovV;tL4D2ngLQS2`rQE#X};F0Ek;!e2pC&WFvmnJS?#NDak^3uX=zL%q#>%9Rqr8 ziA(`N)0Tj4N5dAmFu`c4Gy<{>oj`@jZa4K?0r?BN{Dz%J;11th+J*t>UMV_H;))TOM#^J(=&zx_?97C%D0HP0wIQ)DeNEE5P)_Z=l=hYUqgt+}c7lJKNtWcs*npF5#KU6M9GWFoB04<8nAw34;`aTf!DvdC- z@_f=+>V}WF^vul%omGrB544+r4Z|ZBHx;K9=PNsYw3tOnSzTlf#kdRtiT@dpqk2A! zEln)>%I()%St#gc5!r}R}^KbGuoTi2(N8s z5yGNa>U4F@W4M<+oQL_>rGh5o(`xMoH3kwUs(^y1ro-klA$n8;#WS{a1lB_Ovz&_*PB^)0S5 zC!CU$6>#00_gl_;4zP}4Sc1$@%hJg)FQ|pWESXa?Zla!TkyvHFMb`xT zPnK%dtKi9#%f`BYI0b5WkBXDVq?Uw0BL3dG4z=TE7w-jX+Tqc=sG>oO%!;QtOQWa^ z={Q8xpQxTSoQx@&j~+jOsqiseZybE}Jc$S?ZbIr|r@-Sh=K+>_{F1J*)>OZzR9664 zAuKUT4v(}2OOS_zd{-3}#$gve9^^CysvSF0KV=pr7GFJSOWxL#Eiqe#nK)7ft5-xnC4w#qDb1uEo z$c$s218X(Dz|71gs(7H;#8>PXxCOvcW^)W5W#aDS>=d6vX+@#ZFZQMRur!Br0E}Kh z6(dU+#HMNYQchQ%-EBnp9RG2}%^B+H!AG>{S8pR3hVf!@a^i%JAh8^X1og-4_X0((#QGdUlA6mC44@&Qaz|dcS--UhTI7**ZL!*o57t;QP9a zASIX*>I6zyQRs^AJYd3(qPpE?$Gqw~&{~sOA6DM7o5kO~2eoV-cnsB*aKV^ja(MC( z>SE)ucvyz#%yT3y4Ec0`j=xVln|5rKR8Yd` zp?DhV-8`sM8g$Bbahcg+{56H}GzVXRC24j_oP&R>@4K02t~Cgxd&p3lFZ`W(KI}7y zUdp>zQyjlm=a(ezIpVrf<;G615Mr3hhJVu@RSZ^A0rJ4ai)s!%zK|;%ELWvLW9g}` zs&5Cg049Ijd!KcG!*BzDL@Xi$XK{>ilr>_R-j#=>(}tbMfiIB#GWY%pR>w_@E<7>w zwz7rj<;g6d{9w1>m!JHP8p&{ZgBW!e36d*a5W zilCXwIfP}Qv8^3^0b+rbc+o(2JW};X-8Twzt8tCdg_R01{m-CB;B~@;=Y!yyhuA$E zx@3m2pqN9tjYEhd0p8a^a~rGE^L=VS%S>Hr0`F;bzl0t149UJ!bwMM(`|fgJKg$Be zZbQcw!|7?Vd`#0**Iuy@Cg=$+C2qFx<*)Zkz&_$cPJapHo#~MD9`fpQ$XZ5_%U@)V zsLLUBU=aTM_LUS;>K8?iwCpg%KYe_5PqNJs^um^{`LG+?B)@#;>pmc_zVCKIBiksIbJgkM|2VHm~PYyqk6bTTgKzCBGC%$zJ!e ze}gPZFZO#RXN9)q?b~l9#qH7pk`j?vN9lhAQia8E9cZu$YoRCj>Tjf`Bv88Ir2^fL zY14h8QA|*LQP4EZ<%km$5RftOp2B^kiCpqWpwV}1v?z_2A1LT`_xgJBB~P}K0h&=o z`RK9M%TNXc@uF&a2UCYYd~|qC1u_KM`ec%}sPFi~5z^BafFvA`_W5ePbVI7sNlVo^aijuZIKF@2 zUH}gVpeZ-zf9JPSsdUe%ImsA0A7B5@HhQq5dVe+-@(){{Ek=M)Sog6%q6hm7lJ_of zlrxK<1BXE6X4JA!FE#3ohi+CsJv?%v>6l+}3h|cAV)vZvpyZDxul;2;p3Wywx_r|( z`+};(eWFMB$As#LuuAd38$5R0(g%%9)t7LS2(@SnO}+yd=#CwUB^3YUEqmlG=hN5~ z`52!%KCBR~;wefXr%N1CDO6E}M2kQ(Wv0LLS^*?vji7{~{Oh*kMHo)6OgD~})J}MI zJo_Qwdq*Ugg)&W5Ib5YX0gjrv$puJDe<$nB`ZUt@N=2T`hBQj=r^8U)&GI*pmsP4~ zy^WR&Tq0JYrIWe?#7QXAAWmxj+z z-5jb!#BsH3xz>Ix6BjX>&9ijkVoV`j7q~=YU|Jj<8Q&cxfuD9Q1CDsqv7k%1G=^?w zlQGAy_m$y3T_|E$LI>>)87J71kH97rB39RzuUJ_of6(G(xFsot2*$pV3F04C6}fzx zsu3)m3t!RLBd}?J(}TnQWDgup!i!*C3O;$aVr=$?&MX_c4#*OojGzwy$(B_j5pNEq ziWX8YZl&JDx|FN8M#o<(#mg~^Nm3q1o4%VPd2jd9_qUy>Q?{6VVUR4BkI&OUg(u#)66oQg*Urz7BW{R1W+gon^RwMj??RdY0JAX)~J_H<$ti zn^I4rBo3CyM5OB3wSKmF0h*P~)<2_Tt{KS}=+Ty|z2wlyPq|we2m3m*mC&)DmF5-K zmau-#x(jcev4|aito0{O4vwW0OTd$?y>Q8d#(9*|Zp^e8@-_)ly78?tAnni4P`1F2 z<5};uWu3;ixK~hy`Dy$uzW&!1l6?dy>k@c9vHx#Q`~`9daIEbk^bkE)=mBu^7>zGn z2U*C*^Tof){k|7fXh}t@9unYuWvf48Y9_TfRQ`t}_J0*+1U4@9k@Q>b|4{>L2mrs- z2^an#(0_hlqY$VZ#p&R*)Bn1y44{bo|9m7V?ggNVfB{bA#8-xl>$1a@-f)0e2rH#< z4oR!xreb1}jIWq5!N=noNivPI$x z(w`U|$d8cA^y?jSWaB7Jdq}J>0C9}n#%#S)?l$CV2vE}robjImpvWla0kr`7M!R}L zH|5GKfH=2|&))ozY?_9Bdyp3Tp6#T}$9CY((zDaplM8t}bFr|FcHv1B{P^OvPdOqD zNJEz!7t=qTRRrBKrF$>GSppc0q7(P!SFeytuc~fByUIaIza3maw^)DCi|j87D1GWG zR}=N255})G@a}RQ<~qPKY0`2M!k(}l07%?0IS1J#eVZsJ-FdiKd238lt=*TG35-yGPcnc^tc?|$4vjljI5eXVYh_c-Y)eL}(-Nre@rHNsJ@*LG3 zHHS|%l%?iD00}@JfkPj~dmJ+gVDAUd4Y2tEVh=)C#?5tEzi14vR4zMaBr4t!Tn6Bd|mCt+VMIo$gYv^<2>^pAh(kt^z-*({C|vLyXwDE?Zr zTT&*wwUbyn#sMC$=hce3;&cLQo;)2boLlT=(xSvzS?*;Xm%LO#MP2*|m}f+O0By#^!N8C92ffyx)F<(CrsMZvrz7r+APX zhdDpwkpimr9#oqTidS!?6Z8?FQMuw_s%`;FTH?S`G@iq{gRr`!r-Nnyx0>V*h?kH$ z^R%>I&}Gtw4xun~GNG)jVk}M8gHFwOen+V#Lt5ACS)+aLE)1aidKa zsKd#hlwC%ftKRArj`-3e%PjScc?g~10~ za#jPa|4e$^GsH_|Ys8QBGZkArh@FY&gpR(_f+4>tLo9GiyoZA$1k`5jTaJC6W0}ncoVGmTthS9#zXa?B%ma*LlpUyQt zK$*A=K-n#Fx5&V3e*gOOk~3H5Z&eYL1tW>p-<&Lz8UsedP7DB(hKDp>+&w@Vg-WY> zNL_v=-cdKuAmpf<){H4IbuX@uN@8O^Xs2~ux(Up`%8VZWCRrfqY_W^i)7XIruNxc{ zALd1paiGs4pJw|Bzyd|o)MaowFLk|sdcYoJqZmY}#78kJ1)!ko*#)x*iG$DIWXQsd zZ?|-+(2C2}Z=MX6gA|)C$>2%2_?T?$?t?<_5#F^bfmUe7@7!Q4?VvC)5a#zqV_omf zSFR6udd`RP3WiK?8USrkRE-rE^|!6n zjn9lYGt|&<*_Pt!xderwm?D^{SDH?z z72t**!PFl^QxKFP8ZJhvd`qExFH2$q?gnp|yfIF{~B^HPOg-lkaI%yRq~a1_S@o-!eQ#t20$`y z#!aH7hhh9H-MJJ}?4pv~8`)1bjOgZHg=wUj(M?MrtXmnVxE^Poy(HHlS1A_;@y)Vr z^e4b3cxiJ%r6prvenN{WoyQTK<5}haF~gnvq%Dx+?<*?&q!n z2yx1Ti)DOX9`0J+S91#y?zW44?^8B2)8GS7jlX6fh)@jK#QpIEzn~5X5qlUFjPPb2 z{0ZIKQ)8!Ytoa%n=-H8oDFzV6XetK0^wDcIMz9Oz-i-Ao zKQ{)lZ^R$Lr5{iiP)Sj{3+-vyO^p~sl;G=rY}loal;#~LO;FqCyo+9P9ZX9g?NGYo zLzMxxv$hLxuOcGF8HRG+l{N5f7P*qTt+%b3n;&Kr*;+ni@?1qQ#ZWiJaCeGA+cdgs zfrZ**8UhiZi|d)N8SsOMmN$~&DWoBizSCE>z22{a%(>bP>|O_%LV_bp1)v^iF` z%}K39&9dB!{Ojp>6`*3aOUI>ztJ!4kao0h0S#F~Bg!oiOm0?bT_m?7-cN&T;1mTJr z)3R_f?eZIm_vLwa+IXvXWja5FZIVbQWHxJ~Ld__xpzn5TN@kx1TF4t0dycU5mi^g+ zg7Lbj-6p@5u##9-;LYa%Mtzhlg}C?ek<`X4r{G=A1vYz>xUo;CHY7Kf=3`!KdS>v~ z58rZuTIk~5m9^bn*&JRZ`|Aq?+2nP+=q3%Oengfa2#IvT7<`-{+P1kDQQF%mH%aUk zr5*$5S`Hf4=oM72f4zOPN76GrBXiJOwfZ{i<_kt&US|w-9Jc65$B(XYqiUK)TP*5wtm zIaEx<{j~e}==_hk?O)xkCK&vn+EOu6W{0V$>sR4DyO>n^@ zz++x1=Mlo%3sb2t z{dFN<%JPlyp61<|WjK-3_9osB1P!|}ej!Hd$1y9{EYBEga!f4$J~Ihq zC4*L!;rZ76CmTEqJC(}7b`s!Wu;WSAF|66?;*;I8vaDg`_0Z#v^Q3#MgFsn(O!K=~ zZ-le|2%fX;A3+&7xZZ8yLRmFBy4ZOcB^#KiABmaAP;1Qf9ek4o#pDnlV}T8#Or4d! z42|+a_rTHREHPp+f-^2jf`O%v3YC{gi8gA2lM;>f8ri ztXOVf{@R0_9!Gr)z1eRL;^Mb}Ffoc6t%P%*OdB$g7ulH4Gf>Gtj7Uvz+yzDzhMhH_ zTNlgpS168l2|3tRw0L6NH>mKR&l$d#QG>3t)XzQ1`fdyzsnb~zkPmj6p+7ELCA+oM zr8>^TCFFdHdwgb{9>iE%*hP9?W6Eup*7s!xkspldyi3B$4x9Q2T)Lx&Y!94Zrwjib5|f%yfGk-#Iw zfE4pc!@O-rbYTN@s}$2Wg=z)G22lq6D8LCu?YAMn2Xb?i3@al4ue~pihq?{>Eu}<} zvXvyQB1s9^pGqp(9@(=eTSE3dl{{HfSst>K?EAhBMMEo^vCEz;V=OVqHs`uUtM~n! z^ZA@}KIeT;{UtMI?%%y!_qBY#->dtdJ^KHg-dyf-$|OOjMP(t-)iVCnLPEn6-i^oi zeAs>A?4~jl%7z^&6j$ck(F#Bq^$%{~?zVPMZ?L3lp}N8Hsj}2$E$!`N7r7J@Z>`Z| zezW=h`%q0k>j_lpn7n)M?&HW0>JJ*i#zV@qg|1MroD}0~6&gu=a_HJ9dBJ2zbc}vn zl3sXAkz(Q`WCHFWkV6`F44%V}E9(;lulOE@?tsnftjo7m@9|JHz4$2haq7mY7_Pgn zoA-7}t&;BD=Jzb9GuUji{ETFV>I|E1$G~hwVdrsI#s5rVTtYPho~4zZ*|GG!fK#6NEfa=mnGqvzfu`&dej z(x{x+uW__fBmVvR>h%?7V?zn4nkrj_h#PORk&HFoTb%z~U>w?+T1a-t*H=1>J4aa= zNh%{3*x^p4%;=FMChC($@NC}#p;Y>4{o7x{rxnjz>MBKJ)`kk+EhYi?OBFksgvle? zF(j<___Q#qPS@;X?NDE#tk3=mZc%6{M#p(D z8wl3`kimgQalb}S8Uw22KC(G1+Z`ftP+p95amO0jXF*Xfi4a5k*;BmYC0PIZp)h!l zw1W!xX9Otv`q;~N(o%nr^$;@lOwLk3m3}~Xe`6qh?lKHP0Lcu-1vh~gNH*e!wnlcU65RUli=`~GB`{I$mkqo4X#YM;Bi2OJ1x6gL=$jPPYMn!Hwb*57IV@LMYn5;J7FIZuU5&>yGHZ z_$TKs%H&MDeE#^#Ln53amf}G%>5%K2M*P7}dvoE{T3SoaHW$}3@{I+E)hOLc_;Sx< z795f}Z?Yobk;XyEPOOFKCj)uPv$17g&gPI2m3xa=?2?$C}*LPWH2h>1EgYc#2LeFJAQg+R($ECp-}Fj=4c2!e_4WL#8)i< zG+oj2l%ZUn9s3sox``G9CP)L1Z?N3+{2_tg&PKLZ*pEjuamtj*fnJDj5IvhjMw$Ippb(tAa zZgAZaKsB0z!o*mPQh*34XGxF4`xNNPR+~+>M&<-W69Cz?VI%M5k~yI6mZKsn1UVQ7 z4i`u-dYmZu+rWj>(CW}Epb{v>-GOdfuJ6MW%rv&hs$M?4O9wjIho&y2oM!44tuW_C zR)Is8ppviGU;EhOMB%iKGJ=dbXvjf!24#W0x+=V?8W9aL)FkfT^3LbC3<}49{luyb zro%zCs^A9sXT;~%ZExe~5w=b{{A$*@U3Ep^PM0H?02A=fr-cxIZ)?R`<1bcG^RrWQ z2war1me5vge`x_uDh;{_LnHzzR{GRAcaLk$#Jx0H8*UY^w`69zqbO|RswzC5D$$T> z#8W@xsg(L~CE~(@M#un0i;=vQJCy$1KC*2^ds-#^Lv8z5Z;< zKPT`1Z}am1g$@1lkpEq7!~b+8Yh#9vx0+y~cE#npQ%u+eP~(s7N80tECQ9WxJ85o3 zK-x#_Zqs`Gz@T6`diku+o8ljn0LyNMWaH)p6VFCeT0v0q41mCT@#Hn_9x55H?t)w@ z-Gc-2*HqjG>k_-by{1i0$vWQz<*zWU2D%?;R^JF$MZuD?Ytj+=_7bQMb-mmZ;JdnE zfB4lclR(ND1t{=wXC{PUMv@_(z#2$z?wwQk?aBsUoUozp&?pxJbr-~98vyAe(*$+u za#W4=7amNQb^x5ApyU2Z-)&n%pb+0yj!Nzyue<&18h!QqYL8Qb+b;V^h!41S+c!gaW^rg=DvM64<@}aM+`1I? zPAT1bz(BA#_0~|^n+-Kvw>(>YD5(gQ!w@*x6FloZ>kJ0IKuNQszl{ws&n>$=nVIbW zie>hD95~=ORzW*`S)q)!X+U`7b5nbEqa&S&_Een9yloXBN9-4*3a5lK}dBGSl(_tc~&@S_sr(-!1J?$p#di44QX`W=_+r-ok| ziRRTvkVUH-j|1bx3;1fD5FX|~=L71{mo6wB(;>!q497N_slj^gg;+)AzC%AW)p+xCbW?uf^K-CF0#aq7;~kQMV7oSTanvl@bS zgjSe=I*$K?2KzfFN>n609`x{39Q{I*DkJPV^knk%e`-3 z_U93Qo7O+8_-CX3Ig5YH&9WErJ%jegqW-xw{)=4w(b7_REqDCfpz5UuVy#`ER7$sq zH3p{sB6?U8L2wtLxB$rw6}C0)Z_YMCnB#Ilc(Q7>3$Em+6h)#C2e|M15w)(%!hKeA z8Um$7-O@?$ax;i|t(8ZTCu~HaTG9pN^hMo;jTow85Y~*a1eSPJXR?i>&d=PjuRe&4 z7)qo!YT0WQ^gr?G?f_IL`!%2^W&pC+3kq3x3`4Z+!m{~5>$OtyDylZP&P;5m5ud%< z1TbF%<)-KB*+t)dF8x7CWT#Rm17clA_X2ZmY-P!wT8w2nXaIu>kleNN6vg zJ1xtT=?T1iUq{%dFM!2iL&9u}>S!hK5PC6w0%2GFomlsL`?zZ_8E3Sj18XDh5vJ_G zr#=rPe+p#{&}%{>1LOw&8o;MD7Hk}MGY1mB9%z$9!e>>!4BS)k5B(n8o9{KTGXHj` zcMAReUJCZ}Q6Ex`+DM5p_6f&a8ii+0WRQPy_#|`J4t5E4+E8j=l(*XafhQAy=H}PK zK`OXg__p?!%C_yz+SY!BrAQ@M+hq*G9f6U}(9j1&PE!z_syoR7ejx1nkz z<+$oCvq+Qmo3$D=!o4=lR!GYo%`FwlU)1A;KrFR;H;V9)tb}(utMh$vovWGH z?%GP4!v)c=o|yrHk`_>o&Q`}hY`YF+o=rpUi#0Kk!3Ti`!lpHZ{EqC2FFI1X^OIQy zq!jma`;4jvF_>SRdSf3+8P5ym%_@a&f*BoLC*DhUf&3?JfY_v3=ZSF>o}Ai~R!P>&O#IK>8MxX(fP5kza5H?K~Qc8!rpjXLbN__#`_8URQbYYC^(a}BM~F1O(& zm7(u<;o?uQY;6!;b|Fay?z95^TZP4;GCmx4Q_cUwf|%?P(W5%+1cBbX{?mv)%f6_0 z66CqA+gs)S+1i+I7j^h&OxRKuqmc47bxG)vQb_aQUs_6^{?uzK(+Y^sHST`7N^&P+ zpSL3DaA(cW>4T-gN-9yn>;O*LtpM~~DnNynzCdY@wFO@YbC3oivn$PhcKt01KoKd| zVFi^=>}?Xt!IW_CvA(7SXTU0yqhgyy$ylYgeGA(!^RYDChmzqIXQibY5k9^T7iQa_ zUIFPJHpqq02diKCS17#^3dwDCr7k)vgl3?;LX%TiZua`&!LDa(=zJf%i*g!FWE_bW z`Ic$=(F{fNP?)EUb`Spp6pYI(B6e3bCwsDf+9AtU@S;@oJ4WpZQ@pI8pSA5KQ0{&! z)Z+`1-**bPonkXz@o(x*MyEV8gQL?0Bi@8`bakuK#9e0?X4@elWBHb^ia}@g^ombm zN)5Y14DJCvuoE;$h8)@ITMJtpzAD-L_z)-$kPsxP?t$P-I>`$~IxDV67$Yb5sJ;)< zUqRC59j;SaAN~T#lnO8o6@WwgFho1QQbtYE<2)J+y$x7?!^O=SbHQmiJ2t9{qI-^*yqdD_EEzRDf_VD^eI{ao|gDM5L2SkgxHVSeTBR z>{}LF`uQU+UW9ohAZ1CQBiS_pfyO7IRrl7Wa=<%orUf0MHR(^h<$s+LwBSiRwU|e2 zN8KBNHhA=15)E&O<)c=!Fgh0`k@MS*@Zj3;P`L|xq|z*{a6b}fpBTU&0C6QAq_AZ@93j0%1EM}-{3Ek>Rvdf%7TA!;s#k!7 z#@7Avk(Y-0AgS8z3^K>gV`59%mOpO=P6cxhld64rPu3%Px)3o3UmpF}t@4vJJPeHv zFni@lr+)hF%BC!JEm|X??tezX?Lq?Wp5Rv zq6Qa4n!$^j(Xt+m=&T~4CCSm_07DuD zIasgb4Qeq>eeYwWy-HnYnf^mlNRu;g*zawc0#?nHk27)nb7f!u+Cw5lZQTmmxSY)STTkfVdB^WK7sb@gHTTzAasJ3xzp# z$vWB{EJ)=R`CCFZRv2rCwU%xnsip4?MvpM32_kTO47iA!Es(~B+s$}{7LD5gqT#z_b*0Wo2pVV)G z+;DKt>`E(<;0?K9<{=bTPGNIuo{}Qpu3HuSJoGt7ZczpL25vU_>o3O_WA9({Z%GLg(7n0&HpgzJ ze}p(pUFdDrB_*i&{N-tSG*oxX$&)+Rex4a8l85ltIvfFSPD~AvT91WbYQzacn6eJ$ ztm~!4lPznv?~d-Q-+bi&hvbb_t0~q}ZQix~gYm-~H&tVvMtJ@5`(2XJ92C^roAm6q z{PceGhaP7(a5@}6-{Sbg=PZ5TpD#jl@@MK6x7q*en_-^yda&4|kAPy8B{ZhXi`WgF zy|r%Bd7!1snwy(zx#{EMWKAHnGil8;H-SQ))GS;^@erijG`GwBek$hHhG~pCK^GX@ zqZ8J+ZmbXf)4OndD4zk&ciS->c*74st)ccHDLbMgYOwq$xvh0HDpXklRrg21f+{;_gYD9d%ocYLTN zH@Qtd-w*TGuSRB9?Pm&E)-J|zOEDP60K{wNUJkDh#y>Z_|vu?7rZIEN4h|tj%jAPD`t3~DAmuJ1=XT`7^1mFt3k$^^|Tkpln(b&rGt+ZH*G>Pv>(JT&a%b4 z1S7^jXA3!BTy1x%g^h8mtM)nn1 zjl8SsNY~Lv;<#C38qcRrE;3cnp5js<5XK>DX9RF;51zRlB$bjCas1~S3HH2X>Ed?z zh*By=kg&wfeHq7tD|!#0BBc!Z!rGS4DW%R_sfmn;gz()4jt^5h^WjUONc4E(uDr|c zP1}w>6)2nsTs`NO&l9AFbm3)e^!c^metMtzL9BUE2uU|jofVSOQUr{J|q^p50(`PO%RBw$iAnwP>^dpdnE1Za=k@4}8JyO_u> zO;5t3XTh4}((8Q3k3~OLos0W7-u@cJ+OWMD`tLsNk0=a_XH$Qdaax()BaW%0eAIQs zS!WC%WK^}g0YbEPr_;&vTj$K1F;R=LNZ#jhhKApqHvP={L6;w| z@vnx$k&4ryH_%9N%Qw5R^g|ZC!szS$^%VP-ul?mWeB_Ak43dtNI`?gTmwtG8M)Em# z_(htB-}n00dAjEX4#Q4IrhUJz+Ah}Bl<8Oh=27^57Qc>Aawj+vjvv(@{yJcft7{ob zvd-m7Zustx{CSW+EAn4wYel;Y1X3>@5#!Kj-p9jP?b0sE@gEJ5#4aDH{?%|Uc1qe% zeLWY^m}iBJ;y;gDd}>pr38jADi~aI4>U%xF-G^-7+2v z@BAnJqu@hZB}X0nsFpl)O(&?w;!zzE1aokPnj*j5mTNyV7@IyPAU!GTPjAopF!RcV z3tGrQD*xQED{s%D)cSUi;*a+|ok>VaPCJjP88f}S?*GPt16n!}%WG?DwEoxYe9;NR z#9DQgdk$2xT3kSKM@V`U4>?#ID6iZP=GAF8JYzS~9)Ws&KIyjwkg9!is@BlQ&KLu$2#Do_N~$vFXqD}3$aQ^i*e4*Wixb_!W; z2dwBNR3Sv#FfE3=w8=w85@z`$lG>;B%p23Rh;FnW9mId|kC&X6+G5Hu0WZ}HDrMGv z)%>P)33eyPoZ5Pn&7@CRpEjc9M9|O~Z9tlZAbbm!gGd9brr&7O86?y(F zu9iO94!os9vbQQHx|7Y`$+Q;goifudNZv20TShq{KP%fvTn}B{aYK!)Gja3!;~W$` zywUE1%?yL#P7TNWRf3>DT%4>}Kz4xiRgLFnA72J0rl#_ws%7p5sB)%VGq-Bm1r%rz zNgmf;JlGKr0f@*2(QQXRQTUA0u?nQ*l_?9VSX`~=9>FwIeBA_4NleBvTf1Yo?8)s8e(Aur2fJ?5 z_KRjpWqg*M&ln_n|Mnjp1a&qP4cf(vr|vwNUcAsiR(el3%RNTc7*Vp7TI-a5V6-wYoH_Q(cwan% zdhnWU2`EH$6ZxGdhbHQjeY8 z@HohZ@h5Vcwt;%lz)#UcVBo_1tbJGw0l4Uih6Ld0o*ffz;dZE3yZWmmyf%vojyCtl zj;YcqMhPuLy@nL^2s>V#(h|J5$j{Nj;$DA9ajlSNU&`+$f(zVA<-7dUH-Gi#C8HS> zM!Ej{$!|0NXU6_)m_O&`kGc3cYw^EmE$^sszFM_v8}{7kzb-CT2L2e^KX=XlrrWu; ztXWZXb|~8f^35jDN6G(O?P4oBpT?hgT9I((nT+{i|4J@g1V{Y@G!SBm1;5qtf0;)5&20OU& zVm7PjY;4UnQ_XbmtB!zh!L50o++*@Z$VqWkah;sAC~MHPduo;Pj(d_=aM{Hob?238 z?jyGt|DGzx7d%vHrC5tW#p-U|Z{8^OGHLbZ2i1+di;)Sb*t9jXEP2xEkx)Cxx~iCI zUQ5x&2&Zb~Cb_7;Wn|(-$Q&C`4{e;xAFOORE}u6TJJmvPv#H9{b~aCS(lalcmP*zx zB0$t2fue>D`Cb77QIj{a*)0abC)d1_rn)EzF--QkIzdv3(T-HEd8N1boifL0bigwv z5gK_(1NeBQ(XZtTx&#L<1L+iF_Nu#6MH*1yiqwxx|BPWFGUm%8=xXm_Gm7ILEUTxh z4=gBr5Qa^HAD&Ltk~b76b#*g5Nlyh1au~!0PLy>d+4NOOr?%|6_uxk90OM;r&zbnp z`&`47W(_#opoG<`$3TdtuAw1VB}FZ8zuY4=k)GE<-1sgpc5P>^GAK7mGY|<=QnQGp z1}$moM$*iIqbDXCA)GrYybupuNQ3R5h^ok$jM2OGOon3YV=YKi$3tWn3xFQJuk~-t zNqBN@T<7jZi!3MEanVaJ`+?*}ftD;one71ci=Wz_8v`X27A&J@+CG`!r>`CyhzA5! z{3ccb2G2VXUonoQYZVJ%snR~HU_;Zn_vh%qQB%xt0vQ#b8t8}js?C3q-RK=PBgGM; zU15ijWH@bbw^b|I@{O76)jVVZqHSSeQo}`5<)EaVMBN{r6xJN+V+Il`V)hyHLVqib zG#3GQ1|I+&dimWK<|aTdO~Uo_gQyEu$vu?JmlSRvg$Tt47^{ut-t$)B&GF)-#3!yU zhsE8S4MXZ%ntn)|ENAsKQ5g}M1IBWix6VSkNrdc`ET_B>k(ZNdM~tWHA~K^7u6;(LDOiE$L+3sxaz=HD(L_?WO2#6!YO$ zfA_sr(j^VnoAgxrNsaP7*AsoH`|=)#$=gsqEdB&R8G(-`0O$Brt*YnRdfFYK+N31a z*uXu~4PoXTrDCW7%E0k-M_FV8OlJ*x09(Z6@7Sd0RXPhmYHJ=3))649fwU(+-fNlFGJn|E zF@_P6{?d5|sR4UF2PmQf%MRN*MYeF~#p2R>PKDmZz#bK(rN|8UJlgI22@XOs6J>2Y zGx5WhhYrAG8c-&EkibrrPp4TAWmXL{n!0#t7G5`u&jU;*SE>9N_9;NnH56B&tWs$O{v> z>8^{Z1gu0!&#Sv54pkzH#ck+GrI<4h-9|#6teRNtK|UaBk>n~a0Dan0p4=lrN2}Z> zcF%;UxsL}9YLh4P%Ve$mIivjmI0^lPOb(`{&Q-ZHly+}_(}-QIJyXLBN~1O)$o`PG zHB~iLBM7w*c(RJQ@fegGs^EL!#n1# zG`|&ATD_;_sH{tNIpEeG;GwRk?VI6U<|{t&m@bw(z2vAXL9>JF@WC_IScz(*>PP=B z3`8v)MEbbcge0$>PMogL@I?*@t>Ofs&Uj5)!BIKI37V|H6U+CFX1Z7Z>mTCB%IoO+93` zq6g662^dwKIb`R;prStmpL-AKbT@uA7#@*~38rL*vP}Ux4M?V5s>2Zg|8diX=jYD2 zLeEL(({&QWz^ZV#PL2PORGy8W$7+I=;In)@=c1{pnYpc_s=GRBn${lZ^FbDxMu(xm z_=Ww(887zy&6UQfYWmYAbCf&F91n-s5*iq>Y5txu4s1lUe0iJ-Qjf1%gcR_6_U2AI z>rPU2SN7fR^WLy`i&PS_z|%BYZg*R$-w;wdI@;`fOTBk%9pc+^RVTP0Hmleg+n=xI z1~%=W1Cb^v4~Ewy?U6JoZ`g71`U{Ka7GQiPpnYKJ)#ozgi%Do(L_1O_6Qd?o`AfH+ zA=g=L1Jod&CvxlP2dGrFHZ zc=d=h?mHQk7EhUZG%osXB-BNRvytaJ$XXlC{Q|WNGr6x3{-M9pyJh?zY>8&?mVC}L zfr;=02i9+7Q7B86&g&6)@L6!H)ic;Y^=<7}aC;8eY(~CW|K58tumoz}tus?sw71SH z#Y^nBLpMejMd5_JGYzYKe_c(oC@|#Z#{U{2X67W9LQ$4v{ULcbYG@s{(8a#+I5dQN zzhl6LcJNR%iz`C~_XbbJ}j$G`xZ?0OSS01c_oJ8Yk)}Be`xd8d`tfAk`GVZ zolbC!z>{?bF(){YbJB0}2}QfzR-Khr&YG;0FBKzX04w?I`3kmxILyl^Tzq+t$7n5A^tctioi?$%gyMJ-N%?5-PfuiV?@YK3cz1?*y@`W{QF-pdiT%9C|~>wHd8^3%sL3j}buk zj^|U>OgVGuJ)l=9ijqb#Bd!*Fk7uDN;&^xB-_QzE_Q8JHBF^P%;R^kgiznop431h?aWc$?#TI2U7`I(zPvoWC%QvPTieRs z>Xj^B{j;13K??_GUcl{<#iGm>bN@}LIfQxGTe_2hu??5V7h^I2apiG``eLj*bq-60 zf?L@^db*FZ_$^Ua87?}?Og`u=g;*}%3hx4CgwsDVgx9Q)w$ z;DN*df_jOAmEs2tu;=C!)0~#?q9n=+dil7<`JQlnstyD7#^QL()4k%uJ^oRlF5+ig zu`km?jq`iVZz|sQHmH=C>MVTmucdHpe;FZ zi+wISxq{K1z6@PvAmL##IG8>5_#eS*BSQr*@H|xk>>XD5jj*D-GT14Uhog!J(%tOj z>?`jE|K={`?2jsXn)v{B#U{pt>*O5c_fNV-bfs&DOC_+p8v5lk7$;lray-;6+7jP`0DfOL#yDH>4eS!1NfGBk4;;r`NyzR>l z`1Q>&aE>?nd%u?EwXHl51qMM5FQAqYC=@5UWV!;SlVL|Y;D!yQ&4c<7 zQ4e=8{!x~oSHm52oMCwPJSEA8o-piPn$_sl6zPud%y3ZUz~tBwTlfJ>-|Y*1%VlIj z4$Dfj*Rx5LJMV&SA{%j&RASK};3_;aG{3Njz4?&_&9_n-J^AJ0< zn;FE%%eO#Oi0VdRz{>Tuez71VcgX7PakGY}Jeqk7*!(8)~|DJACKfWZ^?N=B>=BHAwn>%$&lMV+ddVCO|P#rUCxXf0f(oXRAAMEGlI*y;yu2{aGL}_C@`e!Usm%S z_I|I~n|1lseTMuqXE;(zVl!QM8x&s(L)#}@EA(ix0YX?DpF!o*s-4a$cD~5X9)J>? z?08>|Jc3O^GD!Z^osEeSw~^O?wz1?JxNCD=s42gWmTF?Z2TP(Y=-bA1%zYBXQYEn% zGS0rfsp{MBeX?%R-L6AXIGzA1|N69Pocw`1lqg zgIP{6ZEdf5nUNOE89YB*Mt1hTJKUp&8X;RH8@y$|@@=n4!CvF~mtm1f-swPwWR?nzbO?v_GpBda(1Re9v;0CH$| z-Yb*=fd(2!u%deC*qb%BW?~j?EKIA13PXRZH#0G8IWdV**9A`>`|&g1^Er|BF@L)d zC&WhFiX{)KmT{fY!{V8H?)vkUCmp(iO}&%(;sHyHCF8gok@#8AeO&Nhf<3YR$broLobms{t_S0Y=KC4O{ou zvB$L%Sumtef@LQVA#nhDbjbmqQO=6h@KK+vg_F|ETO)C$50xzPMwg21%x5~lU$=q% zV(%7je_=U1aia64&8c0s z4LK-^A@nJ%{dvgffox#}wk!h68}$1dG<0CCNYgoEwr6FaowavcG>=1M{@|f{=77?e)lGtKi=`DY?^WCv|WB4}Za^`2YX_ diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/images/sft_5000_train_100_test_compare.png b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/images/sft_5000_train_100_test_compare.png index 1142dece728950883e84f33e568b7767351cdb46..213a544d6941c52f3c7eabe8ea05c9169fd62780 100644 GIT binary patch literal 26506 zcmc$_g@9nvK&N{YaZbc1xaba!`$O5cKjgmfb<4bmXp9ea4c?|DAo zz5jsCalpM;_sm+eX3e_hynb`8Fl9wqOtdFxFfcHfa<8P`!oa}kgY9G#B=FZ=?8O}n z3|g6mq@=Q(q$IVngRQBBl?e>YtFWXLWC;EYk?(=L$18K#LVOSWLj0GnQCOY+==;SY zpwI=rG_qwdd#hdIzvP-EM&35Dt(n(bnJmxLCXxy2uiE z$jtQI|L(f6Ap{SjnMQ|G*!~r!@TfYFW^sSw^A-t{4F--ceg3g;yP+A{R$JRWOq{@t ztGgIXu7&5EmEcs%!@Zt#bjMF)co;-l^5H(?Pc&Y5qpntd9Oy9bL{_GL_TkAp^0lK$ z6uu1X8qee(I4BnQ7(9BJmRf~_)V>X~B*&7>feRClyYk7j8b40>8C4+bB{?P4vUZ@( z*w~l=N*#?=U?&Pr_K;G<1Dn#yZ-?$T4qqW*c-cAwkxL?OY5woj7azV;JZO!*m3DL_ zdHVV_-gO`~k-Ps0)$>0e@iuypk$!T#07g3P6{Qs}x~r3f_NSZt8N|4Z zUs&lTY(;}T$zU+@Q#~7l!gs$ORqNM1|7i>v=W*O{-?=>Maa2(z?W_l5Iu4Y5Dg@HL2<+u1S z85o^?nHNC`Fk7M0#cP8%r&O@iMzF0x->l*|JSRi4f9S*D_QD|M`roo|!O`j?mHATG z!P&IpU;0zyAmz8AwIaz;W1@&-K@cYco+`uD`e)_hP564}Qj){=h%;}&5VsQA!Oz3p zEy3ah;2y$723+W)`BRfi1vAE?6(atTERH3V3ecuj>ZL;S%g^Iy#9fhmYED=K%jDOe z`}#A{2j6J%PkHDgJfDdhkTJ!LM##4CBmL#7;L2dhS|jwI$cNl5akvR&qL{T_+0itS zR`~8Lb=aYM!Tx}~=rFuQ6!wE-e4dE+n#?3PKab=y^*anESURb4al(Rr73a_N?}SRw zoyq8Anx(v@j7AAw5LySjNZjOhj9NJ%d0;jW=!8|s*5$J2*%pM1tZr43aLG^@h1+*> zaH^zA$zv6xs74F;3Iu(R{L>Suzi0%7(3R0lQtkvqb<-PZ7`}meR{fklnbzIqaDfZy z__8E#KU7amj-C?4+`iXM(Vf+Cwk%^ub@tDj z5e9xde-v^FN&|EQvAit11P+PRr~SB?VMCo9hFta{Q2%MECV8({Ud(iKMYQYG-=xAB zzA2VSY=53m`$F?2=9zmuYMA(Y8N4{&1g!X>g#Eb1_&#I!uVezjnbM2$Hq?60#OQru zi(h5D;Vu#wvn(nt)GE9w)a%1BJ~i3mVm3ZD={HgAolkUs{w9_u)};5V@l>yRUv@G@ zVpeQHuVz1DUr}FDKSg{a{e?UwgDv&J^Wix5#NP})Dj!q*^S9oT5b&D7n~1G*nMlEw z>VIy1_ez~ng+G6!fTM!3(!78qFSQg>ZY71EtyC@4+zZ3ic~o@r(+e6j&c1HH;X&kK z;HiBNowfgVvt88?<`H`R`pkQKVH^KR`jf>ca8V0ShJOOHB? zu2}4i2!1-tEO}L=s``a+;?0D;wX-#qb(~ANOU!v|pqMu^*5Uox=n*yJ>YlzLK z{rc==UEf8fW|`)jYXtA~kMZ|xcWifZ@R9+q0-})0k%lpnFw=ssf}?`5qk3g3WJqP4 zWn3en-7i)NRz=a{hATcNr5_B8njE?;Cm8ny;Rkcht zjU%0-&k5X#`iVxwDNz^8Z0%>uQ4Mshh<6cpMA$6rtlq5XEOV^CA!!h3c}dwmB%!>y zZ0dcTW=k#Sx8>R0Z{IE7KryQp%;zj_x`q}N&8Lj{Oe@W(%=E0qUa96Hj=#3hpy0sT z$FiglBo`H&6)+W`biEzu8+_jNCw9ELD>&kN#05L0%cQ-jz?5Kw&Sy4zHnGM~oo_mt z70wk(^G*x=Hpufbf4WbNuMMu5UC~|hZk}Jr1=AYwtTrL{cgz(`7xbtua?e+foj_Bf zHS#qIrt0g;FJ2#}o=<2VDcYBV}|i6he^l6eOm^BuK; zA}D-l#6hkyy;3X@=Ao&Pr=ctn1hTB*?V;jez7du%ztzYLOA0gT$`0-5lqnH_sPX%?vuBcvd|$j;cuG5pou6lwvRkr? zEr2V);IQs+^?kve-aX@3=A`-9M>9;fPj}|~YtLWTYLcaYN-d{eTCPkvRs7P?s&_HY zx@fpQ>pXkDebwtU)Z$f_`LD-S}9pf?e^5#v3(<()kW*vrpspCmSTFNq5Ow@1xt&Sn`i5J zGQ|W%m(b!~+_lz}fZKqT>)7iz znCF<7Arsiv7J+2aQAQyt6hE9hS6^u*XsTLQzjf?Wv3uuqdIhaFg73MDcFoSsG8g>f z^Yxlw{hdAfMEn%jV)_J=O@z&k$ew3I^}?6CiOP)@%H5Iu&w>SRrURNQGws#Cht_j4 z_}Td%jwM!tmWQT?kckV}7Qff5y&X1b8PKS7)A>FveIV%i>*_-fPmkrN%JGW2#j3{9 zG3z<+ZN9nRif7YjHS;wsGcE(m>YfKHi`Cv0-o!SZ^FJIy+&5)>^nDIqUbZf@nFOpf zW;PD*7cF{vi{0E$1UO@JVn(5CXu|KgYd&GU>P4#usXla7a_q$&rlsuxx?=P?C zGREqdY_We-yH5$=3Ljm1&)$6LTft_c+7=rVb@2|reIb~1LVEFiCE_UMyz-+S)yFqC z=L)TOUrpb`66D)Sz`@vJ!M%(|@^wImSs9!8;USYdW9vr>_Yh|0c|Zz=F{TEGnzVQaae^qw3mVy4vl$rM z8k(@VS=&8Uff05S1e?|-P6pI&)>bx-f^H&D|5HK`Y(MU1e@gwIB2JbfPaz7*)RMLi zCe*xa9Bdph^0ReUn zE_N<1R#1Y~(cQ+$z>U?$k@kNM^3QRkOdO3IEbN>tY;C9?k85CP>+B@*^y%Y?{{8u% zcAB_Z{NI^u9RK&Szy-2De#6em#=-vYu|ZYg$Gw8e7H%e1no<_lpnJeML^*jlg#T0i z|MSiNo$;@lkpHX6%fbJ5&A-0+f7euVG;xr$wFakj68*nB^S_n<{_=k-3bQ|6`CmQp zKi&MFy`Y~((S+Imy=S6m>rGjOG^_l zG5yca^?-H%;GjbPpu+bBy~_9Bt93p1f-Lv({(zxTUxqAt9yE_s zn<+J(7SJxC@WVy>{?ErtWa<%GolRmruQ-!C=TiO5D zZdh4rWWPLzF_D@7+L;@GV~1E4;CuFg;(zNue)LMV5>^Qt3orat)(=Yp?8JhbEOq|( zT;QPCuo|t$u}4mSw?PmE>=Z6fc0jUw{O=QOQ6qd1?R-Bml;d-+nNZ|; zvCm_XJ;I6N`=Bg|N!Z}LGg*tY=<{%AH^5qF?p+Yc^%~-HcQ&W)sy)hF*@P#0xfD<} z@%Bwu-^cxbdI!TE<%Kcq&DCa3N202t=1Jq-`CJJ9Qo8$6irzG0l7UI>BRd3Vtx#RJ={_|=1{^kJNEYn^_J-N0}#_Kl8i*gw5Ll7wr% z_2%++?$y2CT8!8|i8{m}sA^dBCNPLtmp_twh~JOs@s~n?NFpDZTch(OkJ8FIYdCbb{5Jm^+loy=NR2AL#l%9IOf7YN%xyNKR@{BgCR zYSN)yl412#4q}$&e7fj|O!?Va<-90GH~;e(ZHzFUx{kA&_Xb^xHrG5junY5J#%?ey zaxp>WT}&|>qU&?#yoY=yF)?s3p{n$2hs(!JaXt{4%B#xs6T1xP-um;4rjsFbVdDOy zEZ0Sgx(>4(Zz4C*>n^t=jKt@X!(tCj=ShN*qW70Z3LW{#%}3p22Af%~lh@X}tn59Y zuzHVoK@>0A;j8rhA2&98RNh9~N}^Y5B`m)`-jvt&IL^4KnH-}3Gk9|&Xhve;)K9JI zp;be<9LqybEX$znlUlYESVFrTtl?*?UBr4AcQ+T<;K{S!T4;2gHqEeZR}#BRe0R5= zq8AMAINR-k&jWhn?$-z=Lq?AgaFcL#z0cHiI-x%dpio5b;xyAS4=5L@$}mR<27Wh% ztM=a1htl%mI=1g?@pi4S$e+iYKo6Ne=^uNis!-HV<3~m>6VwZe zn#+}lQo|?A6_96temf3}d}8~dX_`j2+ptps8$ow^bI?+1=yG>9g`Dj=scH7Bw&i-8 zxmoW38;uYR2lB*Takge$u~ZJkqdKPc$eHrG#S(RufdTt;i(*tT60K5=q6OY*MjRLr zN@`|V*Un5FBb_f1j#K&E=DlVazMRtY>8iV(|0f=7#iP`y!NS85y)7fDe+KOy1efF& zB&y2~P0mHw{c*F8_GGg31ytXNF8{}RvKE`y#e!?e#M#~)-|3%mSym>mdT0L*09lF#6o{uY>m>|+hdV%P)7AJ__CL?VqQO~Cxrcweje^9k|*P5({+w&-WEA$y}I%kR6M2K&Df zGQFz4Ig2ic7W}|}f4O>gRTaBDTNJ~IyA%AjXE^c4<&PYpf95tFOg~DB83R?BF>8Hy z@uT+bBk=qo`VSaAv91|ic_YFnjeJ3e&J!4fY|=rro@SPfI+kB(!jm*wOq3-xxExb z-w&}JOb-#nu4+9J=+hq_8M;uq+$l4Uquc-cwFbN>SAV#J> zC7}O^(>9%H#yWeJjqgfC#Hy`{W7GCj0=A>wX+23}TPu)d1fi{(x<^WzkcebobCCEV zjISf9%M3>u2AbY{*p9vo9+QZO&Fq3fgRA9rm@*MaOw|OAzFunnbw2Wkmh&+$X2IP%G-+U6nVV z&#HBWvC<}^qtjH#FzypW9Dm{#WjHn}?SeW>VQ<6K@Sf)2va91TbPtzhxeUkrt4cJa za%m8E!5~&4j{BaYjATcu>FcIBfPgkK?>Jm;%ZZ0#yD;pk>JK@8`+IHgJX};4h zR>rFK58wJWzl_~?wGyfJxYR~wr^Bg<{x-B|UUQ=dYu z?-|Ag^;s%9|4@ej8V*#whr%^;BMSa5Y^H!*Y{*DrnNAmLJ>sYa@?rqKMPB4=;z?*s zd%u7ffm^oU?UG!TsIAmWe54o@S4_{#=Imzl-(dCu<;5vQl{yucv_Bs*wEeLAnVt^A z%SM&<5B}{0jcrf91n{w{(5vu=o_q*Y4&4%3SKdBO?2_5Otvoh#eikaGZqr{I6%!c7+J>&t>r0swj5t%o5-zUKl9OD^xqN0{B$Jse6*WE>dGRy>e)95NI*K> z%Fj^bo$$mnBEz)JB_W%0R=yuyNz#lRKbbliq2N&zc|7~s=%8NCBt1}wq2D=*p zXI2Ea>cnXlxIK5v(Y}Am*de@^ToQ8@ubot5-}^^d0r}awmWZAB6;Wy8S6i8LNB$1BkDjZ~H(z z7OXe)tLr!E@V0F3KoEeyB1>A=EKiKv6V2A_#cX(?(R$x@pQNFSnxG{5{r>uFM}z0- zZ=VNmhl8@}p^}Ca*Z%~|<44KtuK6Mh@w&9~$5_`Fn{FbT{CNd|^+bd4=fxwNsa9Y6 z+Fvu5zjJM~%KP3Rt?6sw87;7Za%*%zYMM&(s4ZZnYnVrY+Q~c&Ji_eBa+#H*TK&oQ z8zblAwMB^76c9TVUgxVZVpv?4sY0MfJeGWdi&6c2_P=j3J+L;Pj^!!kh#CN~gZ4)t znQbbq@?y?bEbGl6!=*qm&Bnl(u#%>rnA~Po6-h0$0MMUg2DX5G!L}6bM*;!Kkh7;H5?bb_UZ*TQjLOH z>*l!z1vWF69~B;%O})Y8e&fNcit`^q-dS^6YtK<>V$A=DI{l?NwII0FIv}SQTK;^> z(S_;@lWOd{KFjjh{?cz|g(pZ0wF5(G`7sf|n6qtZJ|@qIR^VHB;dR)CoCO_lz6r#0 z0)ma+-KP0$a;=Fwd)|3Ur=N-<567i@bI;0K;>TAk4sglHlx{DR^b-$A%`!R3?K^Sh zoMVKKUr|$#3LbUhGyHL9&`h3dYRcA)dfB*M&aD6Ia+~n=IMBSw^8)+ztMMn7Ye2MP zf)UW57>da|O$~LS$(@ON$NOa5Jxt?R?>~Ggatawp zJubiuHG`Kd%ea_#j5$Z;TZk6g|Lol=eRX>>%%H;1#6ASM&G$BQd>F_$g&qvuCZICoZ+B~#1&-U$A&kGlC|MJ&FzV-C)Utv~ z>uaBccES3$xSzvg@l$7d9ruO*1Q|fy#mAdHKDDtjk!-}eV%*IdNxj|xiIArVq_wA0 z$NGqj9!RB)qpOc<$i}3bLITNn6!d7J<1_~s;%9RP7Idz#+6EeCbHw=7rtTH8|HYJ zxH}>Jxi#Ig;>*pB`r3T z1sd&aLX|lnuG=`sJO(iof~U<3fI&Eu7>847CtEl_MU7jw1xRfavMoa>Tu^p54D=Cp zK%Py0eZE(!@HL|N4y5FwyNw4wU@Zqg+Lnhd=%2XRvK&lSL%v}zzd*mcDjLHx3F)I7 zevI3}u<%F-2t*JG>ERZT|C>MH)-1s*P6SC~OT_v!po_ewFlsw{Ozb3_!DEgti7isY zw!$c1ju+0kuX;1e-h1r_vrOsTs?f8-aXxLdV>9UjjnXH%Am*#Utykh-4s19aOMX5% zFRL6EypC!Tastv4K5alhn2c5INPHqRj3QNH+pzMYgYf+1Z~H+1kX0GJrS#Ig-R^p( zz}5F`Ryv!o3spDW=l;qWHyfm6r>!{(*`*=*uBl7IW{9U@XqqfaoTg4{2PF`kCB~VQIB=9AQS1e1D8U*4xVaNqQkY&iMYg*W!+(tMca1v8_ zpQ@;_)fej|?tuevIqghF6^;>HWI_EnNTwT%7D*EsdXK3bdTGMB)vd2P8j$y)R^l{Z zqO<4#Ngly=pVb->Nd|K0Pkc-4RRo`3tH0_+Hb^GL`nm6GVvePAmU+e^)~p7}G#v!r zEHZJF-^P((GKjx)QB9I#$f|i7E-g*3UF3SONT?~#+-{Z5245nJyB$6Sh(g{}J3rw} z)H`G1&r@79r$0~>o0AuEbx&jiV~bH$=`6mgPQI88FqX}oBB#OqN(fzf*NOIN>gO$P zUBP7ENtc+)%Tv-XsxF*_nl;Q{d$+%@<1c3mXVkMWD+wP-NvFQ|!FyRbwN~`7bO_=< zoptYu_P7o%C81`0yl^66cSR`9ucotXJ$mK0(I3U1zT#0Jf^9Bl+a`#J22F60O-I^4 zRY9oG9gi`Ub5VOAvTQ6HW9=XzE55*_jRs-Av=$Ltvw@LxmTH5*h%Vefnt-&K9ld4|UP#lI2Xhxg zOY;TOR~(D%e6taGh1)!l>(@6=Z<)W2Y=480B+PbV7FrAngl_5?hS9xH&>nN%jAzVF zw*?u${6-NAeL6i_<&eT`NDs}bcmiP|DKBrW=iaxJWMtLPDpv9?^Y%Ry?-w_w(Ak@E zr@G1XVQHxT6hx)0r(mWp67PgBGGHI`yrCyr*5^XXq=vSeXY~@T;X7~k(XJ9-V}=pF zKMBaNN>2Rrd>H#9OFS5%$Kv_I=9(MG&$1DlZKNlau_4e}u)0Pe(49J_WPh^5b3)Q& z^f9+A?kIxsS-P=z%Fh_xxhKI*zQ&TOXDrD%Jb3;+^7quP5J?Q7Y_jkx);FfKl96U< zdMmi2f{KEF3QN9fe(jwt5O&Ntsy=T+HvIV%b4ERx?|ALS+003L6J0ss_GtJqZ+)^! zPhSOj1g;QintY$)SoREDG?ctE5b;y!$l%E<5?NcIPn zK+>rfSGo|Bq&qOUBhM-72po|t5s7fRvnaT3VGQ!x-Xn1Sz7z*~sFt#g6i&#lu;nlX z_pVJ42@|KZL}ik@@M_|ihytoB4pBS@>1kmWz(h)w&?V9t^35v~m3NEXYO1zue(ZsS zrl0eI#~fJ@AP7si)g;?d^ba-YWYwvu(#V$2jc?faS-(ua@N|MWu_5Nn3Db8Gj0we;LGQ_TCrGLVUm?ya&rEr zSwj{MBg9zb)r_=)3P$}tHs<9*(Cj8}KmI9*07R`r-|dQ^C1r+m3QzZjsF&~h%p?W6 zwE;`S8q3~GcZb=-OCb%<`+HXLIn^nBRrG#DG!lDe`DdxZ6l;V2zA)6iB&L!}hjNHU z2=KJbE~VLDrsaPUF$blc0YR;9nnDq+Ha8K6h!qq~B|#aSl2z7F$}yf#u=IM(eWfo# zxvTu8Hu{ylz|p)szZMCQe45VXW1K6FZZ?iS5Hs8|Ve|xCV)%8<*fK1Rii_JwbOc^T z^m-|4|5cl?=iSqRv{2>Fu$Sjo5$cH~+u|@?N1lZ|ujy+}IMy^tQ&LGfQX_}upTiLc zV)0KzjE3(lWd@-b;`)m*DlJ4lw_dsTwQJP8t9|t+YJzGdz)rPa@rApJU&AuZwaFgt zrg1+98_l+{N-ebd>ZP8QA8!i0Lieh*^M;6INkkF*svCQR*w^+Y39sll>&{qom*|N4 zLG7=02-{qA2?7c9s+Xa^1MTcLSZ^b#CRC=)*PZXHE=PG+t-K>8@m7wiuavQo=luuG zwDTrH=1_&ptQNKLN>#FSCl|TyE!AA6Z_BLnjtJadv-Ap5&AgHvB%NfLKV#HCRAA{K zv!!c6xV1WRF^rms1Uc46Q`pry4=%EGG{jv%mZnCKCgCSxR}-4t$(aw+qCVlx^^xs8 zQPOCA8_-FYEKA~0Xp7bv`>bPL@64{Ya;b$h7>GY>2BmiZ&9fDhY7U8UNPt=xjiAEzSj3okmEZU zyOJ55{Jvt78bj(`*>Q4(EXmVvz0 zB6ub#*qRO{*2(;B+aiS_)#fqRGq9VuUahJ4w}UcB+1+qRtW~XWA4XR~lS25MNL9;p z69f2`kh}K*j2D8BUc2IAUAhdyX2tgQ@atfO~@qTtRRD{ocD?6mc4mHm!S6-YYyLLLE+dr~BC_a))oD34F%$O#IqSZ~do{=J@ z%t&oH9-YqUvn>*^Zn4%YgVr8o+fO`oiU`Ndk|fiM$xAJa?=7p+(WUw#rRqgT|oBeSnmF%a0V3{z|J8qobuZC~M5L+8&&}X&Y39hKDe` z!KB^~^$@vQOSpC?p1**;;akX#N;~@%{m0A2THU#E1D}i0*Xq@@Q^(w3Sd&NOV?;lj z^%(fxY%use>1VvO`<}{B@3Ea#B41hbWomNWQM2h#W7yfM?TIXRQFG+`n2}o*-myqD z;nsoS)E~ILZaa}-1B#a|)Pp6rD~R+qDG(SJQAUwsj_wQl9q2$U#Q#$FI@6;yD(i^^ zQ^dB27}TPkW&E|Sx`_P~=N0-P&u;#MTz*@Gj(Q)+xgn)dnVVxYMLlXBzfYRuHVk{W zP{mxo;euSF#GdZ;D;*Y5Mvcx?)F3gZMmydEY~L70dU#+OAI7kBh%>fKMo8 z>}=Y(^ecQ?5*=Upy?V^VFnw#8VfkbVTMVu*=8Fv4NI0u$GK*5pGzGEQpQYu;GVd3Q zp-i58Xb{ap;hg`0jV~opI?*7>gq3;y!Y9z6;Z{1+O136J82^ATR( zUHuzaSBnwuj7 zviS{Oy0YW{crCdf63*vqz1X|!Gt1o1W`D2Ix8lB{t?GRmQJC|et2v*5}wk3U7-;!*lBchHv4zG zKZ^sP`{b2y+uzNOmIgbcyQVB%|FRY2A_1WLf%*vH?-L0U`IWFQr&w=;q#Xm~(@rFS z1oRU@&EF-kpvEAV(Dl&=5B$nqm6`mA&QG;gRK*jsd88ci4%q6bQH@t-RRX=_AT~y+1w) zz-yZJ9O|&bob+isI=lIUS*k(sVA&%zyF8oHE8UOc>Sw%`Plmh;S+~460%Q~oTjW$3 z&@!s_{0O!uG9WK)r+-{y7xHhX4`zG?gnsiJ8%z>m}a|ga7-wFYj^;j1VT#p zwK$GJj*h_Zo7s&e%?Cej6c4=5erX8STHLk(6n(f`Jsb?MGtyx;;47v5+{yC~TLm$R zm0ObObtPyRe#9m{nF8Z2J`YTRG8E1e90M%X?~3f>dhnB8(=}ahmkM5VgB9AU5ef7l zTp1FY=h*D6H-|snR-=ebbG*+7Z){ucTwbaV^3GX3(oFe&$G4r_YH4@*<1~k$YlF?Q z-RS|7A?v3~kmr(H5S>g&a`$=_q`4k`iT@q+uM+ZPIan3rKm2u4BkX?EMeJPu1YA-F z=s5DnWWC|;BEyH)m)rwTV=j}ki{Cz#7h@1;g<1J?6E{YY`$TupZ;~;TBJIK_gQ`3g zpijOXuL4Iv#=eB(RJ(bR9D=C{U@;Z)AQ!sUGD%Wn%HK_7(_>$KOemfE1JnT8#S%FH z3-!`OgN3P#I%?KhVu7Ox!Ww$s&Or_q9;c(S-^x(V*Cqk*iK~$$-7cU49UHNMYa%he zrF1K_7AqMq_*K!^LH&_bur`8=H8UA~u}xr-SF#Uyxb{I(Ic?oBk&V~QUY%TyRMnPa zXzmxS0!GnrA(BPA4oo%oVSq);jlTU-A72G70%hmlY!&q@XwBHzoyMzJx|WsAwHx2T zy-Z-qf5{wu`fHXSObrP)`H33I5JOb&V&_Rs*#T}g-XXben#;mCPB8#~6YLGkF0HG7 z^73r`T19n((KP@Fx?7f4?tO?=Fw>k(WBH<4Q~mICKN9PACC|jVn?XGlj091FBl-AI zB|$p4-ak>iv#)wmor9q=qQ)w5WrM379_!mjpBPd4U3>SzYPhlqwu3CMyW9m>J2c

`hg{vyE&A$8S4vl;MGb*blLaJA7H^Uzf=PSjNmaIx2;9#9!Fl{G<*|BdQ# zRp2fA8Rz2BZ2Y{{@v}r$hQ>Z)KWVucfW;Eq)V1xq(6GmtR5roWkoAsMNXy#x(Yx_t zR5&RPq@&|#_-KYtsY7u`fH~-Uagti=qZ-^3ga7c1=(z%5@dGdV2&y2cHBD#H`f&K8 zQ%@0hH!^I8%1W##(t@<)=Usn$e|zBIkq_a>7t7oneOz%gNyUM}(F-Ycoq|Q>U-@4% ze(eC`LrV!$H?|*=>4ZV2VdNOLtsNGp%Nmd9qt)nJ46htORWjD_*+HpN%_e@y3kkYf zIb+@<8IoH$GToU?3|UTZY2LaE19afJbvfDVw*W_%lwBvDVTMn^&aD!H|))Y5dY^0dS%e=Cut_*Vba2h zeD_x8<{D*{Od4gnr*PNZZnY5~?p_K6*8sZ0yyUSp{z(Z(oLEuh?49$Oz9}Ic{ik;) zeITy611l5ZMBp1%0PKAmI-*6+r#lJ#c8+>!V=xT+P=EzPPcjO`PfY`m$e}*HOmL`g z#uB?d^c5;4Z2w9B`ehVZS)AaUiY)D3m;0Ya2=&Cz`Esqgsld}XWM-O%Bqxa=mC=7~hd_ZQ5q- z0rdF8lZSQ4kw=D8!Ll~H;QE`wu2Nu~mgC{}B*ZO>udaGI7#&9wX1$NHv^g~7xSQ%K zSYBd3)a+%c|Th#E!nBgJihToqH!UCFvYpHN9y zAxqs7+3R$scfJ^VMH_@j1fdS?e!?&%!ax}bfdZF9CH-%J>tF!azbPQ~X z>`H!F_?yxc;&t4OuQ{A*w9XTyEu)_L9TG%6$}6@tBNK z#pL!mm*uHu$`xH7h}e<*V6&m<^v_4<3rAQr zw>ST4gm${x3XhF3rR!;pvtylVo*#yHuTUpYsKY0?!MMUWiNbLicytLIBzV)|ZvK}F zE8o=LThOf|Zsb8K5(EN9q)Ca<(`=eZ_!6yDhx&#yLpFieC*5uG z2b;wYUCVuvg1Krn{fR;sY`I~der#-lOSO^2yaFsWp4->B7pq|LZ=5H*20Zzy2eflh z$_O+o30cRu`pssUVX%^Zc-Sf1tTtPh~)(F^VR1(^IL6{2`Lp=2jv`o`X z@U}XwDQp`4&{MFZO21c*ijIxHI@5({X6@OuU`d$y$=(&s?2ttlI{4 zg~CEpK_bn;`@Au<)HFCw*e6FaPXgg4u(p4druIG^xzGPAh!C!SPAQ{bv(; z0hyva9D=MzW?#ojgr>`*M=XH@4y8=sSRDuL%ieui%+e44y(D(9V)4mt(S7p^ z9sa=c+b;T0dt>~dw~$WmKt6tF8w3*CH7WUCP$r94v0sFpHsctL_=rJ0{c&-%{Fr63 zDQ`d`stX}4M7xcmZv$?J#UWGx*gLf*7*XB5Nyf$><#Q0s0e-UWq=oTSMdN;nZQqrfiTS^8)`rH!_Du9iQa%+gWWhDtLrg6wW`_y$~TSFd*6IS z6=39W(i@QZW0I&UGUyUn0*`@tvUooX5H4L}H16&@#oyfC?>s$sC6Xt=sRU&B)a7>X_&T4 z8Adjo>24`X0&W+N`us~yQ3$ubGRgpc4rUl@P`$%dCrTh;#Pu?JziMP ztfWF_8vZ7UqE*b_7Es{gp}cjcAkYQ$oqKnBE4E}xm>Uny6fUtvZ>Aiv^gEJf zw|YopD9E#D_Y#CcNCQi7;o{ts$7oh?udISJGc0nBLVyy-9fmZ2+-*YPc$j%bihB`) zQD=wpW>_eBxufZ;^lezA(Rlj?t|V-R`(Tu|k$Wn^z0R&B!hcbq1FVva4GvER_ZMu6!K(JdzXou}m#cT>rUsTEEN*y6T^d z*I%pD+|XaUtu;r zG>$7bT4`-%!o{!g2arE=jHek>n^Dh3-tLHjb#~^jOB!woU7{(}0cA7KlBG^YD4l1N zT-MYT77FBBi+}sqghy;wdF=_m7EUr`gsXhoID}T|$GwPC`JVeeziI1x=l#1CD%c&H%@vq7SXI%U~a-*x?6$qk)+31DW{ip9qvf zgEV_W+`Gvp$Vu5fJ(KxNNmaY$3~A5GxfTrxE&W~8S76UX*QKs1THo~2YzRw#}0oh#8~dD@~NKp z-3Zqo>Ohg4te5;+RF~vPIiEwrrUGzVlS63RGr9!nMKy4yy1u7IC$LQ5PZ?~UTSPuH z8bYn?D&SwkO^DYp*-=rUIP2&2#Ln!T$PgC#{T|K}WgVYfo>Q^TFSaIVVn44nUQr;M zl#V3^;)R>YS=5uDgcWQ0tSwn>HLE-6M?v7sCFhWQ!wWHpx8w>mjuZLj($lx8HVS62 zfF(PYB*IvjB;~T22t^pm_U@;`kC~8{!CTB%tKTEuMmyNN>*P)zj>#g1<#pigX7RGR z&J`wo_l&vvGWPmw|F2c0OYAFyV+8kvdl^!P!F@+5VE0x1-+>DMPeW|dl&k^3_mxpaDBKQ8p9(jrrP&ef>wlCXCtPGffS zC=4nXy2o;B3L9OJ9+VwVXmoP_;k|2cq@j+;4o>W32liQ06J=T@pzr6+u$Hp>;~7p> zc;OoE4}Xhf_S3?+bY^V!c}%+O;|Jpy<#Z9d3%~op_e($;^_{DO^9M%_h3EP|dDe^m zCNg#iFLRx6esrw=1(JaCjq;H%VMpsm{LOia9+Aw8ciI2?76GvFJEsQ5J5RgQ_kSOi zF$zGZqP*kCze!XLGC(r30R>Ylf1T^41vnEkW|EoH-|$A{(f3D5-c0m2y?ITme0F7V zr9oAM?jiH{7vz5L5oAaV0%tqlg=@9G^}Q_myVVqrZJ~&&nmYLVv}m9$|38-W+eb#z zoCTIXn@>g2Nw1|(z>6y7@9xXwfmk>F4{HKhWn~Lk>ZXeX_A!Icc_q#rz}aLBR)sVjnAkUE|EpU4pfHz=plr2-1C`+z$N`XBIkeb(Jd)L;CPYz#ffq z?lc=APBG!fUwNB>U1%A^Ukw3H;582c9znq`NLObo5$q+mvK!IB!J~Ge$t}M>hkGCH zUZ2fUa}AK(=plf2I}MxwwmsXsk>iD1LV6@%X%$)a72rOL&I85HL>#+tHo21DbY_g* zBuxcO)deuWJ9qGnP=8a0gfni`H8m{+sys~QKa-tdUoL>z19zmvyy|6s?~6A#yVXkd z;O48@OTezdy4M4T0Y@a?qTBnu$1E1OlCoerlmPTt2C1p)x@pVZIl5)}x8x0R5_YhL z%QT{?9R*o41HL)CN9z$Nu9jd6^58091KVjA?R%CG2#6LBXg$U<$gCzmax5q7dRXY( zmQzL#ur*pugBLzl3ridcDp+f$C2sh}<+-0%v;4SRSq8SV$|q9HHNeYNf``WpS3St} zhJQU#wH+WGB9Qj*?_Ge^4T2&5RSag{-e>@5vPIGP^9OxG>b@*{%rm4rFlHXqH1$mDju(cDJJm81J$w10LU+4tBf zCk6Gx8Iv4AIG>*OcK9xDtz2U$;G4HnqKux*px6f_^dph}!khrA%aX&Mx6x zenMF#i5G4n9S83^F+7?#8)7-@+I+_37=Epwi~+}!Oix@8s!0^rATi3~G@zQ!q95AD zeD<486m=L68g|$yHXq$hgf<&?!aZc}!Sx@-s9vB{VORrL*N{92m}5yImD7m00r(n1a7NMtgfA9<&^1wz2(n)t_xM&5JekN^n>4XbA(_1vH)L)_n!w%4@|B3@ zt2GI4b>vX2(n>sljJW_w^)ag>2v`R734>sR2ksjHIHy}7!lk=pJ+*zhRS%=H8FYQ-LU zYPyWT^(lxR^vefb^YSgO}1ul(F5V+{drM7<|)A6p^_BJ>NbP`{4@S zP*87`WZ{30Z9T@IiE%18G$7z!3DWdI&wP1xDS=V5-}(WVTqlJ)JH3`O3ft9>#OdIfUZ1PlgnQpEH*ReBkY`k(O$Irn1KEHhN8{t1H#CT3X#1+3hI}sucf0q;2kfi z8&^P;x4)eY$G|g7lk||RFaLCFznEtbbLI-pSN_AZ8jfL*StjZhaMhX`-`xZO@8Kof zBo9aT!q0r(@`fg>O#<8;jw0J^vqoAbDn>MLAKe*x6l|7ST0=`_w;l)le-no8 zYDD;vs5=c>H}gft5-){h>sy`!l2Yz&&H6l}E^^l(3U|jd$i7gg^u`~ZZR8C7%FyC0 zI?PaQvv77;qj0Rx+a0~=+oK?VV z9D7xxGT_-5umhvv$MeW*So$UNg@eaPz45?mECzmOUK!fy=}^lFa44%@^aXDMy9M@o7Eg@&{v%XHFCR4tH`aDM7e8!d|34Z69> z+h+<|j#`jyQ(cYvoLRuWdC;L`qV!%bbRfchAsESjaIgAw<#mtg`N(u>%qxMKJ;Ac- zPl(3pA6m*Z)zr@NhSnquvE_Lk$nUJ#cjY#NYvdJtab2^>#9~^3knoOblqRys+r#FT z?eJT&R7h);{BwLgq@6DN2Ie{3M17-npx*Y?6gl*EpUH+B!^cPPYC2eAKC6L=5Yod% zBgQk%osq(xFjPzY^@DWX;_a_nPez1R+33wO;*6MceoDWF=2vG!*j=?AZ8q2pynwm# zxs){obL@=srE)=bB68)1=G4ttO6^!xr>to#?Bm4W$20rfx;y4@hf(}x!^2cCBfFT~ zVibbU!_`p;t|lSBLgFk2Vs=kiAyuJfwE{(i;{kurqzU4!&pB5YziE@x^5Gfdn0_lUH+ zK6J>sU3>{fXf+J-0b;C<1~upJmPLiW{M>8jF@}^=w5DtIcO8D z8r?eApj6oHtaQyvdfcnv+6QKc`?Q&YE|mI>`9FMXLwk^7fbeFh^|?FC$FI<~2yMd9 z(0Ir=!m%G&lJw`;Nno*ALT0Op7yzT3JiKSvKiE_C$ZIYQ$E3D!?>YR$5hg}N)YE}q znonIE{Q<4p!|%vI)kdw^?7Jk6JjS84PZ_!k!P;-X6B_E!ofL--8Q8-w1QmQ1h0=?~ zByC*#q^0$nImEeXx}71dyWxF9RMA1G{yA#j zYG<~wVq{d(MpQNBD${%W7WLn-y{%}xcT4pQ&g+}Hl4;GYrvoYLdXPoDKef4h$x9EZ>5$?pip%0> zOh(74lIY(#o=RME8O{yF-VUZo^JEcKXHMz%{?IuBoY0*!ne$rIFE2OYRSq6@%jdWj1WL-k zet^jwM74xX-&QfE43FB?Mf==-RO0Vw)Qn1ff94s7JMOek> zS*CI{r@n`Q(g-7R7l62h_YY@VY#f$vEt+PaYV1`w41|XUoG7mj*`Y zF|OON9^TH+j`i)R06_StRPI&Y#mUH8qHs>d6RV=lPV6p*o3uBDMd>+ZZZb|&Dph1% z(Ppj5Hh%!=hK)HVpXoI-Srk-k)#~C7SYD9pz5du}cIxhRdWmM|t_37B2q`r&U@Yz8hl5n5htvv(bz?o3Jkd9*p{6VRfn$xH} z%+4;)$3!BwB>e*E^0^fc^ust>j=Y>G6YnzQDP-%o=RAun$rPpIRvgpD_!}ofd1kg( z!*y%k$M%8pL6k$)DJ?JM{yPD7$R-@VhiW*HQJ62>6)Uo`5lp@6ck~^CV$|-DZ#zSm zM#5Hnmg>ds(ym#V^xwIENOb60!r^9k!eR@Qrn{Bzy!zT2;X$+!rk5CM^UgxLoc9~R z5D+}pFZp66>xmsWhL+8kqGZZ}PfX0?pM1P1vL{}rJevitV7PUBBgk4-)f^T(YFx)O zpnkMoNg{e?o{yu1X_3geq8Rp81}C10@STBnkS(cfa`Gfx8YQj{VZog63;HB? zUfzm9GK+nhQrLW((04X&& zXWx|_dH91WOZF7cM$!7HkSNdSB}$^&+$Xemu3zD(m;0>-8;tyy8*_ojp?Z%sZ{7}h zWF^y!ldlS9MJ3{05_;@AM0yZ2N8iEr;Px=B?4_B#S>DT9TkuWhH>S%VUUpTL;w$|6 zc8<1mDD{h$Ker`Mkt5fi-xa|H?oSBgZF9jY^O`kks|C$8($^OeB4n*4A06&gJedCQ zYAFp;5z}3NzBt-74!k|g86iyXr&_e|{%in6NJ5^j1a^$w zr~i+*%8aYlC{1Oyea&mP;z47}Zb$(0j;#PrG*~hFXRy*c#$@t*pdyUp7!5yQEYhq* zZ8CQ$II>qreIKvocx?5>-7#TY`7sT?yNEPn-6D;nB2h;X$RVnlKe{X! z1KarWH2p`RCfVD2c@ojND}~`%g+jhpp?1_cb}<1#ibUm=AF`DCaOYi8Qw-a9WunSp zkju%Kz+}G|Lw@@KX&eL#aiZ?`vMfjB%`jG+WWPX{IUp1qcTPSSr+?pfPp9FmbDQAh zx4oN2_Jc#GWhfEq3pL7;Z!LOlgVR+T$|erZL|w^~q4#bgYEUWgNhS9L4Bu<*_Dy8( z8Mm+Ds8MA}q7`w2>aNW5J_-~XoZ#2hCqOu2i=wz|jx0XS7=cH({~Y=b!Y=sa2=VIE zcKU{2CbXS&aq5te=rLNVM}d4LO^)O@rM+q*^mM#MtIL`2guwi25#0Gv_H9+;D-nrx z6P=$dm@3##x@_^<1{OWZV1D(q=`bUcIVI=Sb%fj;^EcP8yR~(?r}AFoq$ythHp>UFv)fj}g2SbCj(LO+hYOhK`O%DWS0K z(73Dlug-S3uC9n@z3h`fjrwVw|Nl{TatM_JDqb8F*a)v4qIUGy(N{7{fqguY02n~BE#M9U-XtwMsKD{O=tflY?sN9kM^F% ze{kT29`LS$idF*E{yAN9%IN3J_^j^lw*Q_Kh~nzC$e?5>mplH|xo)t6)8f?c-}z7K zR*>wb(L84VkIrQQ&ee-me&|0C4@*8*!6?W7YlR>KNdq#av><7`{TBktkS`VXM5y1t zzlxJAju}9Aj{kG161CIZdhe3ETtM`a(b7a-fBAj!Gsz4$5QAFU1Z+ff4CDn&vfXk$ z+JGum@N}XFjIJyNV70y3BUXqZw?>d(!6 zZz%7++Xoy#=6~%B%|LEZC4vkpGHa>7eFadX4}P)ia~hj{V07aGXiDg-R;SEjG&zEN z4A7QQSf1QBc`b{504Dih_P1J}hetxm#U-pQn+bf8G}yC^H`s1^gg29|GO!+x18HYu zS4by^y9pu6my8j;O#d~=R7Zb&1$?iMXDz7i`Ue3sxC;&c!$MF`vf0Vyi;06F#Z5p2 z?RA7iuidk2Z6JfnNv1(^;6ivskgG4Q@)6h%(<%-3WC`7ZRs`W5aj|X%m)xGa3q#<( z@wmSaSTG?D@?*&cTuY-0*n1jV*vBAGd8eTA^&PX1?*alr8F1lB<*Thv4hDPX%xm4` zS{*O1O2rl%uz?^RV^ajD1BiPe2ElD!20_8h=Ft?+7;Zv zSAD&J-XUy5S4!%bR*OsvRsaAq-ChUp2Dt`r=L)4^Mg^5D+tc6$!SKw*Ud!qt!CS?r)fRv=M zG#*nx))IRn)YFZg>?(=^fEp`eqqjBYKT|h&eoc5B+&kIa1R>W3X4#3fW-boC^r2^9 zFDqlKXdu8g@44cy0Kl$%Jsc0nbl}>QEOE0^O&ov3?E+uM z7CKw5KRMF8+nyH7OJQLx34Fcu?E&%|X_r)0gJ6xb>-M+mb|z#O00}RM$^Ge9h@%PX zy7BydGuKnQMJ8rOYP-0j9v=40Rqdw>$@>NjMyw_${hriF{P0VIi((YDiIz5@PI|Ae zSX1xr6bb%n*Q~6C!}J8kwn`&EJ!t|)lG?L*0DsG+fs!YA*A!To!aN1e;q%0v0Z}vE z34f*3TctH*q!<(v5c#rc4>{rKxt#<2F1n^JlJ;TI^zUCky>exKM{NPTeuEP$fB8gm zChb=O9Xuv4n$1~Z^qY?lWrUU<`+vT#}vf|tkp!Kcsvab#G+~y%Gs*P z>XleKNc)zV%Nfkv0oLG{cEIeR9>Scs88#eGyQf;H(>gpYT44j z&Z2N;g7qO7%!cY1dj5Dt6CanDEe%}pS~ zy*SMqZQg%Rb(zOoE(P)Co)IqaP5EtxY?I`uFmLc+G65iu8V3qy8#(Rh_6lIXX0;tV z1Bw3Bh@X2S>P6!#cSn6^b`aWGu8>k+^V?wcjDgLaVQV{|k`A~K-VI~t+f-M|Y2QRxqX+t<&xmF~ zoheu=8#U5s2L_7J`U_^I@F!bCzE*XPNM-f?uQu!K08ka>$eS3R2BoPln~yHP1f%6L z|9z<{>mk3KWR&$7a^MQk$(*NJUfR97Nv?&kzT=lA#rs?MyZOamGP*8X!+t0nzioIM znP4v2I=7~TI%|@RF90@GvirL2EsBo4@;7c=mjb)W)X`-Bj!ZJ6Ll|EGpN#W4B%vJP zj@_R2?~{=nN@piLc~@twlAt3EM^fGF0i;C5&jDXBBS`$#xK>H4=|6Bp03(o1Ntx`u zks@tjFTs6K2+$oxk5&Hg3{GL;9V6cDl;fK9zP~#QOl2DB-&Z(ILFYsDLWXfXfW6JD z*69E&7U6o%+TEw|jTTRTX3NUp%&9%D}$B1f@ZUH%^t5(da@~`J#peO@iWNhI~65~(nTEo zcwc$2Ye6u7`u)xK3v5+$!rx( z>w@zPPimotet;l*G~u92ofpMjcRN2ViUkyow*#qCNg51}oE1jDUeufMnC8<{Y+b;B z!BcjHb*rPB106Gb?ZLwVb@|?IA++i27qF=!m`+8)6Y#hdF!*C-0|m&{bg>P!w{@x- zIUF9(@Z3G#qBulvwF0+N8?qJ=42BTpQSshyWok71>gMIFcnr-n7SoqP9vq@JD8|@iBVk`r!mwU$Ci2r3wzniEcBgV-iHHq(C|K6 zjVszn=<#)vEF+E>>6~&1w!wC`ffsb2#bg^Q-zqrKoNW&C*-H9=9dtSsX&~;n($Vpa z@wpKF36+9da(1!hX2N-~Y*dV}Se9vwT@J(+bL*l6 z%3m&?<{i0nARlSkddJL!|u$F;1KoqL!ArY1|)g z1ZiIPuf>g|eR+6JD6;nAI{P(QY8*o7_}7-%R5h~dJzQ>L!bZ>UQY6g0Dzk@gE{i&l zAzQr2D*L@E6p`yDuQhsIz1Qw`=ofSgO}~33NfK?F(V$N0tl#ZP#55!X+CO6trwyu8 zp?Q~?ltH4JI0sRp?djA$xqF{_;Z$VH7ry(NW_c=43nzo2KT$4zVI&kV|CAVM5qIt#9BFK0qMq_^WUDuX<}-EC+0wkkHq*n?BF zqcT`}H1~KCf6Vu_%vr ze4EdJc8Dhx@_#xVv3CgsiT5$Z5zcZIDyTjzmeXNtmzp>3xT1&6~`Qg1`FsT?(q7Tziqu5m z+vVnDsHZvQy&%3Fk3+UO-^kS?v84TqU!W43?L*iP#tNu!U80=`o}HW z^5qhl=h!)~Zjwkqu*1C~LE{n6nobUWF~x3LJI1&C{_EZ;6|Xx|8(ybD1mI#z1zIB7 zu{@M;`j|s;^o$GleE-3_s+6V?>=;+C?ho7#_KXY-FoELAEN;!3#rYloxfbm{3nq?3pwarpR~XmapKA;5+gm9g>;1 zdG^s7DkjpM%4v}1t#SN4MZZdB6JgeCES^733;67-ZRea^E30dFmugtLmU80UT&esq zL;=NA(giQ558r~8u%jInn}e9gn39paC0(DUn5_%&unNRg;^z%9wx2!x`caJR&I1iX z_kAo%EE{&m#kofMjCqshjSkX|=7x$|*1DPIWnKKHRlCaR4-Fkhb%aL67C5RpLoxWM zaMKPN{zvo5x{k!>q(+MPsO;0_?|)F~WP`N^pqC89va{AH#_P`E2<}LKa&+>`zjUi z7Qii6-rk+RCB%SkAUgV=NyVi@4qU6dAh{Y;p!NYt(n#4@~^?|7s-@=ARi>7Bv z`Iq`dyYj0~mi+!B%)D8VHAFCA-b~J4@Gk`AJzjG~#P!kM)4%21LJLLNp*_q=NEnuH zA+*r$Qw%+|_I*F)?RpasmBA1-Yw{h=zjtW!wB!2VJgSs7w#6-tBqD!E z9XmDzLU~L`C1tU)m6Hh-WVQW-hfg0jIczCLV;10z*rK z*B8_u2(R@52`8l39MJ$BO`7Kinpdz}4K)RVz#57$*6S_@7g%Fhp%+5C@Yg{E7$L;w z1W;sr9E5cmG`TQ-8j~TEy1=)zqFY$EFQheu36L3Oe3ej?^9U#2my)mrjpV~?aC%^f zBANtNNlQ$a*yE4|t-Q!JCr}Gdv2gl9l7l>Lfkc6#*b`+gnT5F1i+F0=f>9NU-TUnn z@djxLS#_QGlmsnLiA6V=Y6MSSkw=rZ6s*Im68$=iO(vy)O%t$_p@c1m5{}?C3)>mt;7%7LlUfr=`O>yB^O#@ zK`vzd4~sEk(P9w?{g|7QYsz2~DCY_>bsX`={JQFy=S}^M>4WwIf^?9%oC|FzPD`jw zC}*f_s9?{_9&U@^QYC)cHeCA1vcASXUQ75h-ZR)U&Pcf_W|5>GCAcXvEz&)fJvN8< z%Y+I!L1w;TRBKcSU+X8|9N)BuRGEAdZE@DKWQq}670zwCZQSh;3f=E2v5bQ$Mx6Sz z^BnW+*D5(Q&J5%6W*oU7GqwPkKc62|5rczk;k1J)YgF=O){}bzd(!E2Clz?)cw}9Z zEVDNR`t4MN8Lb$p8LPB{rlgCbRAhAsW&=lx+)5A1B1+TBdPfXw@NH7vzPQz%<8E_p zYZ5XN`lQ&Wh^L^X@Udy>CV#1AQx!1rjM{gd^`BLLv+xE(AZJ0L5vkF%(e+AXKWAV3 zitb9`O7cpbj31LahCjxMjFSvoVDi(iPZpn41+E=T=X^yJA+ZM(-?Iut26vXGgB!;h zdm2+6DQEU)$foBee@_!RoXmYKebiFa9n@ylwTt*}n2l@lrK-K^g?GfU$np+9Az4W3 zEA~Q@f|z}Wz1sujORQbby_dV0yNY|&)6O#i_AmJJ$Qj5Fx$T8W1ZD&<1dS~WoWbKC z%;khscouACEcoBOwWP3ucDMB3WJI>1jxwwkt!xFc_RsY4k2XwQHj&Onze}}kny*-- z5cK9&=c(rIbbvJyevQKM`}O5m|CnbEdzO)icUR)u;hM3=ie`!itwyxE*LJg(S8sCG zp*N7lU!G(4Z;s5f-}w#QeT_+c8Td^!Q!rDg=~olQBHH4j7lD{r`>a=`SM?F+A>{Y{ z;fb&YWiX|naD$kytA`tbn4oa1FsUe`=#eW~9{aTHUgoZTHqUp{@8#L*VgVkm9>cC8 z$LDF~#g3mGvk&TTnJyp~*x#t#roFA&DP5AI-# zVyM!KmIZHZe%Th!q$u1WH)ZC>+?n6xLJ>SZ~n7L*Rp(k);gG8(y^JF(QQ4 z$TUTWV+f#rLwgj!ASWWz!~M;$#N;avRzwi6RF*U8yH~0n;!L3TMKrh8_E?niWNES$9#uaQ2{Ib^N5Qs5XRKwJWn zte5jg^%;(oQ?pdFG}*B8LTOKwNbOTuNa7!{aPx9Cb&+TlYoQkxGDSUcs55d?KIi*N z9v~LL&`)2?zg-;2%0di%b)~+XE1E-;sF(elQ-Z032R=6;-BqQOK@Hz@WM+75^bptZ z1DgC4M_NU;Vd2ILS6#v64wiNSMN)@&QO3`S)C?i5$E}2|)Ts&}iG-4L=CK1~+pljo z>q>6F-ga^*f_6BZ&GMTqk3M#%M5iF@NtY+;HC786nAE%n$E<$()Y#YP;221S4%F5|4i(1(8s{4vq z%oN9x8-7KN-F53P3g1|rX}ajRUEbFASv?NI$2I5X<=P87c(|$KHzA>E>Ur3yy zbBb|15+QW?(($daa!i6wjaxswT(>dHzkH&rC?|Wh^ z(8^Z=ud)NTwe50!-)iGSbC0a{eXpxafi%aGs_E+HB^T)A(%!_+%gkQqmnN|$--NQ? zHF=47H(!mNh%6pV`mCp3SF$%7HalN(?jG#L?RWCUbJKM6c3d33yIY>Wb~|~wWOe~_ zk++-Q(>ta3I*u(s=)Oa9pGAUiSK`O5@W$CA?e~c&sk)~dapTj_^P~y3?r}Vj$ z+dCOB*h*lgo1w7;&&DaO-5_ zVL=0Sdi%~@6f8ma&ljS={qtiG9nC+VcsNSX8NAe>k#%;np%LWb=HjN4M5Cdh5qGn; z71fec_}}Efza;4FJv>}QK_G8$Z!T{>E@wA85RZt62#A{(#LLSGe8K4sdFNpP=6vT) z|38`hCm%T*cPlps7Y_&LcQnuWT39-JdPvaGJs0%fKmSut8?eK_CB1Y1-(~>~1U

p_KpB!e`~u?teEu%#F>+A#+=^^>=$oy~OzhC}uLUGV@%m3<$|LNv`o&xP0Rcc>*{~8<15ZO8QKvaKvxQOH9MZkcS z?ZV;ypNl;S!slKX$Mi21ME^c~eurak%S{3o9!P7B^1l~;4mjID0ge^D%zvMe!2N>d zqx#>x{<-Ak<8osg$kM;NBK>db7fFg;7&T1ea7ce=$N!ui|At=fzuECa5MFP|ubp&0 zRV6e)E`A9d|BShlhIz;oaQyW-^Ih9V-P1`$>el;{PP36rr>37v@7j;_YU_G~SA0$+ zuFnRk+pA34!<&zm{@c7dl&$l?$Gd~-D!=>R0{*=b*v@rbDHj_jU+=5L_KK25hP9G^ zuSZZ59b!vdP88gGcHBH(uPzgt`L5^q-#N^820XP9rG0w3-O2UVH}TZnclY_XLj(}q z{63^OH~QbKM;w>r`nwsuYb_RA55->s3Ore8aM-lXuu1(KZus-$ez!2LP15hK1r?tu zIXev_g2L^C&-pO(^0?H~;VIKM{%N&FHfGN0qvq9Pu8vEs>y0STXU4MWN9WGRxyPH$ zxOeMe#8uiNsXzbQG>+${;r@OzUzPm;K7;IJ&vQ2YuuM4fyxmS~H}U$}&)K}>_O80) zw;TA49#D0mw?DYM_gZ&y+>M+&8pSrF`53;1lfT>DSg8l<-B0N`qxo(qo;G$T*>l-* zUilBG^I?k8V?x?3&W#nhnu6c54}}LU;AS1U_1F0BWa09!OlPhY$|@eJ>qTd&;m*0& zj*+$bw6iN6KJ7UYJ|p$o>JqvmbSFaz^uuhIJ?Nm8qo&1IE0F1zLY_ch$4k0}AYr-< zQu$Oo*z;X!Nb%c_kvJf4%OsDUd}ObRKM)T9tM`OH-d!baHOTL6rhtzF8+7mzai^VfzcU;XGP3etj+#S}BZ&6W6*dP9=JwENlPCIWr5#SsY zS^>-6`Hf3ldz>m|QTZJ5I8Zwknt|F6>k1fF{jRfT27Md?2pMb8|Fa6r$>eF$U{kN& zh1Q;aH|NIv{>kUKbr}|!V(q*5h1&4P>&(3 z3$BBd%R$IgECZP1JQ~L>n^^NrPDaBQ--K3Mg5Z(QhdA4I3*L1F!UiMGXI=EJjjD#8 z)GU`$WA^wTTfulQcPL_Mo)+4y)oL?7pIw_5z?;X<8f?6`G@YVh7`e@1!#4=|Vc z$_Mj$VBl04zqdADPN|MoZpKNOVp!ZAb|X_0RF{68h+7VY93V8$bPM zQBl<}rLqBm_c=w|tM2@RN1y85@T5;hXSMUcR%Kcgj*zDd23GzfaY#JFO8Tsx(Ki4z z^+z@%(60gOg=p#2v>!DrChfy{o9Dr=uTOt`c=q34?Qgev?9ovXLLIzUASKpLKkJIY zUYBHr@bHz2aa8A76O>1JFLQm*BR6kH*sBlUeZBovx-T$qtc}6(P6K(oL@k4E+_~dQ z55gQBd$u_kb?sc$`pZo=0xRa+JthJR{=@rR@CGVG5Lh58KVD)g%eXweZhl9({Y}*3 zk=W_Y?3WZ0iBa{bS)VDqjV(C$7U7TKrll62TUTL!`g^; zp#S>sLy^(wQL#(jeO;qcz@yR{i3M%sLVjf|SAgJU@gMuER*3RYj>tX%j>bd@vA^3ZAub;ABoXbYM@R^Fm(ihJhc&cL zknb^yJ*6))CH^kvb;zQSlG^c2aIr<-%x|`~JkEt#0Y!nL?^q0Fb1y~TG+|W+os+?4 z2>uBfK9E{&(Wt>5H%yrM<)prz|LtN8vGr27^=J@^omDh~=xE z<5@p(+WidXx@%|`d?AS}nk)f`u4vt;mE{b1oOa-#eq54W>N~0x8}93V`tTqniT&H@ zo#I&FaeMo2LDbgcX*}*iaSUzk9A;r`E}7rDyO_{__NC9d2(M=pduK@^(_(fId(o>0 zTUqmzj zPmp+y?ELDcECP|ppsVBQ&;9Y`w7U9mf}hb@7kTN<&(_amuR}dNmkBL&=ODGHbWsM5 z3ig-4Mv~%f`AEDzC2S#HK1n1AZ-v1%wrj*N%sVGz{7eDV7z)2>ssdEc|F zE8Ebz8w@xk8q@mbRsThas~K&J-!tyfBH~hSvAPyRu2Ix6(GM^14$5fel!HhuCdB1# zTz}f6dG2I;=ZIY|xoLnM7B>Yl8@3Hau0>cOEXri8t*E zV+8RwznHXp)!9oTkE38~#fAs=@@HAW)Xwb#+fs7Fx=g^_VL}}B4z0y$ohjC4)SsC~`%g&zhQM&z z&`Kf!HRWD|AWwxbM_v8IRaWAC=u^1yUvEuyR{Za7@mii{-n(o6&W!}`kfIO0H&*#; z+jqDlH`HoWByAET_7V)Q*MI}=QuDo>qKBs7XsfqWm5*qYwLX7p!_yi!z$lZo= zPx{gbrKUc)(U7CTZoVMlKi`<6=pY3lp-St0F4Oz???V!}82BKhvp@l?(nHR_p=2or z0O)j)6xsX#15p4#7_@|v4x38OrfUD6`46DU7dAND@D;qmuUY@bms<(|NL$3paWQ0$7yE%Wmd{horfd%XI8S8 zdrv2oj&lzHQfhnN93_AiC_w$k8~G2pm{-L!B>WPNtWBP|32PN;mX-5A|rm(F|p#6bWW)U$?YabD^Y83x$WUOquqwlHh1SA51f zzus(|KO@Sh_W+#r|B!=8AzZU-xX?eXE_I|N_HL=Afx+!*CpW;$Ea0(`r6dvc;1p-` z85O^aWK4~}Hh5dxzcgp$Y@%f`m*19XsM$5sF~U~f0)WUVP5>qycfn(OkZ?}pExYt$ z;;OT(x`-dv^cu@vmRWj^ayFynJ_1YheR+3Wbar`F+sK*v;$~8fp}(=TAl z-bgj`hY(X+o%zoC`b{Y_4;xsDcaPI{0DvO6Kb^;_XOU3k2>{>4hX+;dT;34Kw#MtA zA*`Fb1Vt)kRm*EtnywZ(Z0=VFG69cQX^R0*kIT>dyPmPD{Ph62GXbIZ0I8jlRR1lH zS`qKT7~eRC^CE;4WhM>kbdF^30~xTGKHxp7_q+K82jawWl6R%g7SSY?CI=Wk3Yq(_ z_s65!Z?mVy0IX1@o-6roR#)Hi1bPv>XcZ^1biEp2R>C{_4Z4%#qq0o?t|eb=;{$yk z4E(BtF3Us!xii%D(g=Y2# z?*TktUOUXV7>djP_D2oNw-9Vel5G7=Wm6HI7$;{8Cz-DW$Rs76`F1%Fg=q8WlkHTa z@7aF8e`>mdpr^r`lDr;(HxPSVlFa~lH$AoRXo;||y7CE@Xzk&LRb1lYZ9Xpg zT`$XvIQ$e^^{rpDh@hm%VS~zfFMFm*ewj`%oC4ghBf#krjNm~k!+Y3DQeTorreqi( zh0hXv-eVKPPbULm;fszIK8u||wR6rQyt~qSYI=KJ_sdF{!il7w9^8doH{^yDA;8S@ zgtB`!zFQV$I{{#`SBK9vkufWF7T~3-LtYtiwe8Y1-~P%Rb)QNwbx1SuO7L0%Z^i_J zR?t3>Zc5%C(nGoks?``v9p|ffdyCa?H5H-iz(RF(z_)0V#g@=n=z9CRvU%m>CymVC zXH;gsXJN^c^>*ss-vs7cuqAI^?2A5}_7>LddEX?9DvgUALC;6ni^tdGxAl2bHk@+s zMVX`?J68SgMeYP*G_Y_6h)?`SSW8+)x!Tq49!5c($#ww`XAvY0qN!lwo=+-S+StCz zWEi3tv`V#Miu{9i}FZ*j_#m~>{h`rUyW5ju<`K{ zc3Wd*m^qij$&%yzPkd~9jWHztVjM6REUfb;-X`nl+sqi^1GCTGtJi` z4~@bvw3sSV*5Vq$EJW?r)VCz+7p1HKuas$*yqff(bQ_|L{V9$SZSMEycE_aJw4+GH zb`qmB!}ing*$UVQK$Rq#yC_r3UW?x_DMfY8*Qo|)D;4a1`~rMsu{OEk6_4qc=Q$`G zE4XmZIkl zwcuFIsRAxe(Mz-=IU(u$mMN4bDX*N5!#%O9{yl@h;&M!QGdeL<161_Ga-xuBeUFIq zb+Rw=u)(Ml3js=h!>><{BcXH-9mEPuRv3SZkD_r0eCG@bjH6SqC`FY@QAH2}m?2+C zBQ6;aSX1KZmwHDNm?BiJs#BA^X3oNRnGn#&Cz{`kxy115X7NjGwmU60O_>cvOurI) z`9)&|!-kHxm;j5zXpNHe9tSPW(VG9kM1_jAlbgulprZc!zG=Xt=Q=0T#3{DTnIz6m zv2{d~*)}F5-ce4|HV0z6CwnYiUfJ|1(<)ZvPY25dXaF}f2LhO{{+6{*?|GWWMRq8J zmOZ9j*@eQKN}Orvc-XvWlj6=09PWS}(8Oh+{H4$D1xHXTmJK@6-jpin=yl^8T`gPs z<}L&J7iu{Z5|{W7x*zp>??|k`PebZY+EN3=PT}({xy9u&D@uve%IA84Wqs70dWFhK zxi1NcOgtuog*edsy9hdJpnN_ZQSURb#mpmqm*m zr1Z`*_8Iu-awVg54Eg-=_%@?%sZjDm8|yqaZ|i5@DIa*nD#we#2+Cr$5zK%Um;~=4 zHK{08aIwiLQc;$pC};`YKb-y6cc|_Z*!Bp-wMWN6d1;PQQ~oX_67M~9BJ{rtd>ZwE z^b;1-R@)s06Bi`Tjepo8RUBf^2o_R8UqLqv6*~%s(xhCPq7DhzXnwxl9TEt5I8-_g zqIbH4)30(+bQ&K2QiRd39Ov((O}kREGLg_=nxNw8pQi}vkWTvf6_;Lt0GfdR2)(9# z(`2cf9e>W_TeKw+qm_HTLU2{7|02;E*hx}$8YH5cJEoEvmfO8S;LZ9qLceZuYqvPb zk4e`$gt4OgERZS^SMl>9`eH7y(uohfY#pENF1s@c5tNpGydyZ(&U}ZkW(i)be~vvS zSYIxY>40*>Y0T%PUqi}W$0-Ob&pyA>OG3Eb`@$`b2l?zz=G(f+r3+O|Ao%MHR%xqj zT3{nfGtC&1a6TRRQREM%(4v~D?UCqLxEJJ#80lBA8SKTDQajTUasmUc%J%DRDuWk8 ziaCw}dtja0?GEU?{P|h+Gp*E{oVpV7^Yg8}i9#lxVq)OgiPi zz3q-Otdm_-7abzUPE0$4y&VcRo|m7~z%qbXsooKrD#r4>vM<5D7!}wgfy(;RR41NC z2`O!*HZ80BZ@u0YKZiTV|5V^TdAX~ELZy8hvLqeB;sq5VI5$chQwV1n=l$v{CpXdp zV5F#Xrai;TxQA%JJM{x(YLx7L32P(M6{V!<^zzZE_UG257EYAioklwH)?FTgN%@L2x;U)oa#sGj%qq zsK}6hGRQy=@7XS3$z|P zEK<}p+Bg9pt&|Ic&0s9$_7Fyj23pbhL!cYrHJ2>GQ`^s!YP!uxjR~e)lCHn^A1Uv>banc^r8RuLV51XL4*HKWcZ0~!-7 z6p&iI?YAuoY*;GTfsFSU+tQp*I%woq>7|Ou>U>2HxMp^J8?GO|_PWD`lLi0!y3r3I z!ip9efH8F*z=10E!BS21A3L0XIHxsX7XbG5(W_^wh01`d!Oyf-!Hw~ymy+!hvNPsP zm5G?hxqk}7&AJYC+cq*TsfF6r?qn|n-mE9Lh}J^^A_gE>wzGh^mG!x#=+&52rU z>NUtOsv=dR-BAt`M$(5|evwMBeVYo^Q%X?^hbtKu`X(HFv?n_TL%O)})|h!1NhvpG zm6?oYg=)7sit_xI&})^}kDB}_L?px(0}CAKi7qlMMLp-|IfZNX)Ka11PS<>fsQT-l z`&np`ht^P$K}R$dYnA@W>O+zJvlzm<(ME4;!$S2O&`&U^$iioT(;A!rY~`RiL*EO3 zg2YtVf&Dl0jg~DL!@cGuW;8h#95{H0%qYGxXgqrBL$6upB46CyMzxWcC5ONStIMbu z)#HCg9j{+dv~O2UUHZWD>Bz~nV`dKu3cLn!aMcfZ1I(6tU?B`q==#=X(#7@uSmS*9y))%Dbs z;wMC7gWSSC1o$4DNq3n($Uq{OB26b*Ms$X7S$9NqAd2f}R{s>)tFV_)r!&W7YR~EM zg#j03nI8df`jUHsP*FBBG+4HAz2LCFWirL_&k#DBEi|5+nb<_cjyJp?X~kBDVAl<~ zGEL!FA!W9MpvfQD%0X3?Z}zmme(p?uuvi&6Y8# z#0|vxIW#VhPwd0P;CeoEm8V8<^Ky5Qn4^#qDQ@0MPt2rn*Re!3Vzuplx)Ns5`amM= z8DUCyTyLHtAlUG&y)_%{71A;qdj(B6(ZS%UCATV#Ly=v5iI2bOm+f(swm@FK-4eBfn6iB$rx^Iw@9q@~ve zy|Zn^UbwqJ}(nGN7R4D(HW;~N)`oVD$M->f&bJkQVl z*>$tgLA!uf-J_TfJ5UtlwVz(_a_q-NG-Hi<3wh2YZ7$tMu+U*_{{MiBjs_bG!%XG1hUdF&Rv4lrX7VOHZkz8RsZ1Z>$V>CKa%1 zUFlw}8>l7yFoENp$SAVA5LnR>4@%H)eEj)6IL z7|G_5)g5t8Y_vk@fv2w; zC}m@C)aMLj^t!O4sR=kYlYNv2n5eL}F#3v6;jxPmS*euWO1XaY4eFCUIX+I2JRwVN zXRpBzP${q%?OhH@3^)*Boj$xjs8n>zxY2qFStTaROMA0#v3Z%t2w5vjqirBmgy-XU$Ipy zN-2CuWvXY@zvA3!>rHq6mdOw`wNIa$_<7xc%h{fEorlZ0*UnQU_q&huAJ34P;$)`u zv>RbDY955-T3M3QmWkUDl=XV89S`JsCR+{V?bH#YMY|T=+b}dHjwo-NU%l;LTyZ6t zDAMh$DAvejcgDsqm3XIBF0tkX?rSVq&unm{&=d?~x=JhI7 z#PcFAA>;V|d`- zk{t;I8uCBGH^EkNwzXQt=oFn7KX^kIy9E()lQeTB4+) z)@o2;{(aq6qG}t!RH=is;{eaU>481iCe5Ttfo#R9?*p#DF{oie=3Q++QPgo78@~@&^LRDyxK-G>`G_}O zp7H0;y)$w(2SDp#RV42!o`yyM)FNf`5M9(Ry-4siKtQfbK3-}!_u;9w06{U@>&{3P zw$eyrUqB9U43It^7h`;JMvj$@#-pj6_^VkwR{+IKECH}*CZSINFSoh2?z%b9+T^s@ z1~m2OH}Q)x`Mu8jV=?9WDHZlsL%VF<9GetXx)I;K;qC3j%5u)oHyJITjIQjVZ^35KdB`*eftUZJ)UELOL~$r<1SL) z6_p=U2cT9;Y_(Fja1OjV{x1gjz9%Rv{|+Kb4D)04lYRu`5m3j;f-oZ>-u>~i4`3zf z1-m~vR+Z`M{@Q&alwpb?6Zlf|(>9Y3>dMgR)V$oZ7Lz^Z3&E=Z0?~-hr34RA!wI6l zfv{c=Ff>Fq*9!WY%L`Z_>q36-C0E6K_nq&PjXBVpji|(VY`G3y;a}-iTU2|unk$-$ zl!UW@s=|9jVu9$-ap%*Y(&!?5X9R-h(Cl#gyr~Z(BP+OO`4Je9L< z0?0}Gr8l|HvWUbv^*3mM+#oL*@i?0H#_oJ;SkrKYg~&7z8JbVD4ASL_Bw^m7r5u#S z5@8q_MRl6kxs=#ZmwNQjBx^4`i?J(rSeky;74%p3)D5CZP9DbSWTpy#}T1-8atxT!Kx`iQuQf zH7!X6oGnguQWg>$7JHQ9iDqXrQe*AUY6(6N%$n{HWYt>|Ta}f1x@#b`Z`&%rjPczK zRc%*%;gd}uBu>rp3^JlLW`HfEE`BBl$(8zZQwW1!aAS<0g%g#*f(XcMC!c) z&<0?M6G^Cy9CTC_?5>L;#lf7aUe!ez`u^#@*boEW*415Ls-GN;rp)nCLT@Z4y%}$%)EGH z_K(kLH*zt>Nio;H`pFRo*&u`HpPS7vJI0X5vN!+qzOUW_Fxid4us>5Gj)7Q&l2@MH z+NcJIw*2mLk6ym`%(>NyMZNu#%qM=*@pk$UBrFORZgc8GfTS$A-)ynA$W=7UGgFAA zU{w%Kde3+W7cI44%1?NIjGy)9IoQ$bG+(Sryc1U2zVVSgOW${6<61#YCU#+UDxSas zUIupp0Lt|1)c^@egJ!XYCkdFOtlaIElAQNnD$xAw+UR42dv0T6&MUv(?H>b#of|-4 z>v~iRGKb2AW0L<`07fbEFtG&BW~S}jKw+idhnL}SOo~KeEfBtFwS$>H^?X&e2X*rpC&+I*ugGB#JpWBb#eizJGjm~6SwcvSasiRMr+ zRN6E-ocj@oV9~8xQUyv`DBZyqs6R_T);U{oA6Rv{)?sBPfXH(S1*NEm%d>~-^Kdq} zy`YY={R1%VrE=DU>+Xg7iZtZ`y+T7Os^F~J?}?-`s>MU>dJ+C2yLlnjhUoK;_t)N; z&R<)tWkJtDxP&F8%~iKSjWR)jVmrX5ZPSY#H6$$pK(WyO6knuPapUHnu>#x0K12)gUSQXUu4|WZi%9EW&wU8J<%pmp75Y8SyeaFd415{8N)|xnA9`L8FQ)K>zZ-eJkP0yyAo|mV#MSlmRK<+abJ}BDf>j z!u21?2*@+&Y^?)B6K`%I&`HEbF5_Swp~BwEUTvRDV9vXLzK3=T+T0B}$*mY=on<%~ z@dRXdWfdLtCS_1nP)EZiamiez(I_WUOcg54A8a0bnwy^X*yXa49JZuzK+L5!7uVgX z(2p`v!oq>U$cRBv(^r$>IzW2mRrCQWGzA&2-;z~&f0B_2NS>@INgU540qx9Sn_5umoekn`>0Ly8*+*0_@N`sbPlNw38Q0+Uf_5_ z7MpV_^0S zRewT`a)q%tu+U zR8xShIn)5p%Isri0aSMZQ>`#AV`ccO%dXtK4~Dyv)1Lg4wY7qySOhAU7_ zq<|heS0+U8F871R$e0$*6_`7ahm2H5vWSL?3Th3F(By%;bPyhZeDNDEV&Nb2IFlIn z6V|^oL9`JL>hkeA>9|Ha2BJXVt!}O=$iA*#E7)|?NvhnTS zf7&;)E)KI>XHFJC5+(9A7QP@R` zL3@t-3bQw(=p}mXM%5oc3D{teVRI`VJn5N(A03Y)Wt=-J_r-;$k3h(l2F%*yHElbE z(<2|h&T3I}3hxsOgiPX0zg17;%}XkHwL%JI{)o|1_yhP!^@p~dDp;JmDGy)`^<=D1 z3B^uN{WtddT+H>X_z8o6{lE-r;vXZ+U_678`z%Xiy0j~@sVIHh)$Z~flc!y(HtS@O zQXrK|E;fq>Ax#32RXrU&aDpz^0Ac=qJjv+c*U)%;6bC4wV|7QP_}XsS@L<; zA05hnrnxXXCqCF~WdFN-r-STGh>NFvAkS?Gn2NcRY3NM$SNG~E7gahxn;k}Rip8}f zU7%WzuqDR5XMZGqtN0JQJ#;W2rIPu6BY(R=LN~P|bp64&Tf^>F%lF#rT3DiDV0r}qFSVY=9npeMbR~?6VCgIaYdR?%^I@y zS&vW+%^`yd5|iZd3gbv&DJ~mv7T;Jp?Aotk8J&Md1iqGrWQZ;1UlEhBLKZYu9Z=4A zjkdJmR7iN^*~B~4$wCSE@MTgTc0smT+!c7YXP03){ZL3FhQ!8lS7taehBH|(EXGgeg_auSYHC~sMNk}N z2_$d0z45K+MCOEV6yXMnB3Fu%qq`a9<3EjptT;4M+J-x{zglx;=x~dHie#XqHkFRU zhU;q4Ghof}u!JTm+Z$8Ve^8BNY}{#YNHIz(YauBHih*$)e;h#e2I2;aNS@+~&*ma5 zHF6UMM2rtMQRxcopO>Yibl0(r7grs7(z5=cmN~nqSS~-M&8>hGB6Tz9hFhn5+^dM= zL?M;`Ve8gMUK4e_so6=F&45KQS9l7{zh)c$E?09dqmFmy z976aoKrFIz!4Rz}rI~oP7vn@4clb8!N3oLWse-~ODY=(W4zLxR8$WR?uqt{>+op`V z4=`GVK<`>fykuqC*b_~(i?;a~{PAc!OS2~0jtu;sU5PAY)iXFKva_o-GKHA?y7IEg z_#(Y33p^!09`6U}VS2>)8oifvxY(u=_0DPq55b(p&M-9-F8ByE8(6d$k(hF`&0#d9q$E{gPHAfeGO%X&DRA!ssS{V1^>v80 z-(#yJhhEfRNMIv7CkwBKR^D6O_cFBki5~G0d}`7m{q*^3+}bZ$QCuQ!YCqW9n#83} zKo^ad8PtO*B^#88s3=f@&-dufm!m&D)lA~4;!7RdupmXUN;0d`KuE8aJ3Y>FA}nQp z_t{@=lDmnpTKkmuu?6^;q9u3Mp(*lz=xW? zT?Jg~TTF?tma$hGt6EkpwRG)Zd1$ScKPP%i#Cx0*_QZ-T9F`ehz1aj4ogyAo7Hs0Zen5(hIX#hZu)cq;}YTuiiTJ0nhD`<%(a0iv)Qxl=(Z>z$(NkH>uQdK3M!engpE*6{Wa0zqLolVv@;*o|r+ zsu>G+@n9_D09YDAqyNLw@arP*60ZyCnjdyx{&lLN6!gqtfUGNx|K&Bf5wrkWgaeVJ z-RW=g$M2Z}@uPLz{#(ly9}KA8Si62W{hjjZ$CplWXw@!&;UZ{4PvIv|DmUT1Jse8&ml+2-#WrZ8GzJz z-EEukFC)~2Qz`-QM6*<4qYi)fQspzFq*r6e^mm86iFDECG|!R&<9geEL0alaNJEs?qNko!7XV?ClVrD*4T!P|d|#(FDaS44I##t7 z@Fu=ex&XYzR=`ync?3kQ?|@KU=^59SYI(@@)^al7lI;OL6T1-LC+=51EuPJr1$Y9U zcSGSzK&YSV__NODjS)Cb-xMPM6+wLQZM|}vy5wzN2cW{$Gz(#We38q_xV`zaFA{$l z2y1{KcG;eKLZck@_CzAKjhN79H%tgPn=u&R3o|YpM@aZ16jWD(k*|DE4bV$JByP8o zjsZEY*X0v{hp23$zOTdHpqirH6a3@_PFx59;UpoO5dtB=F>VLsNs=RjE#YY-ivYDy znF(-IQ8|jJkEq*m@JPxwMz4rvZ@H^>rKJ)fsCPz zQ_fZ3D1%ZiO^D26-h=!KAg)u`k~9;+-7xs>HGrTnfW+bfZuW%u23TjT^bq3DPgprCOBlq5;d>|hb#H25q72a7=P zmI?Aiu7wIQZh!!+0uV%w7E!W<0pfZ!q?b3Q&%UGB?;lpZM)Er(#sD_RZEyx~BQetg zAX!@mC`cXIpCz-C<@%ucc5mnFLF#}ObP!Zyp=0I!K`Ws0B%Ym}AeMSK4cUC8#h|$} z-<`;>0!}0ddgU6Q(fyBJ3a?Ge0RJf|%mzjMb$~_D_0gg} z7{2)GxcvleE;=lp`?6`}y<>!h|96p{7T_d;V%&NrGPUnSW!wIjsZ}+VchEBho-Les zaYfB)U`ksWcPGh)VURsx<#`%vNFJ4(K(KrJn zA+!%^>!lB%G;0NREqQA!d^pA}RJURvf?J@mPJI)FD);~#LQ)g~K&m5fD#B}~>|?Wz z|Ng}LN&Q_K52o8Rqjw@s!Ibzo>A?RTu#3*5BP^e8o?q_$(}q~tB$i`*YS2OSH_$IG zCoYVJ4VLx<9O(9bVl(4VkCl-cuf2(0EU~d8 zz$#c;OqY?kVR%JT6^k9W(DTAwnU>MRQ}k{DM{{uGgK&NVlJ(`AvrT+V!V#5nD?r`v z&ccF#P+$8bWKofa0I*Q+&IqQMNxFbHMw1*VhSVyam5p<5B+azz!(yjh@c0=*oYZ_I zp_DQWfQzq10d)f=mm-pZ-XeDkZt(MIDv~&`U7Obt8&Uz8tA$wRIfN+ne4a_6W(0LdN%X-nZ}PQ-8td_j9|>bJhih++M>0lwHTOuqB;*WNLWhO!Tl|pymb!vG zmNDpYs_&pb!8@*08DBz!aHjUq0f@JMo75KXS3ynXRT+)1I+-bOBlM+DMzFI zW@WKe`9mq0e2UtKoD5{w1mn=~2?A#wxnW607;1ezesT6n3b9%Lrv4<8(Q`UYjt>~$ zGAq&wXg77dDuW?^{ZRN@ppD1{>7d_g-g$u(SQ!pRRT<~FA3=ynuHFwp2Z7zRZ#(%@ z@_bKzqp!8#Q4xcf{6Z6@GdrRk+gJ@WIH=PyaN}+UbxT%$uB4}MDsKiN7k8n1((wi9(#j5 zN5U16co)Bmuu_>7oq;FAc;jjJF|beGG$VQ^wC`lh{)`Pp=@ixi*vXjbc2XBACQSyI zF29iSilYj<-}1A?GS_!Kd+9d@ED9a|a5*cXLv^l1ia#L`)Jk((U%JlrzMI9f9*Bou z9AmMyQy!dehNR&YZ8UGF`|pOpAVLN}!$vzVu;oiJ>Z*x~bRN@CLJ!RQKAxZD)%(4! zirYa>$x8U(7ZRc_HG&Kyobx6>U~do82{EizsW%R72Gxq% z6(9L;{Pd6+kv6VR%tR7|MSMZnIYsI)qo2Z3xbg*Y1`onz;K^qt>`O=ScJB+?ie><)e~MxSe!9%g z#WiT$4?O&-P{yw{o_#}VL4XI5$d4SUf^*tDRGnlYobJNpF4|dryw~$xHAE@q-hG7E zJCu?k(w&SsCt$BnJerCRv+G^oEfL$}`hK(aAzm5b4s{bFe)E1yZdhc8Ni&0m&{0oELf6dK$Z7uFFDF1RJ8|%=xXoAIqV9Gx zCl{01$(c787?nb^2ZXq#66)J88FWBg3^IpXNtfHjsm5vUo-nd{Ls$n z>6I$Eee}w=vaT`@@Y`Ma`s?dGk+HW?Inm7YHA4i>Ft~EJ#7W451X$z*QCClbyKrt8 z>N4~;ifiz_WoD#mhz_HttC~|1#bBp^kZ=G-3{$86C$|ODgWI8X^kKZYWY*?j%2^&Z zGGV~F#?DKB;3%i57Vl#sIs&dQDd8_uMKL0d-Wfgsco4ca(lL^0$ibv*-cCc9K_2N$wlBKk9 z?&^yCE|M+U7=gNaT98-YWHi`cMJSQnQ563IefUMm9e;Cng`I_^Ocy6rRD&~%3=#?nar2y#T@ zo^Hl`6M6wv%KQ`}uN(KZFww}|YCIk$JuiO@ZNcSfS(FAjT+q=b`!axrc|-}95D+k7 z@mT=ZKUjW^aq1chPc+5q-8!xMUUcK=S@Z(mO@6ibAgNU5RLzjH4gwZYhY*OY<9BaY zzlLKOEJbtYGIWXg4`E4C`PPry>CVxi1kzywE4p zImAir7>Eefuconq7%xzVr`y)BoVoeZEokuCLbk4wY0UFrME5L{Rh0`!pONgBu&kp~ zaC>0m`?wGDI}FTVW0*E#);k>Wh;Ev4&9kaWb`i!-IpvxBoK(hpev{{G1k04a6D4=3 zhsn}%?z<>T1e7lrE?;n8kRM+Yk1o-V6W>=e4?|61J>F zQ-utN6Y96_T-4&%OB*%9uQCf?RVWq13%DEqu**g;#htB|<`|Y?6N39MtByZsWf(HK zfThMoIogsqUb=M|Eo|I91ek7{YWf&|Q~}M0M(Y@VABd}fiBgrrbcdtNMYRNG+QRKf zt!vzr2}55Ql$LDVCO+hS*6TPeG!nVZrXZqGSP`iTxK-7{IWvT|V<+N+xN+kAr#SKW zHIm~S3_KDVwGW>^QJx+Q$ffEqxFjk;V}oPqhmT23g*o+b3?(vFnLb{ptrZv4%)652 zcg%dLi|)cPGm|2E(JAbr$f(=YJu8U~_yVh~b8mW=Xa3SNpEy`Iv{RFVSBu~13R@=& zdy@x^r+7!iZ@F=r^zWBa(Z5POXdKorCCl-8qfw&ipMdv({{+1nUdb_x!B)$C8Grr~ zj?#a5HDEqj(kii^=+my3sL2xhlgI<*etSq@FHKYtWzpW!brrv4d%jp15;Hqp*e^a> zLKj6e*1~U4@)U9I(&x>21Eq~TPPTyZty^BD(mxjaW)?KBs^Rwb4q3@%@A4aF;}}b! zo9=h}+daEN$eE_Q2ig%lY>s{PmDHn)Z`&*A&ifyycY`Uk&6%`iUsb#Qo7%SHAw1}{ z=ME`bs@gt7epq#jk!RMpDl09PRGZE0_(MV$X`y3AU2cZx0+K`B=If*Q4^ITb4I~_+ zw5@g7umT;ZfdT)_iG;D(kp3-HF7K|L)s$}2i#?|E44&xoci;1xe~ff-#9_Fr_NIQ0 zfbElangGYw4bv9ql-2dY>x!xSw?^X#(>|l#yZx(x)NmLl5SmMRwv}H}@WlKpqa4Ho zvFISaGWFiSpQi*ej8n*D(f`3G14ytfLT~r}6~IJ^ia8zFE6Xe=GyXjRD!i^mb)o9! zzx-3184#DUVj^Dr<$Xe6LmYV6h(pwx=U;J2Ujryc|3?Q^iS_n40zLjTXggQaES<>{61()@9$^7<2A~; zYP-?rQSs{AtxR{`X6yqz$uol=)IhR`=@RmU*d2Hve;0fOoPHJ%D@=X1fzFfWYxVex z>q~#CKyj~WncsL_3e07O_dUkpGj``dw3*R@1}=}=`p%SQG;zs<3qbBwwi9cdYT`gL zNj+FyX;s=j=wk4kFB!D1y(4hgxm#p-7}fYctdaV`!wb31yWOl?fVg64DQZ=l70{hV zi}bH=fJtViJr^%&Ib7Zr#Cu)0fJJs#PgeZQ$L42T(`S4(1@j=l3?7;sV{3<=4=Vk^ zW(6IuN6alM?1Cw(kuNvkKVH`hKm6__3|EJ$n9prkbM!29fOUK$NcecuT|c;}<^zgl z(;_g|Nnoat?nSB9YisG;qp0+*V8N;(CLM9%@;1X5nnQ5NY0z6|2v$@b+fb;jni^&g z8RTD%-LUKr22QZ9TI64-sHqmT@ZGVmzCMDH1Oh1G(i;Ve*|w-45TT-%kiD)0=pt8q zw#B9M8zv1vssr|!{|t_PYK4_xWbTu_yBq0{f)5-K$p+Rnz!cd4_jm;Glo^On0PV@t z5nQEYasF6wzY<^`h3P&7LPwwC6vNqxL)-}Wi^N|5`Ote}w#4pUCJztBznOhz6Zx}& z4R0h8Oj#A0vyqKr4s8VP9r(uXKa8iwW0R}WY%jcQ2-q6^1{HrzswJeXP;nY?)+b~B z^xMx4CFIxxHpqk(KeovR1Nq|RDt@Averc2{9Z3ykSNu&MdEuxnqt=)y{R|w}ghgK9 z3b>U2Ii{w8rSFo(@kdsL3taVKVkn8?gH?tig>!KK|3?}X<|}J!1t~8?H^M4}@reb? zUGD&LH@obu{sb`vp#P1g-&l$oND zagpS5irifAd0+#E@G;;+2K;1$+e)uqgbNEt62(P@{Rw7vfMo4=0241i%^|Yb2Gjl2 z=^;>yV+uM5c%RaRkzOi|6gnl=#0MI117W$U(BkTvUYzkUDbDz16SzZ>uaDGPL#Wl~ zOwPc3EPeb&LhFhJh|I}or=*fM;1@fu({GzxNl7OfCczU8 zHWIJbBp7@i@HekoaL*7k+#dEIq8Qwk^6G`%GE;yJrVfqkA$X$8(^WA~-arI)W>yccYV4Z%*Pl$du>2qwfwy^on&B z@Gs#w);Opr8qXR9DtLV6Z}{~6q0assbmZ6CFjvz}pPXXn49XiKk4fldczkH?sY|&hr9PZ zn$1~+u1dpU9Uuy-KN(mT>p?6MP7%;nZ&M6$^3=*Fm0y8{JvL`ymhWk#?|L2H88HJ~ z&q<#osy60YTfYMNdWB_T2h{$p6M_^^M2zu&o+hY>%M4J|`6u?7+yNFkv33Mx)2)`X zV^1=F?@@n+8K+p`Zy-6vT&D^|!88KXH%~cN|0LR|f8F!A)5y5M_y}lQ*A@a1;xv=t zBKfM*M-QB){al%MiB1uaq@b{OQ;Sq?$l^*+=d@P1C|=hulJ{-%c>%hqf0-p?1OQg6 z@p=ulp#KDA5hIMJQfIEc|FH-e90E}^9NyZ`t6g?skDl49jjlm@{RdZiPq{PFx`L77 z2IO_h(WF>VN4CN!uHA3Es`@9ZmSdeRyl*B!8lLpGrcQUOaC#&Z4he}y=Af3X`OQ;PSAF-9`7FqpwN0u z42epwbx(mqEPJs{6VT50xY;+fc60CJ$TS)(H;V5r5Js}RJ<&~~o{(|9*8c*mgNxXq zi)Aj7HG;8{l>4d>5x2mcU;dVBAXx~+NzRZ)A4v0eB;sBBYji8k>&01e>9d%6w$mj= zfvxWqZ_=(S{B|_*?WEO`*(SS!J%RTf?O+w$PTAFi7&@`1p^)n|^q02f(xNYR<8A!k z7_Dx@@En||2tNR zT2GyCN3AvQxn=m14`Mx05*Q!gF_-L|W!-Lan|Vu2+2Vln1B&T*mKygc0fjxKW3)I2*ZXHc*n&%P2sWM7(rfOuk4mU2J2^9BO7{mVD#RnAg9Xf+xE_Qau!dE2S7m+#*5BS~ugA*BLw zslDlP!0j7x8n*-zpY-j7%o92r5C4XsQGMSVL%BU1xMa%i+Di-om9=*)o@u({Psl30 zTE<6S(5zJNTi1Eba8`@vY&HM3Tc>Uf;K)3%84z`U%oi5&zC5j5rrozZ4)3 zLbkM<7qBuOQ7_D>@Rvd=C1}Ec?Fp3m6@4XKX&PLK8GthL1YpA3G3kYneQSaKMdgpy zy2b7+^1NL4L5n31CH9fwABz4?)C(Su>y-CLmKria!|=i$%HPO9U99PX+kaw+-Ubvh z?P}Nfp{)>;$a8HVuz>ihE9Dy|p-; z%@!ifaNc9c_}QhX3q3N<(#1*k6~!gPCT>#Gd0fVAN|8eIVy4~NAt#O88L0yi=N$wG zvEm2fa_=7Up3S0FCXp7`+HF{ZxZX6{4G5PgjW{26Dq2Ahn^QFdwrmnC3RWLn9PT`R zgZoS>FWJKXp=V}AOJ)~DH9W?!Jx)gpWRS4|W%35`mOJS-Bru#d&WUHgGvvUtxid1akBLziL|4 z*dx$5S6_vcE(~1W(Yw2l#je{lY$1myyz0;>)jLz4;;SbBIcz|w-3wuhk#TOjdXzi1 z>gc{k)l#09K)X)>b8R{(q7+?DlXFds4l-%+pARMJ{PRs39es^cF#Kad@~6t#sch+i zWU?-eY&H$eiziMez2YB}Tg_NnIX-QII0NzUl_Emj=s|Is+K~g5C-xJZ?xl1hB<2sZ zZ@*!2XSwF;FW1>9g=2y^cKWZ+!=0!o%zX|k;&+P;IZ2M581$0i^a`dpyy%@LM_JWhu`r*9}Lii8!$L>FVohd_sP?8dis zZ1e^4R|xM51<)n2{Djy}ZKH)!Ln%XO3!md(-RPg?hWt5J_pd*d4gPC2N=xX(h3RGfZA-huzF z6yrQO@108v?Q(!u?)<37w$GZQou2K6M%O^ssdMaohBcBbCg8)q*z@5Q?khUM8^pwi zPD6RBIC*T{(pMFTN8_3~x#waqcuB|EQQD<2Tp&M-9DJlmHLHn;nU5Rp(xgnuo{JXb zPtFZc!0;KP@j-EMKd`kNm^Dv@++{Z$@lz)k`f`Uj`m}EU_~{Um3<3_VxN_D|YzrR| z(Hj+1`|w4`&ow?5+GNS`9Tj9iuO>}vtX*0q%@@^s#)fM}sbNmR|MgZ~F;xQ~;C1e; zO=uszGb`K2LZ~8v>w#Y{w3nwJA?2t?^QirHztC@i*p^OtYOx1e;~cHGbDtcpINF}O z+^yKpq5N_g00t~t+rI|;bB=B5*f!kwwbxRypBGYZr3|u~U!o2&lm0s1yqKV*norhj z!eRZX=0)Mpi{{XQXPdis#Kx~*Jz5U;s^EKOL8CD!`D$B5fNPWbz}-r5FFuo-Q`p*7 zkJfs%`597Br&-C_Q!4lVyzOm~xh|6nYFmDhpRF(5`=M*qyg)V1iPeDyRbYIUcf&bx zA6ZcmiO`eRV+E^ss3M^$O&6X{Y0IC6{Rxwv+J74$WY{X-^{}>l^zqa!fKoOEZu&me z2rN?g^lX>RughYG*vKRkZ|B^?46c)W+*XUym|GVdW9-kP+DZDhDsd&i3(?0Kz86gx zxYc}Ma+=wlFcz&+*0L%=uuw;Oq?xxzE3Rth9Tdyd{FIdC%JmfKRiVXIcq5*7#qvY7HbG=mR>B_ zU!zcZrjA@#)hi%+B$JWQ^hk1%g;SZGa{OUR%)-d!abnSnI1dA^TdT)@R5J$8(`w7+ zNca@ucWEv|y0>{8yY3KgGrYF3<)Ssx>U~B7Unu9sUFNYE=q=gq&MP^e}s(1+q{ zJvY1lB0+gpczc|cfA^H@Uwtc{5y)FCcPFM||M{94PQWA8{EDmj$8yIa3~(aJsn<13 z68{?@7V3dNACFCD&*y)|sq0qY#67ERJwd^C5od^AC6SO&*KJ---8Sc(8a zGiM_^s5wv=EHy$ie6AjpQMGt2--P+ywT?W@{db-DH zlgr*74+NOTcM9bE?i#TCi`oE^wbR++18iFB_sG7KxtG4(2Bz=_U0p9=v0RTXZUSK0 z<{rzI-18l;FS=q;J!3`?U{GZE6GKKtB%Y{KE|z}G2w=*5Tk~T>sGl7;y5WWM1q1qK z(m6-Y3c0+4rv7|SsX~VCJ_6g2qEBK*0gFS~Dl(}>i{&LJ3SjtyPe`<>6`(ymJ;DAjkEsgpR zK#b`Ink1q*^hVtqfPo%kiDp9+=#e7LI_EMmWi{C2WMfa@8o*Bq&LWrlmi>xPb>2$O z(n@c#fbYP@C$g`eLP+|ZLufF2f<=Z?*Pog~YfEm6mE!&?zMFKPGYt!c`UeBWtPOt< z$pwNKNnD-n{o3i(mAlj}3By%O-Bn6lwT6^Ff1H=z*!;xq%)Wz(xYW08TRz@>!)}TT z5WUNg7HC#aikIV;=GMgMhs206VV&{&>aIAx&nEZK6D;R2Ii1j)VBF{874;e`D`kgb z%+pYA!a4?un2Ay89~l#eOw*Y1ziOy2f-`$ro9c!)xR{QUSRO1}$? z2RjH6E8H16j0h|AK^a9+Z)dI_v9|O^E+)K#1-D4675v7GeebdE*_jKgv0H zg#Vo$Jct-yG>AF|J|F5(q%a0U)L)BOZjcDxFE@vi8f8m_)C}Y2J6gZtZ0TamZ@y7N zML7tQ?8R8kFo;5ilXwSc5g@55@UnM!o#A@8pM#$_m|eNjVNE;lZAn^jDtwPOdTbFq z-?hKH>oNEP#p?%7O_P8sjb|K`n}c0Uto)w#9ffGQ5Jujxf>SZ2GEWJD6CQT+s7Ic0yJA()J{Mh(xH{M8$ zkB}Tl*L~7Y(4WzBvng&%bgt;i(2B8!K;*x_jc~sEN90fKAO0t5?=WJSNh(fM5G=kh zp+17XKu(5jJhN~L=`adn=y)%)0jnKfHE2P!^|Pm>CmjVv0og8booEZG6B=XLM;iY%DvtlHf)o6T=>3JXm8iKd3sCnM9C~5tBElF$^_S zFqAk<5Z6L^_nDB&hWLzTB9|V3*Zc^j)c5ai_ARDzyS< z?qnWw1$Cua9(GPjsd|B{xI$TgBCEUsZ2_&70(Zs7>}zwse((qIQ_vTLRLpCJ6lGN< zXQe$_EIJOlJ31f5oD$cP{9;Qba}~GzFjaO1?cCJ7X0@A|BN=umb}IJzuhmO-b&p3? z&7tlg57IYYN2^C@xT&~nxZshixLE8u-)PO7%_%KTCwWV`3dN@Er?$*bCb^4l(n}-@ zl$3s8%*xE#Svgq|S;ab+J4fFpMCHgf%KFLHj+1HUGJfF@atXFBI&I8M((zrRZ4+;M ze1P;y^^SXCd}e%>f)MeS^pAuthn{$!i1DRNM}LL5ijN!%r(y8q)g`ZiyG z!)HA-&3N*7+W54>svj9N6OPg5XTqbx!GTYKP6#k#k|vI|mbK!wBn}h~K3CvZs#j{^ z*O-0uOZCMTQ)&pB;mYC9m`L=y3|fhC?%S*~m)#J>FQ`HJdl$FsU>pGS#&bkW|WsnvphFBVhh;`oV&L8=s$hiOYnG z(B)}lXq2Y!FlMH8fHcO zu>acVLH~iy1;Hifk>*Y+h|G|EyA@`*XE|>nZ$N2{ZKZblsyaDJEmtjXzOkYFPWtSM zADc)Z%ZlIsGa_Z$VkiDx-JSM{$58YR){)xbUGI9Y?!h`C7|b7NOC(q9VxmDLOGI#? z(d-kUXb_@58^}i+q>b8||7~anc_c=KL^~&8a$!;==d5R?r#^rWmIEFu&_#Suls?=n zBqicHggzWyf+6fHffLzR_EfA@O#krE5H4i4>RE@w@dC0D%VZ1&Jay|vUYGkij1oFU zwe--$P~*PLke*(#euA*aQFw!2)s$P`FMeHYTS9waJwgpc*7~#cJMAUeNuZ^p4WnjK za8p^!;mg>6p_OwVGCZestBm)GvxuWo0x5nd^puJGY$e-M)K`$PH*%%)CwZEE#$m<- zst`o#$JxVTXxiU6Dd$RzHmsGom{j2VQF4`(Yziqi$1?eM$!{bs6fP?JUHki{RX0kv zv`g`;wo4z4P@N&2#U^QwUk@rGrH7>!^MV#z^Ntn2v^5)@jWX_!*1XP}`tsJ`dG8vpTZ>{^RcWnq3_L|GvKUdIqSh{+2-X;;u67=z`okXmHg2AEEp?XG zI^6o0@5H0IcJ;^eY~@}D;qS@QV(vUwlM#)r#qQem@!hO6PA1OROX01+&GChC7_206t z=sdEi>T$NUR_j&ag=Our(rzE@wlD6Z=W{0br*pN-*ng`fy=CIGV9moz;PGYF-wBBY zDV%7CpVp`BrSa5ddp)hsd5cYiP-In3IExqC1gYZ>*;VueS)kmI+=xV-I9{d(-!b+T zZp)45;Z-aa2uq16{3YSB_91bqG~TD_`4=A{JOAa^KM%`k(+#vXNbR+5^IRyr7k|8# z9)ApNA<+^Y2~6`ldxbrHBV+;@$5w#KOFGaP*aaHoN9=T1vjw5WgaFDup^XDg6i%BU|oY=DR{o+ zy0*+AuFPwQWx>POmfIQga=PKJWU~pfA_&36+5GRn=Hs z%H+!zFk0X@EZDnXb1+EY*E`^Y3w*%9AmW0+V1RE_;3Jw1{`X#R&TNRke}lQc9VnzE zA|(ZUD;e4w8(TY=**GS&`z8Zb&6z8!JF0*A%x!36#i(y&V_?kaYGwOY1dP{}8~ABu z?5I!dYGrBd!0pOM`qvTM!0)%sOr*qr9pY%gN2>lso>;`j-kA6kBQqm2DL*_hF)^>b zkqNh=sQCXD2mZ%LYUb!@%gw~(;^M;S!p3N0Z_32N#l^+M%*w>d$^aa};NWKMsPD>P z?LhWFmHb_gsIh~gy}7NUxs5gPTfO=QHcpOwq@-^R{qy%f=V|O}{%=dx4*%ONpo2_r zdze@lnVJ5n8z{>Awv}7n+|}4pL)6>~xI91`{4A`@ynh}4|Jw6!i~lL9{%=Y4Pn`cP z`JX-iXGs+YV|x)BE1*e7{(tYx{}%r5&i^gQ%k;Z~wQK_5PZN^_`nerNb@1M`c^k#MfhutbL7% z_6FJSSXlAV0xt{f=F{ZPzr1n5gM3N!V6or6ow3eGWP3=mZww22frY!Yqwh7*C9q)T%sfIQ8yP)4vVsyGn|)x7#*8ou>DU*nITIM{NUYe=d>KY zG%T8=HTk?gs~tUeBQSAR5ZHFU!}vA;T+{>cDE!0r`@J;fiC3U$>zAkd)50`sdnAD; z+Y-}mQ6O(@;d#EEA=oLln6I$zLF8~$)Ae#n$={jCazCPR>BCS<@Z>88{oOA?Ct`Vh zr16Gz?_+hNhGlCW4y&b)KOO~MpOh%9d=t*^@AtFll-DM6L<&x4Ju;l;?1((i{kC6i zevvPFoVBfv^PFkLmsmQn|I>;0Z=ERCjwa+=$#7XaT+nI0Tl3JqJ6&a?eSSEuZ1K8X z94PO2d8m*{X56xxf_(Ck+P;)4FfPUT6o_(8x7laA%$4DSrZ6YOk;kVPS1&ej;qkRa-(hGkBq zamW#_x^73-ny{FTQ`yXNwht zJsV+EZ;oFl7J~*t_dl&V&b;3eeYfuO+7^Sl1YH=@%RZ-0M$7ZVO4sWqF^$zYOw(;Q zv4C&g>rT6=oTt^~to14?FN*S?r|bu*ygsP58=g+m_gKR;WA}E&VItxdCI0HTvI8Ej zQ-;-Sl6XE>48i5~`A(*pD$7$*va1V*=y&~!y^PC2POu4YvxQE+-Dx)~bAZsAG^kpKf_y?ynZQ3ySN~t38I50^?G=K-N zZKscqS;L~dPAi`B%SUs^GR-@m)gjf*t#GE(Ic=BK_Y+@J>lIF?B#0QSOz|xDM!ze( zqzhIBArYM(@cbXoLypknNkbO>sijaLhzd;qLX{-uZiJ~LAqzR%`DE#kLF3~M%(9ZJ z(97eQz`(NN1m9hH&$fsL?rfPkoxQR_b^uu>jMIvqKVo`ZpvhP&oPLfhZ43Qg^nEih4xsA6(Ho%bLyda9z{R|4nz>(>t@37)$|7BPzbbVFhIm@7_y4(zG$iL8UeWM#X2<0X z&YoK!7nm*?A$r*E`k*-6!EwWAKFaxf_y!(gX!^49uXXlh;t?U!Mm!q!Llh$nor}|f z^XJ~gy+1cp!bZaADyWww+%v3jG2JRm(Ih3)LpEYb#zT&ATSx0L_g#xZoVw`zk#!Ro zIPq!}QAqfMk*dWF2cnz?2tCSN+ldVV>lrlfW7Gz5E3ZUyKmWa8_$i9&#5L6Wz%{QMD_=hM6Nyx7CLA}%VCV&-YcA; zKE^PW33h6XWBvC+J?SSHgNA8oIxY%tUC)ca1dqE_7oJwmKv=7%$Fq*UG$@v>bR_V5 zFeP4L*UVH78^-xWL5w`B%GS%U-g{sKieJ_<{WF1}qZC)cP@UGn;sqCNl$2K3_vTQ$ zZQAS{q^Y{v664@{h~Udm_m&Hy!V0UB+>*U~{iqY;b+`3 zYM?3f?^RX)8|>_bCXI2T3Qa4bE8T)UDd#7~-Yj{~iVmHpd0ii=71=!TNG-7j%Pr%e zjtftD0azSSa`ZuP!W0!X?Yitf{4HVPE8}EsO1CR<#q{?Z3M7a+Nd^ejGOlC9R8N6> zmWosL#v>yYIDdx_1tEe3sOgH9w#M^Pa8b{Ro5SKummGhuO7>*(n+zo@8gr*StwJ|5 zast1sk;`uWd9c~Z^a#_xor2M={T+5NLxx2Q7PN|~Jbn8b_4jdaVNAXR5PP_n(HH#P zRXx~QzI0m;_9xN z#B$_@GRzUp@J?3D-h)8#{b|AA*SH8g4%rtO?$$>R=OD5yIjThr#d*O^BuUEzKg`-pl zJ5iKWw?I17zOMH@g@xi6UF-Lg)V0db>G%6Bvxz>+X>vckFASe>+K4)I25rVfqNOJG z@aCH?2c0r%Xd%u#cl(x$%(n-$4!7nJo(Y&Fe6UjGc%LIV)~LBpfguyUPCIP#;5EX~ z;3z@pW~6W#Zko$!#1nT5tc$j91N-r2UUxfMb=cfHkmHswi@1GX)vX>VS{1?>FCbIjx0 zL|0*#Ys`Dz2{Gk#xem;Ptd~5u3x*?4uL$AO)N>cxGE8mg1w+nTRCbG24Xc>DUcfqF zDuelpF7uV4VTJVY+2<$I@qA_0PVHe~n#ma1Nna=$mH5rnrI}Cb8PJ^;`x|hV{!Hm= zySbMs_rr@}y2@G7TEZ^3-wV|^C%F|@`x6tXI#-cI>mFx(Qo5g8x2?XW*m(gVw*6N0kmdNWf z^MxZZCyTE-b{)901v@~@yNpHg${cS|(L`xz-TMe)_qT77Q!XC-z z`?KMXHiMbk`Lbwo!DXo6aE(xW<mj(Hl0rDH3{dlBFXEMFA!t%{Cb zkGB|fhW?13Y^|2-om`_^bDeaWcq%tmre&B;syqK;OJ#>tW6+f=igRXraZGJX4PcArkL1R5@f=_KyBeD=PZ zrt2<*S!=ijZq_dtAeishPVgTCtMzu0mc7(`n}-Wcsn6@nc)ZknnFiZ@feh9iuo7^~ z9cX3od#$Xxte*gp`IvMMS5yzMolrmS#0Ute4B(~b@ zj-^l*5Cy2zcs(2zkv%s73&~JjAExGL&ei$!QJuj&$tY5EB-JN-b0E$_v@zzhS#NI> zp~^nyTeQQoX| zj^rD~DN1M+x4GZgFUe}1^LgAR^kLcWGu+Lpbn;1ie`Y-kjN2ca(6msr_GHzz?Sirf z@{P}l9S!70Iee#$+xQ6MZFg&Fe^#=E{B!Fq4AAwj_QnXj;>vYDYUd1`)Gv<%8HUE? zLJEsvz72^XGJXg_HG=9I-Z0iw_(BJNQM$Xc`5-rZXKpZ-j9~i&2wM}O z{s1|V%UbEdLj8d4!R(KCJXWI+P3xvzQ5Rurxr`M$eyM7X0kM{oJyKqehnwhY zdCf3w6c>-{xgJXmWBVTimfyGQK0YK=vp8*%UfbkQUM1YWJcRI}^X?Q|fb>y@%ag!y zQfS=6X63KQBFUsR*MWt+3-U5LN|dRp?kA8Hy)Lrv1$LozLzV zm;Ulz=Up(nl&F|QLrAU3r#NrPs_&5pwk~knf66l z-pz*idX3LO>@6&R3Y@Y17^y5$h9nm)GG;{%$FpR^B+!(AyItU$OWzldX z!L!IRgI6(BJHV)N0uM@nzlmEA<5Eg+TD~@VAIy) zT!73mkr&3fjEkG;kSt$*6%8Y1yO)=f%a?J+s971TleW$84!V(38sz_hZ6JMP}uZuiSHUnLSAO>Lq7u3_qgc83M`xA z2d&HF#&AFeBEzF0`)w3pfS}j~>PY23{FH(c+uv5YE>h>4Lh=Ej3KECpWXF7U&&8%n z)laqDkC~J|2(Fv^O4q9V)1m%GUhyr~Ru?ylxPw-}-_{ffL!0=q6$^9esRC)*Z%z%6 zl2U4sekkKo<`;$foQ*(yAWt7fko4q-YjC#n+b*o6Z^bIG%RrjZx+_7oz%=2frAoI2 zxe82axDo?Vs2$9Zd|LyMe21_EnnHN>kVapCsG#zY&!kB}If6PdaSu1gGT*Tgnyd2C zoL)9}J9)OZ+gZcpGr|>waA8Tq1ttOmbeDrjd>MUqVkLTuhX>0*0-c~b|4(HA9*7%3 zcf);;oOWWI=MIOkD{)9pobV23y&ER~>HLs(qjZnCaUYh{TGJGtPD z_sjkMgQP-~xNR7?;jS_GcYS@M=^DXWNp|ZM&OE^z&tfdSRj2vHM|C!Jz-u7NfM7gg zKjovZK*rB3%YWJ9!%|h-=9GU2WP1~VaY*#kb@FCd^Ef-3!o8%Cy>Mbrh^6NonYb8~ zB>6vpvAGpT@d9I`deBG8fT;{b)H^J^xQ!LSj3#GS{!Is80U9}77>J^sV>yYvKFm8C z!&#e2l3D8pFhHVHgGWUms&E61|F9?N7i!~ML9mN_t$-bmFfB*8N5KnZesPT5;DG!{ z&KdDKBg^vz^9L|e!@4Q78AFd zDRMm=pytrDr_L_ltQh?fbbDF1JkE1Fx)ClY_2wx(VJ-9KbXj9y??B9wtM=$RU;%9sbaAld>KADax4W2;2M+>g#LLehBiy#`z;!}D-` zar4l2V~x6yH}V>|>yC%&j&jSL70HMTIEiy04PPFo`#;7~Aew71- z_+jN9EEe^b34Bos5$AgzCRo451CD+U!zO@D*lY8#TQ+4YnQ1aAfc*GL#vutJyj7-J ziTSYoVlEo9{m*@CQ);~et8w6SAP=;3z=$cKnRxo~nL;Md3dj!xuGyFrnI$erw@?b( z58j8kP(Ku!nMuwt9pLHY3s$n3y6Hd^3cUO=91?CA>UqN4TjAwI@YliMLwF}h%uUTz zFDI-IUX9JB7xECCT`VTXO?rf*_gS4DDS;xMh=hsWopj%g0hfbO8ndsnPJU}Q=nkIG z6vXx&MZ0R@t0Bua_p)T~h`i~_n5+KmbkV8$MGPS&_Dx`?f@wWtl|4!s^cIi2I{kUR zVO({}Lk9s#*KFMRYddUX&rA+cy=#edSSW*&>iep1hdz)<1ei4N@Vplv420}IL?QDJ z7NAA~Pk0TwVHB<_P2%Wf0gK3rK~9&!9Fk^86FE`g{ps#7)7VWQ&b%uHF$Xp)HcdL) zi`!$6o>F27?)-&!n2q^2FV4Llw)HmV=%4%8yqkZfh+Q&~*6~Tf)Z18)y(FQTp!Jl6 zu2C|mF1KI3(aOqnpj+8p?Hy$judzd2!zQ^}xe7?$eZ_QBAbbwl6nlyYi&K8UZIj4w zML?Mr%fvk^O6iMEo#@7ApYU`u!;LCKIDndiTf>LIgGHL(+6m&tphL{qz!m+FhpGr; zZX}bOZAlRy!A|~Lps6ZG*QvF8^ppu?k%>-o_#zTjBIL z)9jC@N@lGSC=j5zBZKy1s{HvMge4-rv^RY34D9f2IVvd?-(l4Bt~9)?;8rX`mF_Oppwnq zGH1zWrgZhFn+?1e;m)lNYdQ+Q=v5^VMK{O;5q42eS>48gyoFeO{dq`K=GI~GN%6uEI*L-!owPD@9F06 zTh#*)H?&Q|wM3{CF=9w#QPRd10b`nGgO?8`$6l0bsA1EaeO9 zT~gr*1^);z@BMK7eXSIJrBu{}{ipe_Z_SGhPyfT3f&?SqP)_6_ZE5j;dLssG`mKk@k()O$@_NE^b`o_ToCa-m6>5uJmR0EGV6mL_tI=u_OpCW+n+9F z8vbmBtGQ`rx$WWH0ZVKXhMHD2zN)~}xxWWcdKD0l&Va;s$o?*#vGr=4$GSHJ#d<%( zMOONVvE#|U%K%^em(=?cVnWO<tItYT=N*VDzo=`S_i#Ge<{55T(G z1SmoC&%pUu7bL3MI~~o^Bop0b0&)|F&x>cn(zjCMl~({H74Tp8pvurIRAgZb@IEeC zR5k%2T1$TJRX=nr1J7zSrHaSKp*R|rO)E{a6&gD>e5bS<)6dDD8QS!U#4?q{C7MCjB zEvIq=M~UXB+6o5%ap+^JgVa^| z!$_{i$aOX?C-vduOiep+wB?_$8vr5i_Zy~GJ6)(YEYQ*Uk=T;F*9|noFi|As0i;vO zfLOu!Wm;Nh8^d)GPy=a+UiX^|PBZg&877zvjF)^|0JI~%R9ew2eR;6j(tr=BP74u_ z58Fwa*0GOod4*2g_FEV>|6>MVO?3Ml>z)n?LEbrM#idDhhXu*vcYOM^%XR|<1$+Rm zN!;(c2c*c5>4c?FDi7o5$GhZa)-#-}jrcFT`gC~wp02ZuU$%!7xU?!dUfkOlBpyJm zT^OEShi^b{n*3(nVkp~*ROssgw&iKc|4#p+P&Ng6yUZz0V(tF8UV_RJ8rnQmG`x#J?Zja z6!H*w?Vt*ot}shKf5f$$mZD0`A?7?=tTn}fw^Bf%;9T<{`*QjDr{(p*)U@ivx6-oj z+wO?m)3OU3r67>|TI5PV zZWrnaP%rJJY-q%Qs81eDGEKPC9P}l6Fm_6QKPQuyZ0Ngpa>!qR&c?9rjipgV`(iEV z0*c(Frrq!_mr-P3F!+*7OB&7# zGWIh)D)19GvTMy=zi_Og^YyqE=x)VXWRgcJoGm7?ro=CjtLYlaXf9V!=z9Qa>|m?H zbJriFeJPbuvEc<^B2aQcOy8IkK}x&^p7)rCq6$HJ{jT=&H+{|vKok{#4mU3M$pwIQ zTu9B08+^+$yX+;Qp$p(g(*@zm1Pufr8EYWXw;p4kcLZc%+qD)lO@fsk2G7oSFoS$m zm#U1(VzxYlE-QA{&o}cfeD2q_K$X9MSTFx<_xPb2+RC!4Qz&{b*@`?^*ukeF_KS0YI2hDX2W3ESgtw zsS{KDlRiQZ9STm-MLB(aq;2i#&l%ZEGeB!z1?VHKCYHVqFlS~i+mA53xdFOIPJwCA z#qt7?kXOAv;R&?En-gn=waRFe{?lst(~|vHo3Lcvw9)~oJ7YMkTe~EiKyh`#qZh)X8?=;YytT83Sck@+^Cw5bfr8_ zWw+RKCCs%O=h`3SXZP>b{K+amgx1~VWm7leu-HC$ycta749II@jvmwbQZ z*CKsxlKEiTcFLisV4+ho$pU#KFO_CDE}&~8M;=JtwJ`p;l$TILAkk=Ze?jnKGvd)Tc2 zVn1E;yWIS^k8*MS)cv^R&?3=ILtmNgwnWEtFk39WP$-J~sH7GQB!Akh^9#cgblE&$ z{0M83{Ed}mz1l4G$~vcxR2#bPo$QG1B>C60xHv51z~IvRHL9qZHilVU5ke1+@#`$) zrC%I{Z**TKT!bR=#z@OtL6bMpSCEW?q<5*vAkm@5?2j zkEbHEgFI=-2u;iiU!wF?u zhM;t1B4Z~I^&+@!g;I$sR7;Y@kf37-g|8Jb%$Hmth=m2bE2po}lO;CMQb#7S%Bew| z*|>olV7_}pZe3vx4{swD%^pG_62UN|NwAal86+_@kQUKp+P*11T92_@!hpXL&?E{l-`XEV>L;`cqBh7Vu|+h7)2C`A z2ps0MoX!s&P0Vkf@w_CsP|dXMhC2mJI|<$s$@Sc)ex{zDM+->hNBPY;y6V}TDBD52 zBHgin-65y0aR0mb3ncS`N@`O;ftD%JDR0jTu!8--4QlUZ`=N;8fYK(zH7W-F%rZ=( zgT3S0siJ=OloVfP=+3}vl4jFa;-fD6)T#%{vms}$Qo-Jke5YIC8BSwM^iF2xH_7xoR{ zXNXJ`nE=QXtulUe0x)ANgJPM|9+qLO1#F7YPdjtN_vAO)iksAi4?%&P?KdgcmMW{J zPuXgdR1YRuaZk1%_Dhk%hXIM4S{!j3u`BPuFx%+ydv*F>w`GkJw)lwjAWYM-Qwgx9|*l#Q6NX-~8Ih=_^l zNt@#-c25hhv(yh3O}@d{c7|zKIQX)wE(l6`zt(qCao=e+Cf_z{YtPAqUaXbI)#{qv z5dG7{xxSDQ@_sxZ5ULu;Er389g08|{k6?cK)e2J%n2^JP?*mx4at0zFwq9nr&D(@s z>i0AWBy`S9Hn$TEF+zZ$FIpKNR8Q#rXLC!n&uQGC3luaD?*}{0k<+pclfg~lMQwIz z+Iq!1QGJlBmr^DD&5a7@h;jQQ02*S#Xd!p*sB)zkx z%7M4R)f&AJRaOM3$DU49I*`U85-440!?_`XH7Hsuu8_hc8MsJNUReBC0#w#fO+Br~ zo(Uxzzl0~*X^~<2N!9aXfuK@pFCq>vQ^AIwn`(?#3OCO+Z(9@7m6EuE@*YL9JE>?& zq>Si=K_nFFy+au=FO^YU1m!Tl5D(BENJ&lr2AlHaT*LPJ`l_2~_#p>H=B^7Hu@F*9 z%3g^-BQK2p=^N}I7A*E0e|7vr(K7dhQ6L>6ghHz=HrGPnB&JXOtx-eJ`*JOgXGK1* z-|a|?fOEmqxHR-ge74#}ktK(%Y-v*^Y1pL@EME-st`?bw7cB;FSkEtLB^#RrwlS6l z3t%QffmRGl;-BGmBE=%IzfshrR&Yo6qTQ=)2kIeXhYbYq=?@aD;rElB2_^19H0cdiB zPR9B$n)SLLb--EKyq{bLgnQTT-W{9I6mq@RN6hA+cT~MB%8KPoQG%AbpeA$|V499( z1=%z4s=5uuJL!h;rrf$cH*C-@?xGg9c0)XLwqN2N7}XYC2L~adiF)J3GNa?lwI);s zoZizMaeteoUVIU0$o2_w5@Js36m#ELa*6>JHRtPLb-2j_SpElXjy;js9fj?YpHu z-aGJ($K|=8KBimacA-mm;z|;CgtAel+5gFS4AAPLKV5Ft=@HGKs4UZ^QzQD|1#}>E zv-iR<57T))&{xX1&(YnvKC8S2!QQnc=Z`XB+B3cMS(Nkhi!(lYxzS88B`C1!;9c8P z4ch2X+$@tQWITb|3dIoIS5*wVon zcr1+9MjP0am#n9tWdMRXFUnYBR^gdvy6;X7(nB8~ncPXkxec26k|O5=6e$GNE2+pw zA{?p+DjO{^l^ap*rTrm!#}gnwO|UDqGLFcFv4`_QdEpAoX9@F$y&>K`A8tBAWT0BZ za&P-Y6kmln4|^gI#Xn91DzDkky%SEJQCMpX9lnDuh~RS6)|wt|(7K7yvJVVP4B@AlcOi3nm=y&3*|~a)|7?_3DJ%VvdaXmA7(Am)wZIrU`>6A zw{ls-y;3>@l{{Ci)i;qb*x)wuWhQGlq(*n>UhSU z14%i};j2FW*SDk04h}W2q3i@z&igNwVupl&gEl!K8t(sYq;>kZ^O2I#lqW6ArZ%&{gmspj_f8p8xuj;bB>v1o!oWK0-W_xR$ngSC3-eLtXMi*q( zHvb2JDIhb0l+dvm{xKN-bu4h=iZ<%n|JIOA4U64H10Rv}AO4l_W^R`L!BNKe?*)=a zf9oKW_(<=+osotE{L=-qc&b?cxj67Z*)Z@tV}pM&U_pT3oWQl38Pv$|@J=kZ(-HcE zS;EVk9;)l}>~yUuSHJX)?hf$uHrj3XH&q)BGy{%@qjIr5*CC4JMnGqq1d!1Wt+Y&^ z7q|Ih1+w{Or(^p&@23liI{*?5)s8T>ST;RMVRb>2$C!Y1s-XD!SbKQfo&-;hAe4M3c`bc^Nsa!tTwp}pF?==K0;%o$N%%9UE2Pax%<$1VVCoWlg~RfMMJ zMc=APnib}ny}9viB39Apa#P{|u}iZXBg}!&rvD zKiRCUlmk~np$4EqvuyyKI|cHtzfxR=uRlXh?Jj$e1Yj}We1pqCqN4S9dy@C&!>;uJ zx|+A{F3isj-I-Q>0YqvofZsv#L3GBBJ|A(@LB%c8+4GIBfiMn%^wFvTKvc~DkCIZE z(JQbrh{%&j{bZHO(*7L_aKAQt0G0{uT9dJSrY#>MtMPQMI1>Qc#l2{md^i56ne6-` zhBTXt*3a>W9x%y(@D@t*y9nCYEavDNw!-M7JKmtwgR7%x5ew3GcdJ30>(v;Cac02X z_vp>^Ho7|2=kNk`Sf8#modV`sXF#{qC;&Vl-9X>Y#3OMj!au8ftE2*?^6vI)W-8WEik1rBs{QdjSuo z^}%G$2a4x|yy$O=Fzg zg37?j%0cWQr5Og?y+~u!NvFn9L5b`a(na+is zNhBi>`mMM7BLh1NGkoGS7#S>~oUqB8ZzIuD)!1HPE)3XqXA%Gx^v zi69J|>OBDe2!fUXSGyQY91J(3BQEp$2hI&Jw;cdN+L1{_CH&+aP|x!@R3n!XDEw~2 zealQgbJW2(lr+5ar=$ud7|szuvf=E>WM?3zTH7k$UZ_1z{qBH2ySdLMF;khg%;*eP zfupeY{;d9yhO4E_>LeU>QdKVs8=mLnIgT#Cdja!#64CE1J0a!3!1dwaqO{((R|ODD zX0%Fd0YEb9G}}y;YZsk_?altL-p{e>7>~zh5&_uDR(nT4`;iW(+ijwY+vdNQ8!GcV z0JVmE9l+*tm>GcSL*i1jx(b{3a*#~ey0N_<*MhU`6mZrz159-l5ucN}M203^P?T7A zrtpiy&GCG)^W^DUqP0}v)XQ2RcV1G~{&3{ZrMQB8)a4H|azj_ys)#vjRlg{nz?LbR zi6h|M`qU3G+u`HW*`k`90uIx^gGZXf2^qLA<$3?FiU$G>lYANx8Iwv zc&d*&FX+SE0rYO@`R5fAeeI`oHUZ|salSjPDa`|%JqULQ8+X)GfYL|z;B=;#yi}V^ zF&(4=FE=nkbOUoJ(>4cr2v@(BFkm$#mh}g$OT1(R0=D*o){`zf(Q1bahS7;m*r0fE z0m`w4l1U$@kvjTaD9TyD-Ao()WmtkX{XGW_Fp#q9+%i|4f8LF!NKzN(<$UvWi?h<# zH%`hfjcg%(`V5I44G)iwfLl5FX5zv8xwakh`1(dFAAe>!C&dX~E}Y7T6E!@^@|^4d z{@7~!U%gwt_`=qgHv(vc$9{I`@NEXO5rZS4 zho?qcQLY&eB4o)=>gUhLK<=R%in_Q=9}MK*t71GC z7Uk|cf)DEod?tlNLYRZ^&R4dro~*(QMj|5YJL|-?(=bfD>f&w!UcT5}wA7_f!S%O) z!+lAJhbhHN-=Frh5(m0x{bUPtkl{kcfftw(CCTN<)h)mB-*kBmZjF#ubc^P+ZFD&k zrc_ganDt0F^xkXadTxDFHe-s1=Ah9;!vD@WTd54mii-+;6j&EVMa8IIg^WV^aX2e9 z42BYYOUh(4WpkYSxa_#WE%pB;hp+NU_zONXWIQ*E+(P?+x}u2h%e+`ptE7Z3GE6}Jcb~Yb zBriX-pv5lWwznQgs9_*ErkqpWE!)lK`g$#>j^AW)P()VoUF9h z{irK?3a=$1aj!x`*yM-XySJCt#T3q1SnfAAyC6DKd^<}DAfy5pHufS19AjWSmOmTx zV|dKP@KXvj2IyP#+6d8ir8XbFe00+@HoOf92x(omD_K*%8K@oY8mHQx9|D>8{%rGYb(HZ5qd`?z?y? z+%yo#OB^2Km5*%|lC1E1Juu>|%96M8fD}`jB2G$uGs8%W)q1m?@468xeI z&%Z$isHlzSkwo=ygAjB7PbcRd%>@6)ahI`~xh{-~ zxvUl=@|21cHn}8I;;9Yv62{pTRbEIM>v|Xp4#UUXBncF2Ua*6VYmd!%_A7zLeOF_Y?Sjj?}mEZ57$| zVC@kv2B%vEf7#C4t73R1(8Z%viRF3P5*XzaGDefxb(`m5uZ z$`kAP;#^PQHWw!ytg(5&5g*~UCj0pTk|eXo;n7(TMANLm1pVx(&rr&piw<^$k^bY9`I?KVLPr(u6C7o_kiv zWqd@*k2_={uB_wj0^|yh!O^`6Jxar2hNT8{lx9?gfnc|;4Wey4!+fUKZLPuN?MCN2 zJ8d;ZhHj}uuIXxAGHz6sgGv>qiEKy7P^0I5@h-1_$EkPG1)o^&sxLXHXZbz}YD?Jh zRhw&>fh(++F61X1}$?Upw!tWKw-UYTkn?KH>7385-3T}c{&TGgGqby;av(p?8| zB4PQLa2OcS%VcED>pu^>S$GrvIMx4;q7}xckio1Ky5L!j{}NcNGG?VWb9m%kj`xua zU`90J-tKhUrewIMU7gjP_MzseW^VWV(gnzr-S;pzkv^u>ay4xP*6zG&(Me*0*S)<% z8j?xf9;i!zFm_tvcI6DELC+Pm7HLRCOi#ucComzl-?901%NW|< z7j&D(tdGwGttNx|Q2|O~PN;Y&tW!cU)WenS#^)E4PZY>f$f(1&R|9P=*A<-O- zE?jnMEa)JY&b;f5@7m$qv~zYZuI^<7q0!E$`$7m{ZXYBxj#_`;-U*G&JXKTaL(6rA zy-?K$nJ(eQR8ley-B%sN^6I+LRA;pAN&!tLhLP`TW*A{t=;Fs(sgkqB7Ypnn)QA7mGp_LCVk4SN1vS zqq#XtI$e{8Xb?qCZMm=0T_=#PN{#&T3pkPOlbb9@(9;db8uh|xn(W2VwXN^PbekubD>#en>LQ$n{Xtm?ETKhA}kJCq4~ zB^}0pa{FA7nx!Mco{<=%b!~b993{48v%oEfy5bQu zX85NbARyK^Syc_Exbr{(IJfKF5`1AFPB%?f$G-r+ZZNy`-|E{VZ`*fiN>= z8)Kt_a_M=)2sqdB_lMPdrqc5s@0O*fW7cKiiZqRN3PW3zR}wq6v`4T;`}iEW7E~`R zd-=v*s|BGnL-d*xyz0w*8kM!L`);r7`)wm;eTPT8li(lx+_M)0dq@sen)Hoox!O?w zu@+2@@a6W9hM!1dkRcYtZ>!sT>WKww;3r1{Q%4DIG@T)Z)^6#`SfvJdz(tQ~-_Sp| zj<7T}2^OSTkK~1K@i|nvgaG<_-u!3a!q!nQVN;eS`s-QAkNh-a3D?tnTfn*gL*>SS zN$?+syHNRQ3llcdshgzlS?PhFlEGv+veFB}-t?H7Ge1Og9Pnmr`HM>SR_w9qO|jqf y%MaxM literal 23883 zcmc$_Wmr{D-!@FMX%M9w1Vy@I(=AA-APt*F8tIOWfTSYbD57+Oq;z+8iF9{A!*$)) zeffXh_uKp7-NylJ)><=b*37K=oxk%uCsb8g4hNGA69EAMM?qftB?1D{DERINK?A>^ z72FCSAYkTNN=d0INJ-JCI@+08TALyu$cM(pqrZMNMf854ej|dY=JQ}OO)A1mNsCPR zqO1p>@Fg*5@uj!B7a@_Tj|f=kgq4n{(Z$z;x)@2QrKdz&R+e= zsrFhLyYK0gXL^k1_%xzlRzFebFboxe3of{9FUGEItqaBBM;a6-j7C7opGvVNpy%O% z(C~5J&CQ?D`-IGuri7pDE!@3%_4}|?5J5Z>!sJjUND4`pZWoS}PeDe=MH|A2`xC&wPH{iC zDH!1Mv~(dEBu3E$KN4)IvrZ2oB{F~CbI2H$7*1*yW^NVzRi^9sA(bMLnAl@3v-j^% zhpEFF8zWG#{|dV5zLH@g`5aals{>TG-M1^{JpwA@n#t71^`WCoP(joLg8! zgFJq1hkJ|g@e%Q=TbE<#u$1X-os(E%S}HlMPI6#fN|}|Esd+*-E+iF(k3#1sVvmNs z7!0L!baNMsNj5b55EqJi{h5Lc`KKk6OtG^hNwKV=(iy?T#KM%|oWyhqa4e>#K;d142;ge}vb0CkMdTOf zUq?Rke~2AOZ1}Jaoe&>oi3U?Th=)e66H@ADO)K;p_d;AkRp24|V@WT0h(gArL67ew z%>Lb($f~SP*g~j!euWYugL)SDWd8Hw>4p!L-p9ghCP~uJM__0a5V__sL(vq}wHDNU zvuf-@I9^NNKI%EzEV|+n@jeM=hWr!t&s5zHWaK#2XurF?=6W8k9mFgdo5ie#^PTTC z`XbI#0+>kW_m54RZwkGkl{3+j!o%L{mKl@k$mWgL)&(mn^7zUyZSkl=In0+O0zg;V$1_YEKz49+8ex510^tsuu zIe|`nNS0HYQ_3O6D0P{)&HTB*W8=rvj|*S=4@+c+J(qg@aLlj!tK;{rAHm<_f3$RK zni87EIp#W+91^Uut*SkG{KzxbA{H8p8OzQ5QvGvo39}-vo^#ly!F;*t}`e zvK43*`V|f*f}3fZ&=a~7*%Q$d6*3+i>Ij|)8!}chJl>(CKS{78Mcy+@{c$frIrqpd zivE-=!H%`rk${Tcisp(qE6UN$QL>Toq08^y)H_}=shbD)Yo!wCQ-1bEFDqixa^qr4;TU(_dE>=;^19qc(TLrsuZ1{I1~~)yHird2 z3GXN`HlHp`)6Q)m#88_5Ip?(LD2&I}+K9rquBp21JTbHeqlaPP>-=y3r){GxJU!*Z z$Caex;kI#Rl@s|h6nw56Dx5_ejh2Wy0@@IkQ|;s4wqED7XDK>@uD?E6cNOg|qUs;zhDWkbZL7k+(0$$ZKDm47NJW-w=FTpkJ= z)Q-6nxD@TMZo8jeZSM)FQU*}+36u+aIXF2!6y_6%6d)CPEVSc5mho&vY9o35O)6)< zet%x7im=$X5ndm)dhAJy@9&kWRd*7nGCF?Hw6u6-wO?{ zKoz=}reV2wb;e^9c(G(Yp*yQ3>~N+7-M-8?dl4oEJmv) z3^sO}sLf|th4B*Su#1*S`(Su>g-R4rJFIOyAp{1mRH zhw|%V>{*uTI4T@+H<0@Z`!clAXY;IP`#pU^T&H}ZGM6rtM)XM|^^{eFv7Qq-{bPc| z^Y08wg!}OGQ%wK`Kel8mc<>^`99Y_spXX;wTu>e{OuDpL*UQdpO1c0 z$H}sbe9TE;>fO>cYp`A^&ADi}Xk?LNUt_T|$gDEj`Op*_9*eFak@rcXqKHpZuh_#a zVj(H1qP4=(%2&s{zR+Z@_Jnh52D2P%{2}(u;+d3prfz1ny9rW$&*QO{GiXU)~xYsM)`EJiaJtHNNRUj;cz}NVgEM zbaGT7toZSr^sW9lbo$dUolS(*j^HEv-1@$Xf{izM26-x5sf7Yx9L;*PmZt*B7W!Ay z$%Wa4?hcXenpLkpon8?ev(Gm?bosI6xVW14#;Zto+t4YccGKnLm^a=kr*NdGYSzAP zXm(?8>NvT@?uA~YULSv|ZKaE_Yt>2bp5V;Zkmpj|S;4a^ttz`?*7dE8sLe+1XbzhC zminV@+snC$GsnFbvj#^9M;Ysx%`L-n+EL8D{8#mAn@>cz*F`2T1eOnOX!{4l#7ghZ zp}PC`9bc1NboD+iRL&Lf?~49>mIgVx(xdz!@_-OV0#ACGl4Lhm*yA}A?+Sbc7G z`o2|>+2t)vHCj<)QFj3DV)Co23!&}@n%@I-gbH$m<>rkBkM4|(JET`Aw}dKZ^RNCuew7>{C{?;P zq#=ybScezSk|9Wb9Qc@qP+1lekx1p1Ymi<}qli$7tHW&^n7#gdUnO~WwK4B1ojY3Q zX6Bk5Z4B)!Z-CMefL9GuO$DzO%4%Dupr~f~v+XUu!yPzEBb}wzFY_ znb;YbvbovV-xooEx(R`=Hl|K68aEqj+jl~4B6R<{LI`}nKg>=?^RG*stVHNEU#QYZ z**Tih@Ud~QanOlk($LUA9Zk%HUP{aUPjT>{2%UwKlf4i-yQ`}!n=3b)oufHBr=Xx9 zI|mm#7Z)qIg7uxdtrN_R)%G3zzbpAqJ<_J{j2$iQoh?@` z_5Y7E|Gnd1B{lzBlAnv`?~;F=`M*o5zcY1|vaX?iV0=^-!T)#jAKVaLO>8lP>_~*?S{CMit4moT`I%dd;j5fO-TIL=KzO$ zh1~4^^QUW~kbkx4e0pd1Ys`#-a>1BG6>Hb9+7GAK%)-bOG8m5!Z?E=()#NBEe&;G< zA=1*I!2R&?ktD?-5JXxI3swjkmD}QeAOFJW^DL`>AN|iI87lM~s;qd<>9VpB#Q#&` zf6h76aM)8?ewMzkv23>c(1Ok5Nera z;CsWnT`{4L$kni(751LgEP*z~{ZFFhr?|V@K`FeT5oO_m)vX_xnE&nSj4HZ8OT3c_ z17AuSXDpUt@#;TGR@ga>x0iW3`vE9N7Y7~G!cL8#ISZT0^*Jey-TVE*d+@Ub-$p4@ zobO6`1(n*Gj$Mz@M|YeJ{u--ecNC%3OtkvdI2|@!*Ewf@`-o(_20@nE?b01_?}Ikt zmd(5(s&C1*HO2eQSjqfTgsr}wwp2?$*V1O7b89K@A`MFE)Aq2~>jn>(F21F_G4QU? zONzWbUkt=+9(!Xz>9XK`#xbJA9hCW=j9H1hj|cW6JFfdf-RyA2{EBhy@j_%=fnWAC>^n01WP8Oexc*M%-ec--aEcmlJ>sZ~YT{p9;R&tN zv~VcmH*S}7P$4Rv&bb=21kWK9A!o9UTDIWuV>6|DIVSZrtdy+HH-y;_D=<|kzRI&P zPxr2zOTKz_2IhaA*etdCZ-SN&JP-cv5mOYQ7vI4gxUgS|;D2vbRL|GT)x!dtb?E5Z zXJ9}c7ThjwA>-7y%#IDGn0H<9j(dN;KpzF=vM%jnl*OdA{Gsc0(1uuA1GJ<^>U3%{pDJ@)OKzCC;{oO95DH9L)`&yvZ4vNwfq~Bel=@l6@~{2OJF1 z)87hvwR;WZxFW}@^wYY?c@pS13*V5aoDchiw(@sEg!s?FgpWWO;o~q>;~J7CG(9VP zJF}MVOI28R*v&DnX{_bhaPcSkRuVS($Ze-$0?GOFU$1)xJN=-gq{v|>{S30h)lQXx zru%B_3`{}kWI*iB`YR#({m39p}wb!8hx-7ZP!#^qS`isH)QqiLV zf1cR=6f}Wuj4`Ye~B+B{I%K)#uKdbWTMr9cX+0gftK>e;)XO$ z7#sBbY$GRqS_N`eKBBx8AO7_MuUTaO@b+jx%)iPVdV9Wt7{zyX7)&E#BNwSGox1p^ zCnIu1MahH{#2CDJ?2~qNePsipwL$Ow zTEL_u9_uY@1-+GLGk{ zQ8!>tS()ano#_3w_d*9r2hw@G z5a=*JCYyXxXl_EMPRV|H$@M#0>QwIh!*KT6{w`fB!wT;}&-X8pZ z3@0-+xTS;=uGNWc%O^k<4jKsa!Y4>+MeEjK%AF2!@bkB#tsm}R4;YwA`~#w`jYE}> z=F;09gdywOK`i6{o_IxcgAehXN!_T@djp+YsR)^rI@`UD`pmTiB)fOPbL|hwQ3v&~ z)#mG|MJqK+?HAsH5J*Js1=jy#xwN%ZSAE5!>!Sgkb7>JOYoq!%TBeV}6n!>-JCAF*_(Tf+DK-}IrCE&E_pqeFhEKjFdSD#RDZ3xk4%QW; zv-+j?&ZsQ%~c2KbP)|1+J?*3e3QNCLL5>;7x zBKLQ{Kk`@h{C=cA^-vy~X`bRBOA|TS#kKeSl;3|b^ZlsDHjhK+4}4ebr>mJV#73@1 zMWHP{wP3}64_enTEVlm(d50?UxbA|U^B45DTzGL0n6zdFKd{c$aLB#zsgRuPDuX>H zgFcBtAt9~_*R95ZsLf(gOEAS|(GZawrFr~iS}uJG{Yx&d&^KrM1j9%B^pfHHci+uA zC|!Grs0?Kx5IZi3-05}^kUhMv21 zKh4pcs^g?r5j`M@L1T9;3z~fAf=bhJjowMOVKMWedsy)AwOJjdCP?*6#y(YbtC8fi zZT?Mi(0{Ove*;`RM@sbafo4Q`EWg+FehXfNiJJCbr6K-QA%4(Nk>ZiprvErRE|H7-!Mj; z`X0qZIh7|p{crCf(hl-MLR4LG3YpdaQAO1K{p5tS`yL|w4LZmG$f`wq6U+RMD&+3( zM^9N`_FtS(kLTZbckAiL`&N7&{I2=k9!P1|Y+5=Fhh+u00?k;sz3;HqP6cD^1;M*_+lKY48G7wRcg z4D#yCR&K8L8iObFJz23iTk&C?w86*BCLLaL*OCQ7pW2VqvlT}?rhJr+_+F;1u?u+! zURZ6^A5Lsu+<%ptQhAriLtMe%_{q|7nns}<1JI+%ti^fHy}n^xx22$+%~eyo>ZSKuNu})RK9`o7 zcJ(JLV$N)zN)KIU9pn`IP25&v-N>Sb{|Al6Vqd!l)aiQ zwyhQ!Vm+RFp!|(mX;S885I>f*Q}coQ-OU*&wUx4!65m$x#bd`bGv|k!C$8xE;`Z@9 zI^~;l8sQjw{VM1lQTFnTD8YgrX%wRdGaY(a&YkOLPne$UJB}DniPT6|)$3oJ?$;7! zDsAbUQd0hEpYuMOGgTlH8d$yoP#qf#nn4_hLGxvM>Att_E`?eKA2<%)O-=&%(`(xn zP%iwtYQYC<9U(!PXGogYPHPqzU&&9qjUV97uxM&A*qG9^Hg%m}9U%Y@M- znk8B0KgQ2|*DsjUQTa9j_8tSo4eMXe?g33O808jdWHc(HOr+BAR~?78N34`F586rb zW&u>ycb$0}6(IsW_*~V^oChyzxa?;92_Wkv5#;9bq^FzwT`y;g)5weXn>3HjA7y(o z2KPzFJ?ser&HuUX=(~2fI+Ek22j)RUeIa>VspuZm@VA=f`86L7`q{QT|q!f z`WNYxX}n&u9JTjDw#I8molYo-=Hu<@z2KG$2Iyd&jE%Y zecH|!E_QqV$KB*?J&0Hz!omoE*a641#TV0^?iJPSiL?U%pAU;-JFZ1)w5zl-2K>yy(?#t=;%0h>c9b1J3d6jn}0%6MC*TVN`RKTIQ)0tPch&!GbW!UR8g- z9A=YjT^gzv_WLWV2Wx(cag@mTaO>cQN&&=jnIe0gSr@ArkFA0kqbGjgnu+0;wsMb^ zZ#CK_u~f~z zqg7K)Q*tyEGV>FQt@@V_wcDb~uIVZL>g<(rO+b-^tMfk1YTcSAD7rkWXR^J&|^8x7trQFeXX;5W@0 z_)LJ|*$`6WyXG7jPB~`P{^SIP{q2)oR1~_?9>_ULzJ^n`^Im|g(MdRdAU;lO%Tq16 zdw8NouFSbMSAPpo{V?=Md0pt#UUkoete%m)e+1_xWwMlqHSPmfaqQYum+3AaW32UH z5@uLqNziqqu*j(}Th`37vi1xT7K`CUR!qiD<&15a73gxkZ#uKVUYtrlSa(^bU5}A} zn3k>d$7i>hr8{;%)tG<&tYFjy_#_fTzn`5me{**a9AiYjT< zPxe{#TwZs|9gQv3V^bg~v~SC}Z9aR4EV)9!%8IGg~aFcJ1WtW|K3dd@*xcOLK^ zpwGL&6U?AWZj;g3@%lo8w#0Q%$jofI*I{USA+$YoYD7qomk7Snf`d9HI7ve= zwU5BnBJ9Uw>%NuHgSkL$%b&7PfrG}|%$suZ{Hht-T8>dwDfLG~nOl+Ei9h93T1fZE zYHIt1wluHkOG;U-+a*||bO`1Nl?5_)R8`I1gs#k~d%S?M1iTq=MiZfH^<%xMCb63+ zmp!S``y=H05H_b?bJKz+W@7yLU~>if>5Zpyvn#Axb{(}6o=Vo7wk+8VZHKQyj|Zh- zfjE3mtZ6fKf45cAK|P!7ilryDkoQ0mJku@CR5k0^gP%%*NWSSX`6Zm|LlBdLuVmqq zK*|E<;GgZ1|7-I)dbB{!j>OKjaF@p|IoTx;Y-1`6w8SpUpq-M$a+9f!j1_ zvky|2IFa;~(=INTA?hG6CxT*T4M;f`QtR3-NR}#XRG^%&2XIJn#9jbmvw%@u$U8tr z`P&GmU@ixK>hq$!YySGmn!MX~@HAtJJHc6eV(+(8f8_#&`|OOFg#_ z5k(_+Xbvxtd3o4lx2ZChSnlQ*sgO?RF3aIbDz@2!P9w}S#}|`fnJi4 zfI0HPqNf`TRK@D#UWqj48=iD+REh0Vw!vD`mckQpx=*(wUW);$#Eic#zn|i4-qS7_ zs~^xy_LHz+EerYMkQwi4YP>qScs-XyO4msk`n&1H1X>ZS&)0#QhFi@6NQmZe8AcC9 zed|}3G_08%_)bhWahYMa0E+%+pa5K|a>q*GxXvuCwi%C96_!|TV(4TFh$oZ8L2QZ% z<`|NsBVeRu8oe#XUOV{nO>zqV!VFVoxb{t`LTDG>fdWGT-r7oo0nyP1bNX1Ge!>BH z=3V!u8TcZn@lX>I-Q(dj^r&~UMknc{4!!_su~ZJQi$D18`lY$vcwI8C;F;0WvgQ?H zBGza;sJ4>PF3#o;0V&DJ&`gm!XGU9CPT$U+r|Zg`R{gie=+`-6K1b~+A(3LUXj54Q zZYo+||8&T|+2VEHR?eSF4LtsNuH#7(?=?sWp*ZamY5m$JAuif3&D`)MIxX+Azd20? zD>D4`A>WhwIG>`@DN$JmG$5iqe5C}Fo~Zw0=}vjGYqvn+&`MAMsYZAgWC|PTq?f-7^g9V)H>4gHC)_L%lS{*(=jpQn*HO@{^gQ ziJI&=W}4FGAfdw_3E?xPTJuKCj@{N;Qjb0o@qL$->?3rn)Lwf$cpI0#CXu}L5FxIg zKDE=JgzR^{J64D5kui@x?;1j@X5DIokHv$lIBafTXw4>-p_wdvM!MxN2?%9n#&!Yp zvm#ZiRPUkwNUH1n6yPY{32?bd1qCN*w)@^0%7uOHHX~jojb1AzWqDnkYS__C+rBrm zMRs)J=!knOy1)@tg0QsY*i8%lsLBr=Yp`0wdS17n$4WP<7g5|zjKul@!LHU%)i@*N z^;h-P`Ybo2efhE=et=Gb&;7#PAt^kPvWW>8C%;zwIo~ajWuhxV?m4Uk<48g4(snGD zdG|HvE>FcNB4>K1j_!$-^fx#lYfSCUBJo z=e8@BKS@(a#S56|AyzIJ6iXelstaJ5-OoR4FmfWo`%cI^ZxD_zjFb32q4*0tvW#$b zW3uUa_7@MF67#Typf+c<^8karLgNPg+4wi1bgk`D-&c^GPG?+aXWDR z9mW~%uh3mMD&iSjv%+KO+UCO!gT!!=iDWN=oMQWIzY$MwjV98c63s%yr1^)`wz*C@ zMX?tOEuek9PkZ0geMD!GADBfhT>m^Xa!d+2JH3)D)ysTwd(h}T%SfEN3^HX4Zxjwu zr6fp*?%2t*;l^)u`^d}9;XwBay~sjjnM5P|#6Bh0sjp`D1nJpyIQku+zqcI( zq%vM+;@zxJwI0{_?kU$LXHUQx#F2(%h?1gBpd;amcmx5(`{V&Q7_bs)p>~WkF{XEs zQgSCr*P=uMhlbjTPE6w$9%}qdy_3%k-28$&?2WJ7sE~wz1-iMt5ows{P?+X}&TIz| zPzB6&inTufj%18EBSUdechsM3#;>C!<|tM-2~W&^X<8Px`Frg;A#QEKN{>F`EGFmZ z%h~|7UQtOG3nrw)*yk4;ckpjw%(R8IXD;{zmvDQ6W8MadZjQFlmUqk$EXqHR;gku5 zasFA&*LI;}W3DeR06}o9iIVUHKdQEeAEjlLGx);lno-&>*eSt!_M9ktEAkyaVy9FA zt9UQn%ukYAZM69v|N-*Ldv98feKElbo_1XVr4g5BXAk zb`H?|th{y`jF4t3YI@2UMOYb)ACtwwZjE<3D&>6We({1#v|{^3Y#$r5t9a@c*1Dc_ zbc)3^@^=1FDpeLV`Ty-=h_k^i&|>~UFOA$kHJRk2zv>04Sv&#Yk$UQj zD*quCI*|dbPFdjBGWqQC{SwhtRL&hQApA|QdHH=2iaw77J z61{wKeYsTx12|>JT8%Y6-6a)pM)w*SCdg79&W}e_kngfeYiBMejiZ7uO05K^VTBfuP^f8Z3BEvGG8u?~SHvHS+W*uw@T&~Xg+4gtR$bT1-Z0|NRzvfQy-v;^7#1FJLOulZX* zR!IREWa_OWW<4fKo4f6@fo8x5SlLLZ77!-H~!N=sn~W7eE6&8%@vuMRa}$l)&Je zG!N!k&{5xr*JWAV!F~JKz{sq}ZG(gd68@-TG71p4KElGW19E?~J$g=ZYlhB%{E~8V z2Pg|;@5yjq)86{zr;XQ#VM~g79b{Jd_jyn(kxDYP_pw?F+t?VM==rbX-6`{QUdsM6 z!Xv^Jgw(zn1l#xZ-heyEFB&wyWUshUubF#Q%!XQt#r}J(&SM+zonYnfNa5Y$hI{-o z9TA^<`#d?&XKYhvm}rW-pEyFXpWgJegi5c|_^|Oo?Yf-~({CW|ScXY|x5)}+EFHMd zQ;8HLnVwtM=R0t9Pgngy!ZKM<0h*$1!FGOG|1{rD`ADUMQ{5XKn~F+{y4q|iko>03 zxt6pMS?Rp%+k5k=koAfY+8 zl?c=~)6)Q|TW>`*hs}9$k(rJRRoh$Wn8vDOUxAeRX9AP#9 zG&y!&UX?L`DMk6yfDtltw}`>17RGNgW?D>73CLQH&t2n()C>+L{d=UAgAaAp%-2I;K^YyL;G^okDCzfH`T2s|Z zJ#3{>C+m^~jFvz9ZmEQ{4Rtw9C=vZ99eY>{xJgV9Ms-@d;$nCi3xF6qJWVD-H#e?? z9Ua5W7?P9jGrd2c@>Nfbmhs`PFHmw#+vM=R!tg~Tr+!9y{fK}4&Gd_%=T5~Nmrc-e zE*3!TG6}EA-T@7vN=W15q8If@(a@HaJD%Y0=xxyxA6RM4^Gn47F z3#S6<@AsSvcY6cmCs4O>q^wat?NxzpnP6N1uPyUJKH?>Lz(DJR`VoyNFhX-a6Len< zybgjYOk-Z~aUSZ9BTOqz4e6vmc3<>I-=;XvhLZ`m&?PB+gt8IqyJe=H1wmq{gdVsK zvtaOjIF8qK3^gw~2(YnLZ>m%?ckOJ|qqL%raniRy|B9XeO7aCMi$~8_7K7M%jWL`D z3Ufn*C@&#P*1YwyQNJQCkdtv05D6P{9=N99c>Zy+P9%H_!M+U9b8aeu?+%%vm}8wC zaA{3`46f@0#I;%F^qMlYm1t$IQ*Dl+O~g*S*h#;;`_(%TsFRh&^FLPS8P;GE$mCx3 zJh-_+U+XIEVVj9#$Le_Y(rlo+eYLTzGIpwiS?@bBVWSFnALqoRiyp{_ zw#hVQim-BCOq$ICIieYGuf%aWwmvVg zmDh&CYe6M4RZVE*ntI<~QSi7t^+3<3rpLerkD$pX^kMpsI=h$H86f1gh&dZrk9#_wq`GDs3YPCGq&T zd$ChxYPNkAzbhg&3@fF0&Sl1w7TFL9_oM2JbMZ0-9}z9jJ%`^_oIAXe7Er-kNPage zytcYC@aT;$LDm;~Y?0$ZTDJ0DWofynK8)9mxsAh&{lJ5<1k%2eQh`0%w+-2TO5i#~4Z` zRc!`=Xv*i+L?EtE=jU}W`8XXZ-<-;dHldjY?SG_3A{SAmwUt6!{+t#3IGk_VLiUyh zHe_sA;<~6Al~Ek6Ycy6(+>L!8xCAF;5jLT033Xt&C4Ugn6gYcx)_8~G5Y}O|6cT0Z z-~?PQ?}15adU+XFc$F*o&wAcC&l4-RX7&O4=skff`qQlm5@!5+ z{uEz?m$tBumG!`&^^r?OhQU2!wSjnE(mCh8mMV2zj7sz(+A0mv&rPCkCC(~@N+isR z5jR=Xh*rXPcw!x0$X~gQb!U+?#&pM!PNs1K&*=jywWeFI;N%>DaI_wr)<{9WUopCC zD`>Jv$c|L^Kg~(ul2%cG@rJ&Nq*a?DVOM&n5+Jq_(&LgD6(V7pNnS7GGab1YdY;wk z(<(NJy)DB$Nkv@T5jS`i?#fucE`rIa)ErYj`%#eo@!Qt<(&$X~!%rPuMICwpzk*R| z5@6}G^Pj%`dm)Tyd%{OR-YLoCk;|%_xW=e0qggI#Jt7mrj*u|YPVufcKljxOUUlGv zNP8ynYjUS-hW8Ji$wwI?PMsDbk=>NZ07{*Wfnr0u;%CsBHo7Qtgaj<_{yZ@G{3;NK z`g}JNj}A}KnMi~r#QdMhUBbT^W3D4Hd}&=fW9m&M14Z~wI@ zqQ{iq_MOkeBzMq6yw^Lazb(P?<{6`L7}b$x)u7l2t%2yd`$lxpL>z`~5o$7t@&n9R zT>fXD@B$o<7Ku;-F&=$s=4F)SXNxOd#?u#t{)WuUK_g!&aR%MIM_Cg5vKx)45u#KK zffF?&6GY7TxkPq#?BQgZ8D$la=rEc<3!*t8!ZuEJC^7m4m5Uc9h+KEqM>F0oQ>Mh8 z5Q0y8NKGqL_`=h0as0WNcDrx^0a~{R4v2%*5VRnr1%LDbBNElNQucU@mz$ae)4U_Z znc4K_IT`dd)B@z)JO)#)(;LEQX=pFwMX`Uc^+b~J5WQJ{{QeUvqBnI_X(`Jw=Sg54 z$I`v?N7n5C*QieP;y2+DjR;t|;_&9$ww|X4Ql^cSdK?d{7J2Ko~%Pg_2!81uiV7Wwbhs-%Z>9>e`3Oe<1qyp{Z$GSaOmwhCzr2!EkC8K`>U3WR zrvsQzwgCyZ07fyb`i=UiQi@Tp)IROD(PqMv=LRl?-J+FQY)%QPwZiE(e8h~3Ut@8q zCMt&LXMWX*oI!)DY_v}z`Ia_UsoE`T-=umCH7Wz^Vs>k5 zEjO}G!mp;f2`;XN4nM?a^sfB9P7Qs^rmG_L%q=h|EJnLi=)}zMAP!4(7Hfca-WrXT z0rRO9*35Va^$xIixE&-b;2$uZUPN}JcJ&4olkLlXHwfP=vkxbEX4-1fPHKMqHAqfg zER*C%bd}OZ6(L*Kb2BtALd$e>qddzzD<=0=!EGGqfK*bW!3qxG4@tVE_|bKAuOHaA=2FxpEJ zWaVmQ`me*7>sew-4AD2QX>!(e^=Vr1#UvlnlzGt<8?H0>L&IM|M|Ks=Xi1D;*rrBt0{Reb zW(P|Qg11up@ghlvA08^hT?sGgpG6mR$@Fq*8?>CamtIfYvmIhu7Z9F;lP(s+h_O;p z%9!V5qjV^(Ix=G;-LItPygCqX{i=wYB4_Pea4O|<768)XTD@JQkb+P)k+w+o?x?a^ zZVY&A8$XMz#U*@OSJ4@WCu%Xz#;AjhxlHx)Y6$DjFy*57QRiCg9W%EOP8LdnWvs;WpTT&MP7hx6FyQETQkf^}8^4s;FBEv{Ocz!X zbS{au|5oY&4L-aAAM;o9uTUo~9U?s^wWHk?oIWfD;!Lk@6l!cfIQDEe3Q~uk90yUW zaM_$@*k^}SmHOp_-NG%)hEkssj*p*Pzg=`UaAPbfY=s+kuH&Uo3bykz@#r!bli88i zAZ0R{_?^<3o-m+Wr+iS&G+cT8>cf}R019dl~@Qu1pO$9{-6hF>Iyj950# z=b`?yz=N`-$FKv!`LSeK)?N322AzarIiLv5NgS;X@;-PtF-Aa84i1YMU{AU$#lMqUyL}*e;zsfv&)Eqv>YV3iY}nZs z0{`!2#Tm2x5O3@TL&>XywB=BjqQZE+JOc%(#zR$u8qXDXVUOd693L;QrkJ?kTiSKv zF#0+LN&+Stmb7*DaEDlNlSxuU_RZ&_P7z&1H_|l3ERsXc#b*9pk|AwR6DmGr6hVYh z@P&^*P%O43_@eBii5-Ok+_v>>Vm!z+%Vl9Cs7afU`wlQyd3iG7Mc)Fdh0 zYBeHBZdo4adg2l~_oU9fbAtHU%|mjH_RIpw9wqq&tQP?Dc9fqfx520&JY7_0w z%l(Tmap~1w(lqh?PFzE3U}b)Ua>P^#$LDmxnKjlMGx)*U3hOwRx_cUi%7z>0Rc#`Y zFltP#^O87=si5TnMr=Lf;|J`XW0*zuVKJt`Jyvbu4$D=Q?&N}p&%cuRrl|8j*k-}m zd$}r-n!k;WK3tB z(I^tvGfd^BR8vLxoZhN+hqRCB+ewT1bSUuWJu4~=MTT25)@3m=+iUP`tQdCdN4*}4 zlUFV+wNy+ID3}#c&`yk0I6j}+I~M7`)1Vb@4c48m=ZnJnxUF+BG-$p-M_O8%CDGOY z$h+%|^JFFVDk-x}l#d{E6VQ}7;y+MIe6mDg{$Drz1wEu4)aQ%bpIKiYdI zLl@dVk2O$le4R8V$aXnRM_axZ$8j4PlY8S*w@laLGZHmKQaSR4WtMB#E-p zKZtcH>N~P>SZm%tTv(@bVQAxg@V3#qgXDC?w$7-mc%p!1lDjIlZXP&T3ThlA3~l-v zKX8e%8x{2*@n!b>xcS0T#hv!vSQMn{pK|hTb-#$cypnB{KAa{Z=GC<&%}jIP=N%#6 zMq8n3nt!=hJu^|*(2HC?7qo@d>S|rP+h)!%T@fRQOtm?6x5PeCTxtK4e3G}>O~}oB{=t-^&u4Us+WITh6B}`#EGR=|bd6)vi8VG;I>P3# zn_h(mx33r|1Z@O3L{ozU&m9`g6`NXWd$U#a&1IbT;@VRzkGy!$oj!?qS8UGxWwf;+tmav!GjC z3raL|Yj(~}Q)daWlD>wv`41#H&drg_v`5hbjpOp`(kbFLt|9p0T)K6$$C}wyrp`xP zBzu!`$3EiWo<(wF1OuFSU1iA8&+AD>lwFV$=%M-|tigMRT_W9u@7P_`LW)>7zcA13 zKGj@vupC?1*5y0>DJO z+?)S(&HkIHe}RHl;2cYAruYx8A1-buld(>X%{x7w1W)okrv)}TP|3fF+!OuvkNnN9 z@o_96@{)^b%6ul**ht-%=w zcu#4q>rcjEpA;A|03}dkUa)oV=zI^{dyFi_zmRsOuYetXY|Q1-r??3Xs|A}|$@i>w zEzPs}UTrV~Pjg#{6G8}_dd_aZl+Xg?jfZHYCipEwTL*_d?7`%Ab)10RWPjgszpcYV zsYW9PcpaI6vxs#mjJ*K@1amVIr@`er{}YgpnR~48S%J+`c!0J0%DVxGTMYKPeMy=* z&_Cp|l+LfWddll%Jhau>F7Ca<_A@qBsJs5ke>&(+motN@N-uGM!!_tKX53iZyNK9o z^9_6q7-4MrLQC$qsKLQ=H_lDIm}vuYd$JvwzGb@6K(Q4dRb3aUlOli%acBQG1whoD zH@|8-^@oU~pojA3KxsjKr}0c7w!uxWBshEt@DBVOvX;SBk7gjsgfRd8u>+f}JuoSrq3;84gk>nWS<#)!>~A}g zElSr}P6{9w$`U2(dYL+qM$&o2DqRWeTZvi>Yaec24!UOvi6HnE@TMJ)G?fqfxOfU|)~;~ItCMgn@xWsgpzfH^)K>`PM?*sWgK7M;wyCqLV>_#sO7Q=>LA zO~SArdeqASK)J`KIDzpn-dNUV%){V*PR;?a-ROQ}TbYkA@L*qUu@fx=gZTT3E3j#5 zz$voJ7dV+A_#ePdy$*Y&gI52NgLd+n01MzN#{%tjyQi#MfaN=y;05{-@aWvMy;G<*wd+Wr`@KS-1U*n(leAIhuOg4+PRw=N%hJkYhB04Td59vwsR91b>Mrc5cYuwt-UAI{AyeEXHcXqQ?; z0NCRLKYjzGdu8U~=8IlwSV10&PGCGrh?P$KK#EiH-4CC>k8<~-k;TB0=$QWA;; zh*arF5s)q&flyVdN|P2sS4!xSa%iFVDu^N?f&wZ51f(Vi0g);wp$MS|j1ama>fM|V z_c`9b;O_4;d!BjLp0)P8?{DdYMo_UafRc-)ossJ4NoI(>;>HfA##UrDuhMOwjTfjA z98m5dz%wZ7fSLMbCp-`kYo-qrPvgb#Tt1&55>JhjrJMJrjifqc)qJI76@QB2`A#{k zT8P@+cQq8$s}gSTdK6l;A>f|F|Kyh4q$2(JXa`&sd5bt!&P=94!H8`8`CkENZ1REV zRNxt+wi(^)^I;7n0iU+5-_FnEe4@wPDDSFZuf_x)+5_@Mk(e8Z(b&9ZJL-tegTux; z6sMFqF97g-)JsNxMzoLSO14xglziGFeje2XT4tN+#w?>$i^&0r8n$ zF5L>9C{9}(#BH2&H|)t5o<5WVa43*@kLm8=RrJtoWYHgEdaX&LP!i3JPc|LoAq9dV zS@xLe$yV0AstGjjF1E;$jq>?e{I}~*Y-h+Emq>Pt7eE0vZgXpJX(Vf~3G{$o=Y|(o zU(NPSY&}8%si%>F0Wxz6Z}xtgfU!~-Md!{Nx}Nd;z#qVYo}*h%NKU(J>MaSATB+7{ z1T5*x6W1?yyLA*d{n>l`hj3?f+y0ZMICxVH&@c$s*qPbr&wn#mv~dhHtQ|Gi{C18T z4t8d$M?$FAC22;t0gW3wYWc~g;5@Upw1BPLtfEAWY&%);da_r&bBT9Fs_&R$4yy?h zO|CW47;Sw3cmRByQO7B}Sk4SekT8jO;ZxqX5J zOi-@w;|9&g`smF0ol*su;M@9HgBUB8WoAVyF`_cSF?cv@c8Q{>(rLfon2aP1Y6pO1 z;`OTqIB0Jd+6_E4_SIfXRKVFNFJlIpiG6Mjry915bVT2wL^`B;AaP_esUfuvpFg)0Qee0xx?GfhMVXlt#a=AchEOa1zB<;AFc@P-%CCqnj zULPIU<;ulCgDYwvc}Q^E$NW0*vwg2WS=_zI-IdN1Cw_MQg=$!SFqv1x+V7bR`UI@< z(9%pNK%$*wQO$4}^P1Hj)W~ztE82U3ENDxwHNb61%RZnYPMiEqyO|6~%-FgKJ2A|) zI0D7BZ`q*rc^0?(kr6^mb;eut{FpDz4M8sEw8$L743^shKwXVq?O?4yw2AK0dFJRj zKfubsc_r41MNjWJ7cJLEd{HXY@w#QdAhV{x&F00N+kT#_*$DkK;X^)Q#yPGhkLvhJ zkV2c1e*n1zK1E=|e(hO`#p!ErFl`Of+5q3dbr}sTl?+L9DR(bu;FMtv+wiTx7+aBRlml4#$@6CBzw*h$d zPIUH2b&m-9sMxn8RJ{qc0~cN*og>@ff-(*BBV^%*kM-Mv`?8L$f){$VtH@RGNPYKQ&cC!^Nw`|roz-1vpd|mmH-P7#?Tm`!^9n)2O zN55TgmLU@h!#-E%DGD~L&Q0JKl6lq6IpTA^rptEY`h&+gLgBXq%kD7-dNJB5s+PYa zX_?c6lX7021|O2Y4&z#&Gvydu(lYb7%$|cb}@`ck8iKD893kb<+vLny>UWAdA(;Y&Xq7jY_%PWBdYd^6enob;j z&nV%g+Y`Lw|Tf4`7%HGzXUfi-|tnz(x3WnMf?Twa`) zc4<(G%PqwU1z=|l6?PZ(uyY@9xrMj2FQll+X~lt|oDMv`clA2#3Kys236|ne+1YN? z%T42Yx?)V2C2&kA?Y@(Ite z8c>o^<8nDFCLje1$#0S4wYIe{SDwgZFU$c|@Gz9+eK@X>(Y}A;D*(GzK>6Ad|5pm`?UyMeC6r#jBIIxb(q_M!z69464D zdS~2sc%>bYsQr#rFE6?_Gl=B22}9+@%qqBAl7CvmI zhBFNbO-6xKkED}i3D1^PQWCY&S$vtpSbl;PEc)dSQ$crmYRRU@jT{8(6y^2$ z!k6L)r0Z6X>~-b4<1Za2!hH2I>uTnDwTExaYV2T7dU zJEhFX_lL|CKO@bTLR@i4(TOI(uWOnWWlJ;{^@sEbP9|ZZ!9F%6^xYv3ELBn6ZFx#@ zP}}9D+M9nK36x;jblrPmsZiC5Z^v`N`RegI2`7sflRn#}jkgh9ZnqqT#KED+ywX=` z&lCEI4UEAe<^nTdd!Jj9pc~Gg)=ph1;xQ7VbIHe*PdGM~2CiPB&$@bs9Zy_(*J<}- zkKW#+2RSEEfdcXp{`^iKV@(aU}(URX* zU-L>V^RWGO^R(?q2nJHN-bOt&c~M>`C9?^*EPGr{0V&q{Rq**0?VIL z3!MR1uFPrbUeWpuw=jWdEahLE8r*yzU&NmiqM1#{*2=mY+fRi^JIe0k?LnmcEWas( zc3AWi%<$T0ABtk%NyWD4zuet{lyUaRIC}9g$*8E^XTPIGh!TR5z_eTlCvodlMTH2- z?E5&xc)yH+HvpkDJkmrO7dy793Oo;K*gK(FB7i@89JuM>S;Sne)u7*p=i^TehM6oPT_eU1v1~N(BR$V=g%+6wS8bLb>Ynpe-yGTe_rm*$!SV_g< z;ePPHc@fJEtBwZHH&9pXyP?jZi{>30G+uhVgPWR>1XCBs<5ymTg{^o*#?FAhxMPYs zRVShg_k!q_Er0DH$I#uO7Hhx0&x^f#h5}4*fg3B!>4{DJ!zc53=m#vJ`ct|12xsP( z7Lq>drgl+-y$mT9l)5T;TJXY`;)2!nj_PExx{v9E{>K}N=?HZOi-}Li=EXG5S4<+T zONBji+&bj$=`FIQp!07?G|1ebv??JfGxB=!_u#$?mA>OYI9=@>b_yD9S{jH5y#>=o z10SjqL~tHCz`Q8z(NmNMQKMIZTt*6RH&#Sh|0)|@4jk_~Hh-O3T<$8 z!ui>lnd!@wU?f)j=vju@G5+bbWHh{BjANfZKvg=bvaZY&n-dU%OO;H5gvTsXjglmdoODED`(sF&OoYGc)OMjkd5(g`Ze@Brg8Ce9GP@- zPiq{-Rq;&FL^(8yD$npNtd%DJMwVB5ZyvEYE-EET}W__YxG=Y!OzCRIM&=X6+!{*0G!P@;v>yeCBl$7uO)w`N6*Bi4@jmY>$Z3Mb*58iI2uaCh;c0?Tf~$ z@?|U{2~+ekJawiI!TIt%v=-eBgZ@RRg_}B8x`$Epbbbp{?*fUR@JpuDWJUG$DNkfgW~dD z*vJ!g2$`M2BRH!jOaFr^T=jtp#Lc) zqr7s9T6d8WQMGE{vjSlp0l9wkyqi~8w$NfzDqLq;BdO{KONm)4G~hNu7m6IbE4Sjx z;Pp@f7snDJ61dyzmqf|&%5P?_y=9i^2r3xHB9O4jn{wLDBJ?5Q%nC!-d*JQA; z8|#FN>6tE3V(WK|Ot%Zr-Of_S1hO*rQ+Ajs6`wDXYKW2u z(d6MnIkn;fVUY})?yTx`<(DEvJ#yJsm4hXFkVGx3YBO~Dm|mi}cOp?^L6{B<@ofa| zS>=j0pkas`?~nAX-^!fu^k)d;`PH5#Z#`K{=`VUe{*BK0!u(Rc3t!a%X=^(Z{_P3$ zULBS@pr-pBrwmDFN?pt8Mazdh=xt)aH%fA2=YnJc!7`>4)P_B5(b?`HAfnp+B-6b5 zf42;ZV6M3!5k!DkB?b(}%m0{5btX1p*Dy*9K8JVVam#Th|Av(C3f1mA{qZiadpLD_ z{H=`)vCV{-8oZviO5Bfp^%<@cwMmM&Qa6z*Zc?#dK0wK0`5@7DC9zNDLeV|JIAYZ= z&}(9NcT9M2Np~Qt;udS`*+rh;<)Z6jItvVScce9^)EdWxTiRt-P7>J)rmPh$=Zt)$ zrM3RtE^i2V86_achGx5A_qsKC#8wHFL7{yMYM&3%;ummY)*E+U!-V+^;miM%>n$*dh6YSJY+i$i}Us+y;?0vd6J2*ulSJeK%TwSOlD9%&bloY z#|FMLC|J8baW49`)P1eF`-I+xpmB?kF|HEBBcH_J1B8=}?f%(qsSB-7n-d>t;#n{h zLyxg-6iFyr+d6GO@kiz%=bN{6kA{-raU)!-wV>LHw(q|NvKtfo7=x{I3pPxxX1ao2 zl?7UQK(QP8tD{rZ zJSq)S&`mAx<1adAB&(pC6{yAE!K94;C^``!fS2feDhNVJQ!FX5qOJnNmNnG#Le~Ek zQbI+>n72eBp^xCwCI1+gB>(^b diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/scripts/requirements.txt b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/scripts/requirements.txt new file mode 100644 index 0000000..e219c91 --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_03_foundation_model_evaluation/scripts/requirements.txt @@ -0,0 +1,8 @@ +sagemaker==2.235.2 +datasets==4.1.1 +pandas==2.2.3 +matplotlib==3.10.6 +numpy==1.26.4 +boto3==1.40.47 +tqdm==4.67.1 +lighteval==0.10.0 \ No newline at end of file diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_04_responsible_ai/04.01_bedrock_guardrails_apply_guardrail_api.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_04_responsible_ai/04.01_bedrock_guardrails_apply_guardrail_api.ipynb index ca65261..4412b40 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_04_responsible_ai/04.01_bedrock_guardrails_apply_guardrail_api.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_04_responsible_ai/04.01_bedrock_guardrails_apply_guardrail_api.ipynb @@ -5,7 +5,7 @@ "id": "f6bfca55-c771-4e26-a13b-3451f6bef06a", "metadata": {}, "source": [ - "# Applying Bedrock Guardrails to the DeepSeek R1 Distill Llama 8B Model" + "# Applying Bedrock Guardrails to the Qwen3-4B-Instruct Model" ] }, { @@ -41,6 +41,14 @@ "You can use the assessment results to design the experience on your generative AI application. Let's now walk through a code-sample" ] }, + { + "cell_type": "markdown", + "id": "bad4bb6c-9a91-4d44-9457-551293c20dd1", + "metadata": {}, + "source": [ + "## Prerequisites" + ] + }, { "cell_type": "code", "execution_count": null, @@ -50,8 +58,7 @@ }, "outputs": [], "source": [ - "#Start by installing the dependencies to ensure we have a recent version\n", - "%pip install -Uq boto3" + "%pip install -r ./scripts/requirements.txt" ] }, { @@ -73,6 +80,14 @@ "get_ipython().kernel.do_shutdown(True)" ] }, + { + "cell_type": "markdown", + "id": "e0db97c3-07e5-4219-9b78-6a60728437f2", + "metadata": {}, + "source": [ + "***" + ] + }, { "cell_type": "code", "execution_count": null, @@ -94,12 +109,16 @@ { "cell_type": "code", "execution_count": null, - "id": "ba78c7e2-d1dc-401e-92bc-7ab6c2f58ac1", + "id": "f2ced34f-91e4-4093-8b3e-1ce7a6600188", "metadata": {}, "outputs": [], "source": [ - "# Configure the SageMaker endpoint names\n", - "FINETUNED_MODEL_ENDPOINT = \"DeepSeek-R1-Distill-Llama-8B-sft-djl\" # Update with Fine-tuned model endpoint name" + "%store -r TUNED_ENDPOINT_NAME\n", + "\n", + "# set the endpoint name manually by uncommenting below\n", + "#TUNED_ENDPOINT_NAME = \"\" # Update with Fine-tuned model endpoint name\n", + "\n", + "print(f\"Tuned Endpoint: {TUNED_ENDPOINT_NAME}\")" ] }, { @@ -107,9 +126,9 @@ "id": "a3480e72-04ac-4caf-86bd-0fae2cb8fcc1", "metadata": {}, "source": [ - "### Important: Create a Guardrail First\n", + "## Create a guardrail\n", "\n", - "Before running the code to apply a guardrail, you need to create a guardrail in Amazon Bedrock. We will create a guardrail that blocks input prompts and output responses from the model providing fiduciary advice." + "Before running the code to apply a guardrail, you need to create a guardrail in Amazon Bedrock. We will create a guardrail that blocks input prompts and output responses from the model providing medical advice and obfuscates PII data, in addition to blocking generally harmful content." ] }, { @@ -206,6 +225,14 @@ "print(f\"Version: {response['version']}\")" ] }, + { + "cell_type": "markdown", + "id": "6d0ac0bd-fe95-4d52-9261-70ca1bf13cdf", + "metadata": {}, + "source": [ + "Next, publish the draft of the guardrail so it can be used." + ] + }, { "cell_type": "code", "execution_count": null, @@ -224,8 +251,20 @@ "\n", "time.sleep(10)\n", "\n", - "guardrail_version=version_response['version']\n", - "guardrail_version" + "guardrail_version = version_response['version']\n", + "\n", + "print(f\"Guardrail published with version: {guardrail_version}\")" + ] + }, + { + "cell_type": "markdown", + "id": "3fa8e9d9-8036-4683-afcc-1d27ec240d57", + "metadata": {}, + "source": [ + "With a guardrail in place, you can now test its effectiveness. When processing input queries, no model is required and the `apply_guardrail` API can be invoked on the incoming request alone. The API will reply with an `action` field, showing whether the guardrail interviened which can be used to determine whether to send requests to the downstream LLM or to take some other action.\n", + "\n", + "### Example: Blocking Medical Advice\n", + "In this example, the input prompt requests guidance on cures for COVID-19. Since the guardrail being invoked is configured to flag anything asking for medical advice or non-verified medical content, this request is blocked." ] }, { @@ -246,16 +285,6 @@ " }\n", "]\n", "\n", - "# Here's an example of something that should pass\n", - "\n", - "#content = [\n", - " #{\n", - " # \"text\": {\n", - " # \"text\": \"What is the rate you offer for the AB503 Product?\"\n", - " # }\n", - " # }\n", - "#]\n", - "\n", "# Call the ApplyGuardrail API\n", "try:\n", " response = bedrock_runtime.apply_guardrail(\n", @@ -266,12 +295,12 @@ " )\n", " \n", " # Process the response\n", - " print(\"API Response:\")\n", - " print(json.dumps(response, indent=2))\n", + " #print(\"API Response:\")\n", + " #print(json.dumps(response, indent=2))\n", " \n", " # Check the action taken by the guardrail\n", " if response['action'] == 'GUARDRAIL_INTERVENED':\n", - " print(\"\\nGuardrail intervened. \\nOutput:\")\n", + " print(\"\\nGuardrail intervened. \\n\\nOutput:\")\n", " for output in response['outputs']:\n", " print(output['text'])\n", " else:\n", @@ -286,6 +315,37 @@ " print(\"No response available due to early exception.\")\n" ] }, + { + "cell_type": "markdown", + "id": "b9b944fb-0402-469d-b0c8-67afda4de22e", + "metadata": {}, + "source": [ + "### Example: Anonymization of PII in responses.\n", + "\n", + "The guardrail in this example is configured to look at a variety of PII related field and anonymize them:\n", + "- NAME\n", + "- EMAIL\n", + "- PHONE\n", + "- US_SOCIAL_SECURITY_NUMBER\n", + "- ADDRESS\n", + "- CA_HEALTH_NUMBER\n", + "- PASSWORD\n", + "- IP_ADDRESS\n", + "- CA_SOCIAL_INSURANCE_NUMBER\n", + "- CREDIT_DEBIT_CARD_NUMBER\n", + "- AGE\n", + "- US_BANK_ACCOUNT_NUMBER\n", + "\n", + "In the input content there are 3 entries:\n", + "- grounding source: ground truth context for the model to base its response on (simulated here)\n", + "- query: the user input query\n", + "- guard_content: the model output (simulated here)\n", + "\n", + "This guardrail is going to be applied on the model **output** in this example, which means it won't affect the inputs at all.\n", + "\n", + "A full list of all available types is available in the [Amazon Bedrock Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html)." + ] + }, { "cell_type": "code", "execution_count": null, @@ -328,12 +388,12 @@ " )\n", " \n", " # Process the response\n", - " print(\"API Response:\")\n", - " print(json.dumps(response, indent=2))\n", + " #print(\"API Response:\")\n", + " #print(json.dumps(response, indent=2))\n", " \n", " # Check the action taken by the guardrail\n", " if response['action'] == 'GUARDRAIL_INTERVENED':\n", - " print(\"\\nGuardrail intervened. Output:\")\n", + " print(\"\\nGuardrail intervened. \\n\\nOutput:\")\n", " for output in response['outputs']:\n", " print(output['text'])\n", " else:\n", @@ -374,21 +434,15 @@ "Let's walk through this with a code example that demonstrates this process" ] }, - { - "cell_type": "markdown", - "id": "da00340f-253e-451b-840e-ebd77165d740", - "metadata": {}, - "source": [ - "### For our examples today we will use a Self-Hosted SageMaker Model, but this could be any third-party model as well\n", - "\n", - "We will use the `DeepSeek-R1-Distill-Llama-8B` model that we deployed earlier on a SageMaker Endpoint. " - ] - }, { "cell_type": "markdown", "id": "540f2797-24ec-4361-bd09-20d31efb5509", "metadata": {}, "source": [ + "### These examples use SageMaker hosted model endpoint, but this could be any third-party model as well\n", + "\n", + "We will use the `Qwen3-4B-Instruct-2507` model that we deployed earlier on a SageMaker Endpoint. \n", + "\n", "### Incorporating the ApplyGuardrail API into our Self-Hosted Model\n", "\n", "---\n", @@ -448,8 +502,11 @@ "\n", " def generate_text(self, inputs: str, max_new_tokens: int = 256, temperature: float = 0.0) -> str:\n", " \"\"\"Generate text using the specified SageMaker endpoint.\"\"\"\n", + "\n", + " messages = [{\"role\": \"user\", \"content\": inputs}]\n", + " \n", " payload = {\n", - " \"inputs\": inputs,\n", + " \"messages\": messages,\n", " \"parameters\": {\n", " \"max_new_tokens\": max_new_tokens,\n", " \"temperature\": temperature,\n", @@ -458,7 +515,7 @@ " }\n", " \n", " response = self.predictor.predict(payload)\n", - " return response.get('generated_text', '')\n", + " return response[\"choices\"][0][\"message\"][\"content\"]\n", "\n", " def analyze_text(self, grounding_source: str, query: str, guard_content: str, source: str) -> Tuple[bool, str, Dict[str, Any]]:\n", " \"\"\"\n", @@ -546,7 +603,11 @@ "id": "047ac6a0-293b-4ea4-b314-0aead2a5af75", "metadata": {}, "source": [ - "### Now let's see a Sample Usage in action " + "### Examples\n", + "\n", + "The following examples will allow you to test guardrail functionalize with your SageMaker hosted FM. \n", + "\n", + "The `test_generation_with_guardrail` function defined below will take a `TextGenerationWithGuardrails` along with model inputs, process the inputs with the supplied guardrail, send the inputs to your FM (if it passes), then process the model outputs with the guardrail before returning a final response." ] }, { @@ -565,7 +626,7 @@ " print(bold(\"\\n=== Input Analysis ===\\n\"))\n", " input_passed, input_message, input_details = text_gen.analyze_prompt(grounding_source, query)\n", " if not input_passed:\n", - " print(f\"Input Guardrail Intervened. The response to the User is: \\n\\n{input_message}\\n\")\n", + " print(f\"Input Guardrail Intervened. \\n\\nThe response to the User is: \\n\\n{input_message}\\n\")\n", " if print_api_responses:\n", " print(\"Full API Response:\")\n", " print(json.dumps(input_details, indent=2))\n", @@ -583,7 +644,7 @@ " print(\"Analyzing Model Response with the Response Guardrail\\n\")\n", " output_passed, output_message, output_details = text_gen.analyze_output(grounding_source, query, generated_text)\n", " if not output_passed:\n", - " print(f\"Output Guardrail Intervened. The response to the User is: \\n\\n{output_message}\\n\")\n", + " print(f\"Output Guardrail Intervened. \\n\\nThe response to the User is: \\n\\n{output_message}\\n\")\n", " if print_api_responses:\n", " print(\"Full API Response:\")\n", " print(json.dumps(output_details[\"outputs\"], indent=2))\n", @@ -593,13 +654,11 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "4c4c1e7b-dc61-4ea2-bdbf-519981d15f77", + "cell_type": "markdown", + "id": "9be81f8c-932f-4ab7-883c-009820c98e9c", "metadata": {}, - "outputs": [], "source": [ - "endpoint_name = FINETUNED_MODEL_ENDPOINT" + "Initialize the `TextGenerationWithGuardrails` class with the SageMaker endpoint and Guardrail, then test with a variety of scenarios." ] }, { @@ -610,7 +669,7 @@ "outputs": [], "source": [ "text_gen = TextGenerationWithGuardrails(\n", - " endpoint_name=endpoint_name,\n", + " endpoint_name=TUNED_ENDPOINT_NAME,\n", " guardrail_id=guardrail_id,\n", " guardrail_version=guardrail_version\n", ")" @@ -658,12 +717,20 @@ " temperature=0.0)" ] }, + { + "cell_type": "markdown", + "id": "3719d546-87a2-4f34-8f6f-ae0b5002625f", + "metadata": {}, + "source": [ + "Congratulations! You've successfully implemented a guardrail for your model to help protect the inputs and outputs of your application. Continue to the clean up section." + ] + }, { "cell_type": "markdown", "id": "77f33392-61ab-4fc5-9279-a62fbdc62a12", "metadata": {}, "source": [ - "#### Cleanup" + "## Clean Up" ] }, { @@ -686,7 +753,7 @@ "sagemaker_client = boto3.client('sagemaker')\n", "\n", "delete_sft_response = sagemaker_client.delete_endpoint(\n", - " EndpointName=FINETUNED_MODEL_ENDPOINT\n", + " EndpointName=TUNED_ENDPOINT_NAME\n", ")\n", "\n", "print(delete_sft_response)" @@ -700,7 +767,7 @@ "outputs": [], "source": [ "delete_sftcfg_response = sagemaker_client.delete_endpoint_config(\n", - " EndpointConfigName=FINETUNED_MODEL_ENDPOINT\n", + " EndpointConfigName=TUNED_ENDPOINT_NAME\n", ")\n", "print(delete_sftcfg_response)" ] @@ -708,7 +775,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d66513a7-75a7-4061-94e7-0524eaba11a4", + "id": "d2483ba6-8ef8-4307-a4ef-002e64ee7d20", "metadata": {}, "outputs": [], "source": [] diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_04_responsible_ai/scripts/requirements.txt b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_04_responsible_ai/scripts/requirements.txt new file mode 100644 index 0000000..1333cb9 --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_04_responsible_ai/scripts/requirements.txt @@ -0,0 +1 @@ +boto3==1.40.47 \ No newline at end of file From 3d777b36d1eec1c869344b836192d5ef3e440116 Mon Sep 17 00:00:00 2001 From: Giuseppe Zappia Date: Wed, 15 Oct 2025 10:28:29 -0600 Subject: [PATCH 20/22] added utils for task 2 --- .../utils.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/utils.py diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/utils.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/utils.py new file mode 100644 index 0000000..a64c9e9 --- /dev/null +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_02_customize_foundation_model/utils.py @@ -0,0 +1,109 @@ +import matplotlib.pyplot as plt +from typing import List, Dict +import boto3 + +def calculate_message_lengths(dataset: List[Dict]) -> List[int]: + """ + Calculate the length of content/text for each element in the dataset. + + Args: + dataset: List of dictionaries containing messages or text + + Returns: + List of word counts for each element + """ + try: + # First try to process as messages format + return [sum(len(msg["content"].split()) + for msg in element["messages"]) + for element in dataset] + except KeyError: + # Fallback to direct text/content format + key = "content" if "content" in dataset[0] else "text" + return [len(element[key].split()) for element in dataset] + +def plot_length_distribution(train_dataset: List[Dict], + validation_dataset: List[Dict], + bins: int = 20, + figsize: tuple = (10, 6)) -> None: + """ + Plot the distribution of text lengths from training and validation datasets. + + Args: + train_dataset: Training dataset + validation_dataset: Validation dataset + bins: Number of histogram bins + figsize: Figure size as (width, height) + """ + # Calculate lengths for both datasets + train_lengths = calculate_message_lengths(train_dataset) + val_lengths = calculate_message_lengths(validation_dataset) + combined_lengths = train_lengths + val_lengths + + # Create and configure the plot + plt.figure(figsize=figsize) + plt.hist(combined_lengths, + bins=bins, + alpha=0.7, + color="blue") + + # Set labels and title + plt.xlabel("Prompt Lengths (words)") + plt.ylabel("Frequency") + plt.title("Distribution of Input Lengths") + + plt.show() + + +def get_last_job_name(job_name_prefix): + sagemaker_client = boto3.client('sagemaker') + + matching_jobs = [] + next_token = None + + while True: + # Prepare the search parameters + search_params = { + 'Resource': 'TrainingJob', + 'SearchExpression': { + 'Filters': [ + { + 'Name': 'TrainingJobName', + 'Operator': 'Contains', + 'Value': job_name_prefix + }, + { + 'Name': 'TrainingJobStatus', + 'Operator': 'Equals', + 'Value': "Completed" + } + ] + }, + 'SortBy': 'CreationTime', + 'SortOrder': 'Descending', + 'MaxResults': 100 + } + + # Add NextToken if we have one + if next_token: + search_params['NextToken'] = next_token + + # Make the search request + search_response = sagemaker_client.search(**search_params) + + # Filter and add matching jobs + matching_jobs.extend([ + job['TrainingJob']['TrainingJobName'] + for job in search_response['Results'] + if job['TrainingJob']['TrainingJobName'].startswith(job_name_prefix) + ]) + + # Check if we have more results to fetch + next_token = search_response.get('NextToken') + if not next_token or matching_jobs: # Stop if we found at least one match or no more results + break + + if not matching_jobs: + raise ValueError(f"No completed training jobs found starting with prefix '{job_name_prefix}'") + + return matching_jobs[0] \ No newline at end of file From 8d574743aba21d0b0bde25566c7b583f9d4dafbc Mon Sep 17 00:00:00 2001 From: Giuseppe Zappia Date: Wed, 15 Oct 2025 13:03:20 -0600 Subject: [PATCH 21/22] fixed bug in guardrail setup --- .../05.01_fine-tuning-pipeline.ipynb | 139 ++++++++++++++++-- .../task_05_fmops/steps/pipeline_utils.py | 117 ++++++++------- 2 files changed, 185 insertions(+), 71 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index 70fddc9..280fef7 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -24,6 +24,7 @@ " \n", "\n", "The following figure shows the overview of the solution.\n", + "\n", "![](./ml-16670-arch-with-mlflow.png)" ] }, @@ -48,7 +49,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T17:56:50.196738Z", + "iopub.status.busy": "2025-10-15T17:56:50.196483Z", + "iopub.status.idle": "2025-10-15T17:56:58.331883Z", + "shell.execute_reply": "2025-10-15T17:56:58.331192Z", + "shell.execute_reply.started": "2025-10-15T17:56:50.196718Z" + } + }, "outputs": [], "source": [ "%pip install -r ./scripts/requirements.txt --upgrade --quiet" @@ -57,7 +66,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T17:56:58.332985Z", + "iopub.status.busy": "2025-10-15T17:56:58.332771Z", + "iopub.status.idle": "2025-10-15T17:56:58.338882Z", + "shell.execute_reply": "2025-10-15T17:56:58.338429Z", + "shell.execute_reply.started": "2025-10-15T17:56:58.332962Z" + } + }, "outputs": [], "source": [ "from IPython import get_ipython\n", @@ -76,7 +93,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:10:38.890122Z", + "iopub.status.busy": "2025-10-15T18:10:38.889793Z", + "iopub.status.idle": "2025-10-15T18:10:40.611213Z", + "shell.execute_reply": "2025-10-15T18:10:40.610667Z", + "shell.execute_reply.started": "2025-10-15T18:10:38.890099Z" + } + }, "outputs": [], "source": [ "import os\n", @@ -108,13 +133,21 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:12:29.689442Z", + "iopub.status.busy": "2025-10-15T18:12:29.689175Z", + "iopub.status.idle": "2025-10-15T18:12:30.791433Z", + "shell.execute_reply": "2025-10-15T18:12:30.790896Z", + "shell.execute_reply.started": "2025-10-15T18:12:29.689422Z" + } + }, "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", "role = sagemaker.get_execution_role()\n", "instance_type = \"ml.m5.xlarge\"\n", - "pipeline_name = \"AIM405-qwen3-finetune-pipeline\"\n", + "pipeline_name = \"qwen3-finetune-pipeline\"\n", "bucket_name = sagemaker_session.default_bucket()\n", "default_prefix = sagemaker_session.default_bucket_prefix\n", "if default_prefix:\n", @@ -152,7 +185,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:12:30.838220Z", + "iopub.status.busy": "2025-10-15T18:12:30.837950Z", + "iopub.status.idle": "2025-10-15T18:12:31.008197Z", + "shell.execute_reply": "2025-10-15T18:12:31.007542Z", + "shell.execute_reply.started": "2025-10-15T18:12:30.838200Z" + } + }, "outputs": [], "source": [ "mlflow_tracking_server_arn = \"\"\n", @@ -180,7 +221,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:12:32.608898Z", + "iopub.status.busy": "2025-10-15T18:12:32.608642Z", + "iopub.status.idle": "2025-10-15T18:12:32.612632Z", + "shell.execute_reply": "2025-10-15T18:12:32.612142Z", + "shell.execute_reply.started": "2025-10-15T18:12:32.608878Z" + } + }, "outputs": [], "source": [ "%%writefile config.yaml\n", @@ -203,7 +252,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:12:33.245143Z", + "iopub.status.busy": "2025-10-15T18:12:33.244889Z", + "iopub.status.idle": "2025-10-15T18:12:33.247752Z", + "shell.execute_reply": "2025-10-15T18:12:33.247285Z", + "shell.execute_reply.started": "2025-10-15T18:12:33.245122Z" + } + }, "outputs": [], "source": [ "# Set path to config file\n", @@ -221,6 +278,13 @@ "cell_type": "code", "execution_count": null, "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:12:34.760569Z", + "iopub.status.busy": "2025-10-15T18:12:34.760310Z", + "iopub.status.idle": "2025-10-15T18:12:35.448363Z", + "shell.execute_reply": "2025-10-15T18:12:35.447833Z", + "shell.execute_reply.started": "2025-10-15T18:12:34.760549Z" + }, "scrolled": true }, "outputs": [], @@ -362,7 +426,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:12:37.083014Z", + "iopub.status.busy": "2025-10-15T18:12:37.082756Z", + "iopub.status.idle": "2025-10-15T18:12:37.095488Z", + "shell.execute_reply": "2025-10-15T18:12:37.094990Z", + "shell.execute_reply.started": "2025-10-15T18:12:37.082991Z" + } + }, "outputs": [], "source": [ "%%bash\n", @@ -440,7 +512,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:12:40.136027Z", + "iopub.status.busy": "2025-10-15T18:12:40.135746Z", + "iopub.status.idle": "2025-10-15T18:12:40.542117Z", + "shell.execute_reply": "2025-10-15T18:12:40.541629Z", + "shell.execute_reply.started": "2025-10-15T18:12:40.136005Z" + } + }, "outputs": [], "source": [ "from sagemaker.s3 import S3Uploader\n", @@ -481,17 +561,33 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:12:43.868659Z", + "iopub.status.busy": "2025-10-15T18:12:43.868403Z", + "iopub.status.idle": "2025-10-15T18:12:44.216288Z", + "shell.execute_reply": "2025-10-15T18:12:44.215721Z", + "shell.execute_reply.started": "2025-10-15T18:12:43.868639Z" + } + }, "outputs": [], "source": [ "from steps import pipeline_utils\n", - "guardrail_id, guardrail_version =pipeline_utils.get_or_create_guardrail()" + "guardrail_id, guardrail_version = pipeline_utils.get_or_create_guardrail()" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:13:22.801488Z", + "iopub.status.busy": "2025-10-15T18:13:22.801207Z", + "iopub.status.idle": "2025-10-15T18:13:25.970853Z", + "shell.execute_reply": "2025-10-15T18:13:25.970278Z", + "shell.execute_reply.started": "2025-10-15T18:13:22.801468Z" + } + }, "outputs": [], "source": [ "from steps import (\n", @@ -618,6 +714,13 @@ "cell_type": "code", "execution_count": null, "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:13:25.972041Z", + "iopub.status.busy": "2025-10-15T18:13:25.971610Z", + "iopub.status.idle": "2025-10-15T18:13:38.971434Z", + "shell.execute_reply": "2025-10-15T18:13:38.970889Z", + "shell.execute_reply.started": "2025-10-15T18:13:25.972020Z" + }, "scrolled": true }, "outputs": [], @@ -637,7 +740,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2025-10-15T18:13:38.972413Z", + "iopub.status.busy": "2025-10-15T18:13:38.972199Z", + "iopub.status.idle": "2025-10-15T18:13:39.429711Z", + "shell.execute_reply": "2025-10-15T18:13:39.429119Z", + "shell.execute_reply.started": "2025-10-15T18:13:38.972394Z" + } + }, "outputs": [], "source": [ "execution = pipeline.start()" diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py index 3099f8d..30609d7 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/steps/pipeline_utils.py @@ -78,62 +78,65 @@ def invoke_sagemaker_endpoint(payload, endpoint_name): def get_or_create_guardrail(): guardrail_client = boto3.client('bedrock') guardrail_name = "ExampleMedicalGuardrail" - try: - # Try to get the guardrail - response = guardrail_client.list_guardrails() - for guardrail in response.get('guardrails', []): - if guardrail['name'] == guardrail_name: - guardrail_id = guardrail['id'] - response = guardrail_client.get_guardrail( - guardrailIdentifier=guardrail_id - ) - guardrail_version = response["version"] - print(f"Found Guardrail {guardrail_id}:{guardrail_version}") - except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == 'ResourceNotFoundException': - # Guardrail doesn't exist, create it - try: - guardrail = guardrail_client.create_guardrail( - name="ExampleMedicalGuardrail", - description='Example of a Guardrail for Medical Use Cases', - topicPolicyConfig={ - 'topicsConfig': [{ - 'name': 'Block Pharmaceuticals', - 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.', - 'type': 'DENY', + + guardrail_id = None + guardrail_version = None + + # Try to get the guardrail + response = guardrail_client.list_guardrails() + for guardrail in response.get('guardrails', []): + if guardrail['name'] == guardrail_name: + guardrail_id = guardrail['id'] + response = guardrail_client.get_guardrail( + guardrailIdentifier=guardrail_id + ) + guardrail_version = response["version"] + print(f"Found Guardrail {guardrail_id}:{guardrail_version}") + break + + if not guardrail_id: + try: + guardrail = guardrail_client.create_guardrail( + name="ExampleMedicalGuardrail", + description='Example of a Guardrail for Medical Use Cases', + topicPolicyConfig={ + 'topicsConfig': [{ + 'name': 'Block Pharmaceuticals', + 'definition': 'This model cannot recommend one pharmaceutical over another. Generic prescriptions consistent with medical expertise and clinical diagnoses only.', + 'type': 'DENY', + 'inputAction': 'BLOCK', + 'outputAction': 'BLOCK', + }] + }, + sensitiveInformationPolicyConfig={ + 'piiEntitiesConfig': [ + { + 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER', + 'action': 'BLOCK', 'inputAction': 'BLOCK', - 'outputAction': 'BLOCK', - }] - }, - sensitiveInformationPolicyConfig={ - 'piiEntitiesConfig': [ - { - 'type': 'UK_NATIONAL_HEALTH_SERVICE_NUMBER', - 'action': 'BLOCK', - 'inputAction': 'BLOCK', - 'outputAction': 'BLOCK' - }, - ] - }, - contextualGroundingPolicyConfig={ - 'filtersConfig': [ - { - 'type': 'RELEVANCE', - 'threshold': 0.9, - 'action': 'BLOCK', - 'enabled': True - }, - ] - }, - blockedInputMessaging="ExampleMedicalGuardrail has blocked this input.", - blockedOutputsMessaging="ExampleMedicalGuardrail has blocked this output." - ) - guardrail_id = guardrail['guardrailId'] - guardrail_version = guardrail['version'] - - print(f"Created new guardrail '{guardrail_id}:{guardrail_version}'") - except botocore.exceptions.ClientError as create_error: - print(f"Error creating guardrail: {create_error}") - else: - print(f"Error checking guardrail: {e}") + 'outputAction': 'BLOCK' + }, + ] + }, + contextualGroundingPolicyConfig={ + 'filtersConfig': [ + { + 'type': 'RELEVANCE', + 'threshold': 0.9, + 'action': 'BLOCK', + 'enabled': True + }, + ] + }, + blockedInputMessaging="ExampleMedicalGuardrail has blocked this input.", + blockedOutputsMessaging="ExampleMedicalGuardrail has blocked this output." + ) + + guardrail_id = guardrail['guardrailId'] + guardrail_version = guardrail['version'] + + print(f"Created new guardrail '{guardrail_id}:{guardrail_version}'") + except botocore.exceptions.ClientError as create_error: + print(f"Error creating guardrail: {create_error}") + return guardrail_id, guardrail_version \ No newline at end of file From 6ae451b100b5b0e274baf9afff7bb7e42c932d97 Mon Sep 17 00:00:00 2001 From: Dan Ferguson Date: Thu, 16 Oct 2025 14:14:10 -0400 Subject: [PATCH 22/22] Updated notebook prerequisites to discuss model access requirements --- .../task_05_fmops/05.00_fmops_examples.ipynb | 3 ++- .../task_05_fmops/05.01_fine-tuning-pipeline.ipynb | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb index 4d7f249..3f443b4 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.00_fmops_examples.ipynb @@ -16,7 +16,8 @@ "## Prerequisites \n", "Before you begin, make sure you have the following prerequisites in place:\n", "\n", - "- MLflow tracking server: If you're running this lab in a workshop environment, a MLflow tracking server has already been created for you. If you need to create a MLflow tracking server, follow the [documentation](https://docs.aws.amazon.com/sagemaker/latest/dg/mlflow-create-tracking-server.html)" + "- MLflow tracking server: If you're running this lab in a workshop environment, a MLflow tracking server has already been created for you. If you need to create a MLflow tracking server, follow the [documentation](https://docs.aws.amazon.com/sagemaker/latest/dg/mlflow-create-tracking-server.html)\n", + "- Haiku 3 Model Access: In order to use the LLM-as-a-Judge feature of these labs, you will need Amazon Bedrock model access to Haiku 3. Follow the [documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) for more details on how to modify model access." ] }, { diff --git a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb index 280fef7..ab2a97a 100644 --- a/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb +++ b/workshops/fine-tuning-with-sagemakerai-and-bedrock/task_05_fmops/05.01_fine-tuning-pipeline.ipynb @@ -35,7 +35,8 @@ "## Prerequisites \n", "Before you begin, make sure you have the following prerequisites in place:\n", "\n", - "- MLflow tracking server: If you're running this lab in a workshop environment, a MLflow tracking server has already been created for you. If you need to create a MLflow tracking server, follow the [documentation](https://docs.aws.amazon.com/sagemaker/latest/dg/mlflow-create-tracking-server.html)" + "- MLflow tracking server: If you're running this lab in a workshop environment, a MLflow tracking server has already been created for you. If you need to create a MLflow tracking server, follow the [documentation](https://docs.aws.amazon.com/sagemaker/latest/dg/mlflow-create-tracking-server.html)\n", + "- Haiku 3 Model Access: In order to use the LLM-as-a-Judge feature of these labs, you will need Amazon Bedrock model access to Haiku 3. Follow the [documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) for more details on how to modify model access." ] }, {