From 65429a21a18078030512d93425e31d2bf3bde34d Mon Sep 17 00:00:00 2001 From: future-xy Date: Mon, 3 Nov 2025 11:34:56 +0000 Subject: [PATCH 1/3] feat: add versioning --- docs/api/cli.md | 8 +- docs/stable/community/_category_.json | 4 - docs/stable/community/meetups.md | 11 - docs/stable/community/talks.md | 8 - docs/stable/deployment/_category_.json | 4 - docs/stable/deployment/multi_machine.md | 296 ------------- docs/stable/deployment/single_machine.md | 217 --------- docs/stable/deployment/slurm_cluster.md | 411 ------------------ docs/stable/developer/_category_.json | 4 - .../developer/supporting_a_new_hardware.md | 25 -- docs/stable/features/_category_.json | 4 - docs/stable/features/live_migration.md | 211 --------- docs/stable/features/peft_lora_fine_tuning.md | 328 -------------- docs/stable/features/peft_lora_serving.md | 116 ----- docs/stable/features/quantized_models.md | 175 -------- .../features/storage_aware_scheduling.md | 123 ------ docs/stable/getting_started.md | 136 ------ docs/stable/intro.md | 38 -- docs/stable/models/_category_.json | 4 - docs/stable/models/supported_models.md | 13 - docs/stable/store/_category_.json | 4 - docs/stable/store/quantization.md | 102 ----- docs/stable/store/quickstart.md | 245 ----------- docs/stable/store/rocm_quickstart.md | 164 ------- docusaurus.config.js | 18 +- sidebars.js | 10 +- src/pages/index.js | 2 +- 27 files changed, 30 insertions(+), 2651 deletions(-) delete mode 100644 docs/stable/community/_category_.json delete mode 100644 docs/stable/community/meetups.md delete mode 100644 docs/stable/community/talks.md delete mode 100644 docs/stable/deployment/_category_.json delete mode 100644 docs/stable/deployment/multi_machine.md delete mode 100644 docs/stable/deployment/single_machine.md delete mode 100644 docs/stable/deployment/slurm_cluster.md delete mode 100644 docs/stable/developer/_category_.json delete mode 100644 docs/stable/developer/supporting_a_new_hardware.md delete mode 100644 docs/stable/features/_category_.json delete mode 100644 docs/stable/features/live_migration.md delete mode 100644 docs/stable/features/peft_lora_fine_tuning.md delete mode 100644 docs/stable/features/peft_lora_serving.md delete mode 100644 docs/stable/features/quantized_models.md delete mode 100644 docs/stable/features/storage_aware_scheduling.md delete mode 100644 docs/stable/getting_started.md delete mode 100644 docs/stable/intro.md delete mode 100644 docs/stable/models/_category_.json delete mode 100644 docs/stable/models/supported_models.md delete mode 100644 docs/stable/store/_category_.json delete mode 100644 docs/stable/store/quantization.md delete mode 100644 docs/stable/store/quickstart.md delete mode 100644 docs/stable/store/rocm_quickstart.md diff --git a/docs/api/cli.md b/docs/api/cli.md index 127e848..7769797 100644 --- a/docs/api/cli.md +++ b/docs/api/cli.md @@ -29,10 +29,10 @@ The CLI organizes commands into clearly scoped modules. This document outlines e Before using the `sllm` commands, you need to start the ServerlessLLM cluster. Follow the guides below to set up your cluster: -- [Single Machine Deployment](../stable/getting_started.md) -- [Single Machine Deployment (From Scratch)](../stable/deployment/single_machine.md) -- [Multi-Machine Deployment](../stable/deployment/multi_machine.md) -- [SLURM Cluster Deployment](../stable/deployment/slurm_cluster.md) +- [Single Machine Deployment](../getting_started.md) +- [Single Machine Deployment (From Scratch)](../deployment/single_machine.md) +- [Multi-Machine Deployment](../deployment/multi_machine.md) +- [SLURM Cluster Deployment](../deployment/slurm_cluster.md) After setting up the ServerlessLLM cluster, you can use the commands listed below to manage and interact with your models. diff --git a/docs/stable/community/_category_.json b/docs/stable/community/_category_.json deleted file mode 100644 index 8a81fd9..0000000 --- a/docs/stable/community/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "Community", - "position": 8 -} diff --git a/docs/stable/community/meetups.md b/docs/stable/community/meetups.md deleted file mode 100644 index 8e3811a..0000000 --- a/docs/stable/community/meetups.md +++ /dev/null @@ -1,11 +0,0 @@ -# ServerlessLLM Meetups - -We host regular biweekly developer meetings online. We will share project updates from the ServerlessLLM developer team presented during these meetings. Please find the materials of our previous meetups below: - -Date |Topic |Slides ----------------|-------------|--------- -February 21st 2025 | Fine Tuning | [Slides](https://docs.google.com/presentation/d/1rnw3mieAAbMabDIoIGS-ciMGc3hJ7AICYSaNJp-Fk4s/edit?usp=sharing) -March 7th 2025 |Quantization |[Slides](https://docs.google.com/presentation/d/1uSbP-LzGbbvPsemIAE6jCFsggYm_ATxQguCHDmdwoXE/edit?usp=sharing) - -We are always looking for contributors to join us on the developer team. If you are interested in contributing, consult our [job board](https://github.com/orgs/ServerlessLLM/projects/2) and claim a feature. For any other questions, please contact us on [this email](mailto:Y.Fu@ed.ac.uk) or on [our Discord server](https://discord.gg/AEF8Gduvm8). - diff --git a/docs/stable/community/talks.md b/docs/stable/community/talks.md deleted file mode 100644 index 66a2531..0000000 --- a/docs/stable/community/talks.md +++ /dev/null @@ -1,8 +0,0 @@ -# ServerlessLLM Talks - -Materials for ServerlessLLM talks will be listed here. - -Topic |Location |Date |Links --------------|----------------|---------------|------------------------------------ -Efficient Sharing of AI Infrastructures with Specialized Serverless Computing | University of Pennsylvania |January 29th 2025 |[Slides](https://drive.google.com/file/d/17GwXsqaDDS7Xw8nX_-RaKiwpaPQgu9WD/view) \| [Event](https://asset.seas.upenn.edu/event/yao-fu-university-of-edinburgh/) -ServerlessLLM Tutorial | SESAME'25 | March 31st 2025 |[Slides](https://docs.google.com/presentation/d/1ioGCVpsg0x3oCxX19EiE820aMiY22X5MG6jgImZ1W18/edit?usp=sharing) \| [Event](https://sesame25.github.io/) diff --git a/docs/stable/deployment/_category_.json b/docs/stable/deployment/_category_.json deleted file mode 100644 index 534be1d..0000000 --- a/docs/stable/deployment/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "Deployment", - "position": 3 -} diff --git a/docs/stable/deployment/multi_machine.md b/docs/stable/deployment/multi_machine.md deleted file mode 100644 index 21583c3..0000000 --- a/docs/stable/deployment/multi_machine.md +++ /dev/null @@ -1,296 +0,0 @@ ---- -sidebar_position: 2 ---- - -# Multi-machine - -This guide will help you get started with running ServerlessLLM on multiple machines using Docker containers. You'll learn how to set up a head node on one machine and connect worker nodes from different machines using Docker, ensuring proper network communication between the containers. You can extend this setup to use as many nodes as you need. - -## Prerequisites - -This guide requires **two machines**: -- One machine for the head node (no GPU required) -- One machine with an NVIDIA GPU to serve as the worker node - -You can add more worker machines with GPUs as needed to scale out your deployment. - -### For All Machines - -Ensure you have the following installed and configured on all machines (both head node and worker machines): - -1. **Docker**: Installed on your system. You can download it from [here](https://docs.docker.com/get-docker/). -2. **Network connectivity**: Ensure all machines can communicate with each other on the required ports (6379 for Ray, 8343 for ServerlessLLM API, and 8073 for storage service). - -:::tip -The **ServerlessLLM CLI** (`pip install serverless-llm`) can be installed on any machine that needs to manage model deployments. This could be your local computer or any machine within the cluster that can connect to the head node. -::: - -### For Worker Machines Only - -These requirements are only necessary for the worker machines that will run the models: - -1. **GPUs**: At least one NVIDIA GPU is required on each worker machine. If you have multiple GPUs, you can adjust the Docker configuration accordingly. -2. **NVIDIA Docker Toolkit**: This enables Docker to utilize NVIDIA GPUs. Follow the installation guide [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). - -## Multi-Machine Setup - -We'll start a head node on one machine using Docker, then add a worker node from another machine using Docker containers with host networking. - -### Step 1: Start the Head Node - -1. **Start the head node using Docker:** - -```bash -# Get the machine's IP address that will be accessible to other machines -export HEAD_IP=$(hostname -I | awk '{print $1}') -echo "Head node IP address: $HEAD_IP" - -docker run -d \ - --name sllm_head \ - --network host \ - -e MODE=HEAD \ - -e RAY_NODE_IP=$HEAD_IP \ - serverlessllm/sllm:latest -``` - -:::important -For multi-machine setups, setting the `RAY_NODE_IP` is critical. It should be set to an IP address that is accessible from all worker machines. The command above attempts to automatically determine your machine's primary IP, but in complex network environments, you may need to specify it manually. - -If your machine has multiple network interfaces, ensure you use the IP that other machines in your network can access. -::: - -:::tip -If you don't have the ServerlessLLM Docker image locally, Docker will automatically pull it from the registry. You can also adjust the CPU and resource allocations by setting additional environment variables like `RAY_NUM_CPUS` and `RAY_RESOURCES`. -::: - -2. **Verify the head node is running and note the external IP:** - -```bash -docker logs sllm_head -``` - -Expected output should include: - -```bash -> docker logs sllm_head -... -2025-05-29 14:29:46,211 INFO scripts.py:744 -- Local node IP: 129.215.164.107 -... -(SllmController pid=380) INFO 05-29 14:29:53 controller.py:59] Starting store manager -(SllmController pid=380) INFO 05-29 14:29:56 controller.py:68] Starting scheduler -(StoreManager pid=417) INFO 05-29 14:29:56 store_manager.py:226] Initializing store manager -(StoreManager pid=417) INFO 05-29 14:29:56 store_manager.py:237] Initializing cluster and collecting hardware info -(StoreManager pid=417) ERROR 05-29 14:29:56 store_manager.py:242] No worker nodes found -INFO: Started server process [1] -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://0.0.0.0:8343 (Press CTRL+C to quit) -(FcfsScheduler pid=456) INFO 05-29 14:29:56 fcfs_scheduler.py:54] Starting FCFS scheduler -(FcfsScheduler pid=456) INFO 05-29 14:29:56 fcfs_scheduler.py:111] Starting control loop -``` - -Make note of the IP address shown in the logs. This is the address that worker nodes will use to connect to the head node. - -### Step 2: Start Worker Node on a Different Machine - -:::tip -You can adjust the memory pool size and other parameters based on the resources available on your worker machine. -::: - -1. **On the worker machine, create a directory for model storage:** - -```bash -mkdir -p /path/to/your/models -export MODEL_FOLDER=/path/to/your/models -``` - -Replace `/path/to/your/models` with the actual path where you want to store the models. - -2. **Start the worker node:** - -```bash -# Replace with the actual IP address of the head node from the previous step -# DO NOT copy-paste this line directly - update with your actual head node IP -export HEAD_IP= -``` - -```bash -# Get the worker machine's IP address that will be accessible to the head node -export WORKER_IP=$(hostname -I | awk '{print $1}') -echo "Worker node IP address: $WORKER_IP" - -docker run -d \ - --name sllm_worker_0 \ - --network host \ - --gpus '"device=0"' \ - -e WORKER_ID=0 \ - -e STORAGE_PATH=/models \ - -e MODE=WORKER \ - -e RAY_HEAD_ADDRESS=${HEAD_IP}:6379 \ - -e RAY_NODE_IP=$WORKER_IP \ - -v ${MODEL_FOLDER}:/models \ - serverlessllm/sllm:latest \ - --mem-pool-size 4GB --registration-required true -``` - -:::important -For multi-machine setups, setting the `RAY_NODE_IP` on worker nodes is just as critical as on the head node. It should be set to an IP address that is accessible from the head node. Without this, workers might report internal Docker IPs that aren't accessible across machines. - -Make sure to replace `192.168.1.100` with the actual IP address of your head node that you noted earlier. -::: - -3. **Verify worker node is connected:** - -On the worker machine, check if the worker has properly connected to the Ray cluster: - -```bash -docker exec -it sllm_worker_0 bash -c "source /opt/conda/etc/profile.d/conda.sh && conda activate worker && ray status" -``` - -Expected output should include both the head node and worker node resources: - -```bash -> docker exec -it sllm_worker_0 bash -c "source /opt/conda/etc/profile.d/conda.sh && conda activate worker && ray status" -======== Autoscaler status: 2025-05-29 14:42:30.434645 ======== -Node status ---------------------------------------------------------------- -Active: - 1 node_f0a8e97ca64c64cebd551f469a38d0d66ce304f7cc1cc9696fe33cf3 - 1 node_3b7db178afb8bdb16460d0cb6463dc7b9b3afbcc00753c3be110c9b3 -Pending: - (no pending nodes) -Recent failures: - (no failures) - -Resources ---------------------------------------------------------------- -Usage: - 3.0/52.0 CPU - 0.0/1.0 GPU - 0.30000000000000004/1.0 control_node - 0B/526.36GiB memory - 0B/18.63GiB object_store_memory - 0.0/1.0 worker_id_0 - 0.0/1.0 worker_node - -Demands: - (no resource demands) -``` - -This output confirms that both the head node and worker node are properly connected and their resources are recognized by the Ray cluster. - -:::tip -**Adding more worker nodes:** You can add more worker nodes by repeating Step 2 on additional machines with GPUs. Just make sure to: -1. Use a unique `WORKER_ID` for each worker (1, 2, 3, etc.) -2. Point each worker to the same head node IP address -3. Ensure each worker has its own `RAY_NODE_IP` set correctly -::: - -### Step 3: Use `sllm` to manage models - -#### Configure the Environment - -**On any machine with `sllm` installed, set the `LLM_SERVER_URL` environment variable:** - -> Replace `` with the actual IP address of the head node. - -```bash -export LLM_SERVER_URL=http://:8343 -``` - -#### Deploy a Model Using `sllm` - -```bash -sllm deploy --model facebook/opt-1.3b -``` - -> Note: This command will spend some time downloading the model from the Hugging Face Model Hub. You can use any model from the [Hugging Face Model Hub](https://huggingface.co/models) by specifying the model name in the `--model` argument. - -Expected output: - -```bash -INFO 07-24 06:51:32 deploy.py:83] Model registered successfully. -``` - -### Step 4: Query the Model Using OpenAI API Client - -**You can query the model using any OpenAI API client. For example, use the following command:** - -**Make sure the model is successfully deployed before querying.** - -> Replace `` with the actual IP address of the head node. - -```bash -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-1.3b", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ] - }' -``` - -Expected output: - -```json -{"id":"chatcmpl-23d3c0e5-70a0-4771-acaf-bcb2851c6ea6","object":"chat.completion","created":1721706121,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} -``` - -#### Delete a Deployed Model Using `sllm` - -When you're done using a model, you can delete it: - -```bash -sllm delete facebook/opt-1.3b -``` - -This will remove the specified model from the ServerlessLLM server. - -## Clean Up - -To stop and remove all ServerlessLLM containers: - -1. **Stop all containers:** - -```bash -# On head node machine -docker stop sllm_head -docker rm sllm_head - -# On each worker machine -docker stop sllm_worker_0 # Use appropriate container name (sllm_worker_1, sllm_worker_2, etc.) -docker rm sllm_worker_0 -``` - -2. **Optional: Remove the Docker image:** - -```bash -docker rmi serverlessllm/sllm:latest -``` - -:::tip -If you don't have the ServerlessLLM Docker image locally, Docker will automatically pull it from the registry. You can also adjust the CPU and resource allocations by setting additional environment variables like `RAY_NUM_CPUS` and `RAY_RESOURCES`. -::: - -## Troubleshooting - -### Network Issues - -1. **Connection refused errors**: Ensure that firewalls on all machines allow traffic on ports 6379, 8343, and 8073. - -2. **Ray cluster connection issues**: - - Verify that the head node IP address is correct and that the Ray port (6379) is accessible from worker machines - - Ensure both head and worker nodes have their `RAY_NODE_IP` set to an IP address that is accessible from other machines - - Check that you're not using private Docker network IPs (typically 172.x.x.x) which aren't accessible across machines - -3. **Workers can't connect to head node**: - - Make sure the `RAY_HEAD_ADDRESS` points to the external IP of the head node, not localhost or an internal Docker IP - - Verify network connectivity with `ping` or `telnet` from worker machines to the head node IP on port 6379 - -4. **GPU access issues**: Make sure the NVIDIA Docker toolkit is properly installed and that the `--gpus` flag is used for worker containers. - -### Container Management - -- **View running containers**: `docker ps` \ No newline at end of file diff --git a/docs/stable/deployment/single_machine.md b/docs/stable/deployment/single_machine.md deleted file mode 100644 index 758980a..0000000 --- a/docs/stable/deployment/single_machine.md +++ /dev/null @@ -1,217 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Single machine (from scratch) - -This guide provides instructions for setting up ServerlessLLM from scratch on a single machine. This 'from scratch' approach means you will manually initialize and manage the Ray cluster components. It involves using multiple terminal sessions, each configured with a distinct Conda environment, to run the head and worker processes on the same physical machine, effectively simulating a multi-node deployment locally. - -:::note -We strongly recommend using Docker (Compose) as detailed in the [Docker Compose guide](../getting_started.md). Docker provides a smoother and generally easier setup process. Follow this guide only if Docker is not a suitable option for your environment. -::: - -## Installation - -### Requirements - -Ensure your system meets the following prerequisites: - -- **OS**: Ubuntu 20.04 -- **Python**: 3.10 -- **GPU**: NVIDIA GPU with compute capability 7.0 or higher - -### Installing with pip - -Follow these steps to install ServerlessLLM using pip: - -**Create the head environment:** - -```bash -# Create and activate a conda environment -conda create -n sllm python=3.10 -y -conda activate sllm - -# Install ServerlessLLM and its store component -pip install serverless-llm serverless-llm-store -``` - -**Create the worker environment:** - -```bash -# Create and activate a conda environment -conda create -n sllm-worker python=3.10 -y -conda activate sllm-worker - -# Install ServerlessLLM (worker version) and its store component -pip install "serverless-llm[worker]" serverless-llm-store -``` - -:::note -If you plan to integrate vLLM with ServerlessLLM, a patch needs to be applied to the vLLM repository. For detailed instructions, please refer to the [vLLM Patch](#vllm-patch) section. -::: - -### Installing from Source - -To install ServerlessLLM from source, follow these steps: - -1. Clone the repository: - ```bash - git clone https://github.com/ServerlessLLM/ServerlessLLM.git - cd ServerlessLLM - ``` - -2. Create the head environment: - ```bash - # Create and activate a conda environment - conda create -n sllm python=3.10 -y - conda activate sllm - - # Install sllm_store (pip install is recommended for speed) - cd sllm_store && rm -rf build - pip install . - cd .. - - # Install ServerlessLLM - pip install . - ``` - -3. Create the worker environment: - ```bash - # Create and activate a conda environment - conda create -n sllm-worker python=3.10 -y - conda activate sllm-worker - - # Install sllm_store (pip install is recommended for speed) - cd sllm_store && rm -rf build - pip install . - cd .. - - # Install ServerlessLLM (worker version) - pip install ".[worker]" - ``` - -### vLLM Patch - -To use vLLM with ServerlessLLM, you must apply a patch. The patch file is located at `sllm_store/vllm_patch/sllm_load.patch` within the ServerlessLLM repository. This patch has been tested with vLLM version `0.9.0.1`. - -Apply the patch using the following script: - -```bash -conda activate sllm-worker -./sllm_store/vllm_patch/patch.sh -``` - -## Running ServerlessLLM Locally - -These steps describe how to run ServerlessLLM on your local machine. - -### 1. Start a Local Ray Cluster - -First, initiate a local Ray cluster. This cluster will consist of one head node and one worker node (on the same machine). - -**Start the head node:** - -Open a new terminal and run: - -```bash -conda activate sllm -ray start --head --port=6379 --num-cpus=4 --num-gpus=0 \ - --resources='{"control_node": 1}' --block -``` - -**Start the worker node:** - -Open another new terminal and run: - -```bash -conda activate sllm-worker -export CUDA_VISIBLE_DEVICES=0 # Or your desired GPU ID -ray start --address=0.0.0.0:6379 --num-cpus=4 --num-gpus=1 \ - --resources='{"worker_node": 1, "worker_id_0": 1}' --block -``` - -### 2. Start the ServerlessLLM Store Server - -Next, start the ServerlessLLM Store server. By default, it uses `./models` as the storage path. - -Open a new terminal and run: - -```bash -conda activate sllm-worker -export CUDA_VISIBLE_DEVICES=0 # Or your desired GPU ID -sllm-store start -``` - -Expected output: - -```log -$ sllm-store start -INFO 12-31 17:13:23 cli.py:58] Starting gRPC server -INFO 12-31 17:13:23 server.py:34] StorageServicer: storage_path=./models, mem_pool_size=4294967296, num_thread=4, chunk_size=33554432, registration_required=False -WARNING: Logging before InitGoogleLogging() is written to STDERR -I20241231 17:13:23.947276 2165054 checkpoint_store.cpp:41] Number of GPUs: 1 -I20241231 17:13:23.947299 2165054 checkpoint_store.cpp:43] I/O threads: 4, chunk size: 32MB -I20241231 17:13:23.947309 2165054 checkpoint_store.cpp:45] Storage path: "./models" -I20241231 17:13:24.038651 2165054 checkpoint_store.cpp:71] GPU 0 UUID: c9938b31-33b0-e02f-24c5-88bd6fbe19ad -I20241231 17:13:24.038700 2165054 pinned_memory_pool.cpp:29] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes -I20241231 17:13:25.557906 2165054 checkpoint_store.cpp:83] Memory pool created with 4GB -INFO 12-31 17:13:25 server.py:243] Starting gRPC server on 0.0.0.0:8073 -``` - -### 3. Start ServerlessLLM - -Now, start the ServerlessLLM service process using `sllm start`. - - -Open a new terminal and run: - -```bash -sllm start -``` - -At this point, you should have four terminals open: one for the Ray head node, one for the Ray worker node, one for the ServerlessLLM Store server, and one for the ServerlessLLM service (started via `sllm start`). - -### 4. Deploy a Model - -With all services running, you can deploy a model. - -Open a new terminal and run: - -```bash -conda activate sllm -sllm deploy --model facebook/opt-1.3b -``` - -This command downloads the specified model from Hugging Face Hub. To load a model from a local path, you can use a `config.json` file. Refer to the [CLI API documentation](../../api/cli.md#example-configuration-file-configjson) for details. - -### 5. Query the Model - -Once the model is deployed, you can query it using any OpenAI API-compatible client. For example, use the following `curl` command: - -```bash -curl http://127.0.0.1:8343/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-1.3b", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ] - }' -``` - -Expected output: - -```json -{"id":"chatcmpl-9f812a40-6b96-4ef9-8584-0b8149892cb9","object":"chat.completion","created":1720021153,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} -``` - -## Clean Up - -To delete a deployed model, use the following command: - -```bash -sllm delete facebook/opt-1.3b -``` - -This command removes the specified model from the ServerlessLLM server. \ No newline at end of file diff --git a/docs/stable/deployment/slurm_cluster.md b/docs/stable/deployment/slurm_cluster.md deleted file mode 100644 index 5d3a476..0000000 --- a/docs/stable/deployment/slurm_cluster.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -sidebar_position: 3 ---- - -# SLURM cluster - -This guide will help you get started with running ServerlessLLM on SLURM cluster. It provides two deployment methods, based on `sbatch` and `srun`. If you are in development, we recommend using `srun`, as it is easier to debug than `sbatch`, and if you are in production mode, `sbatch` is recommended. Please make sure you have installed the ServerlessLLM following the [installation guide](./single_machine.md#installation) on all machines. - -## Pre-requisites -Before you begin, make sure you have checked the following: -### Some Tips about Installation -- If 'not enough disk space' is reported when `pip install` on the login node, you can submit it to a job node for execution - ```shell - #!/bin/bash - #SBATCH --partition=Teach-Standard - #SBATCH --job-name=ray-head - #SBATCH --output=sllm_pip.out - #SBATCH --error=sllm_pip.err - #SBATCH --nodes=1 - #SBATCH --ntasks=1 - #SBATCH --cpus-per-task=4 - #SBATCH --gpus-per-task=0 - - # Identify which conda you are using, here is an example that conda is in /opt/conda - source /opt/conda/bin/activate - - conda create -n sllm python=3.10 -y - conda activate sllm - pip install serverless-llm - pip install serverless-llm-store - - conda deactivate sllm - - conda create -n sllm-worker python=3.10 -y - conda activate sllm-worker - pip install serverless-llm[worker] - pip install serverless-llm-store - ``` - -### Command for Querying GPU Resource Information -Run the following commands in the cluster to check GPU resource information. -```shell -sinfo -O partition,nodelist,gres -``` -**Expected Output** -```shell -PARTITION NODELIST GRES -Partition1 JobNode[01,03] gpu:gtx_1060:8 -Partition2 JobNode[04-17] gpu:a6000:2,gpu:gtx_ -``` - -### Identify an idle node -Use `sinfo -p ` to identify some idle nodes - -**Expected Output** -```shell -$ sinfo -p compute -PARTITION AVAIL NODES STATE TIMELIMIT NODELIST -compute up 10 idle infinite JobNode[01-10] -compute up 5 alloc infinite JobNode[11-15] -compute up 2 down infinite JobNode[16-17] -``` - -### Job Nodes Setup -**`srun` Node Selection** - -Only one JobNode is enough. - -**`sbatch` Node Selection** -Let's start a head on the main job node (`JobNode01`) and add the worker on other job node (`JobNode02`). The head and the worker should be on different job nodes to avoid resource contention. The `sllm-store` should be started on the job node that runs worker (`JobNode02`), for passing the model weights, and the `sllm start` should be started on the main job node (`JobNode01`), finally you can use `sllm` to manage the models on the login node. - - -Note: `JobNode02` requires GPU, but `JobNode01` does not. -- **Head**: JobNode01 -- **Worker**: JobNode02 -- **sllm-store**: JobNode02 -- **sllm-serve**: JobNode01 -- **sllm**: Login Node - ---- -## SRUN -If you are in development, we recommend using `srun` to start ServerlessLLM, as it is easier to debug than `sbatch` -### Step 1: Use `srun` enter the JobNode -To start an interactive session on the specified compute node (JobNode), use: -``` -srun --partition --nodelist --gres :1 --pty bash -``` -This command requests a session on the specified node and provides an interactive shell. `--gres :1` specifies the GPU device you will use, for example: `--gres gpu:gtx_1060:1` - -### Step 2: Install ServerlessLLM -Firstly, please make sure CUDA driver available on the node. Here are some commands to check it. -```shell -nvidia-smi - -which nvcc -``` -If `nvidia-smi` has listed GPU information, but `which nvcc` has no output. Then use the following commands to load `nvcc`. Here is an example that cuda is located at `/opt/cuda-12.2.0` -```shell -export PATH=/opt/cuda-12.2.0/bin:$PATH -export LD_LIBRARY_PATH=/opt/cuda-12.2.0/lib64:$LD_LIBRARY_PATH -``` -Then, following the [installation guide](./single_machine.md#installation) to install ServerlessLLM. -### Step 3: Prepare multiple windows with `tmux` -Since srun provides a single interactive shell, you can use tmux to create multiple windows. Start a tmux session: -```shell -tmux -``` -This creates a new tmux session - -**Create multiple windows** -- Use `Ctrl+B` → `C` to start a new window -- Repeat the shortcut 4 more times to create a total of 5 windows. - -**What if `Ctrl+B` does not work?** - -If `Ctrl + B` is unresponsive, reset tmux key bindings: -```shell -tmux unbind C-b -tmux set-option -g prefix C-b -tmux bind C-b send-prefix -``` - -**Command to switch windows** - -Once multiple windows are created, you can switch between them using: - -`Ctrl + B` → `N` (Next window) -`Ctrl + B` → `P` (Previous window) -`Ctrl + B` → `W` (List all windows and select) -`Ctrl + B` → [Number] (Switch to a specific window, e.g., Ctrl + B → 1) - -### Step 4: Run ServerlessLLM on the JobNode -First find ports that are already occupied. Then pick your favourite number from the remaining ports to replace the following placeholder ``. For example: `6379` - -It should also be said that certain slurm system is a bit slow, **so please be patient and wait for the system to output**. - -In the first window, start a local ray cluster with 1 head node and 1 worker node: -```shell -source /opt/conda/bin/activate -conda activate sllm -ray start --head --port= --num-cpus=4 --num-gpus=0 --resources='{"control_node": 1}' --block -``` -In the second window, start the worker node: -```shell -source /opt/conda/bin/activate -conda activate sllm-worker -export CUDA_VISIBLE_DEVICES=0 -ray start --address=0.0.0.0: --num-cpus=4 --num-gpus=1 --resources='{"worker_node": 1, "worker_id_0": 1}' --block -``` -In the third window, start ServerlessLLM Store server: -```shell -source /opt/conda/bin/activate -conda activate sllm-worker -export CUDA_VISIBLE_DEVICES=0 -sllm-store start -``` -In the 4th window, start ServerlessLLM Serve: -```shell -source /opt/conda/bin/activate -conda activate sllm -sllm-serve start -``` -Everything is set! - - -In the 5th window, let's deploy a model to the ServerlessLLM server. You can deploy a model by running the following command: -```shell -source /opt/conda/bin/activate -conda activate sllm -sllm deploy --model facebook/opt-1.3b --backend transformers -``` -This will download the model from HuggingFace transformers. After deploying, you can query the model by any OpenAI API client. For example, you can use the following Python code to query the model: -```shell -curl http://127.0.0.1:8343/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-1.3b", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ] - }' -``` -Expected output: -```shell -{"id":"chatcmpl-9f812a40-6b96-4ef9-8584-0b8149892cb9","object":"chat.completion","created":1720021153,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} -``` - -### Step 5: Clean up -To delete a deployed model, use the following command: -```shell -sllm delete facebook/opt-1.3b -``` -This will remove the specified model from the ServerlessLLM server. - -In each window, use `Ctrl + c` to stop server and `exit` to exit current `tmux` session. - ---- -## SBATCH -### Step 1: Start the Head Node -Since the head node does not require a gpu, you can find a low-computing capacity node to deploy the head node. -1. **Activate the `sllm` environment and start the head node:** - - Here is the example script, named `start_head_node.sh`. - ```shell - #!/bin/bash - #SBATCH --partition=your-partition # Specify the partition - #SBATCH --nodelist=JobNode01 # Specify an idle node - #SBATCH --job-name=ray-head - #SBATCH --output=sllm_head.out - #SBATCH --error=sllm_head.err - #SBATCH --nodes=1 - #SBATCH --ntasks=1 - #SBATCH --cpus-per-task=12 - #SBATCH --gpus-per-task=0 - - cd /path/to/ServerlessLLM - - source /opt/conda/bin/activate # make sure conda will be loaded correctly - conda activate sllm - - ray start --head --port=6379 --num-cpus=12 --num-gpus=0 --resources='{"control_node": 1}' --block - ``` - - Replace `your-partition`, `JobNode01` and `/path/to/ServerlessLLM` - -2. **Submit the script** - - Use ```sbatch start_head_node.sh``` to submit the script to certain idle node. - -3. **Expected output** - - In `sllm_head.out`, you will see the following output: - - ```shell - Local node IP: - -------------------- - Ray runtime started. - -------------------- - ``` - **Remember the IP address**, denoted ``````, you will need it in following steps. - -4. **Find an available port for serve** - - Some HPCs have a firewall that blocks port 8343. You can use `nc -zv 8343` to check if the port is accessible. - - If it is not accessible, find an available port and replace `available_port` in the following script. - - Here is an example script, named `find_port.sh` - - ```shell - #!/bin/bash - #SBATCH --partition=your-partition - #SBATCH --nodelist=JobNode01 - #SBATCH --job-name=find_port - #SBATCH --output=find_port.log - #SBATCH --time=00:05:00 - #SBATCH --mem=1G - - echo "Finding available port on $(hostname)" - - python -c " - import socket - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(('', 0)) - print(f'Available port: {s.getsockname()[1]}') - " - ``` - Use `sbatch find_port.sh` to submit the script to JobNode01, and in `find_port.log`, you will see the following output: - ``` - Finding available port on JobNode01 - Available port: - ``` - Remember this ``, you will use it in Step 4 - -### Step 2: Start the Worker Node & Store -We will start the worker node and store in the same script. Because the server loads the model weights onto the GPU and uses shared GPU memory to pass the pointer to the client. If you submit another script with ```#SBATCH --gpres=gpu:1```, it will be possibly set to use a different GPU, as specified by different ```CUDA_VISIBLE_DEVICES``` settings. Thus, they cannot pass the model weights. -1. **Activate the ```sllm-worker``` environment and start the worker node.** - - Here is the example script, named```start_worker_node.sh```. - ```shell - #!/bin/sh - #SBATCH --partition=your_partition - #SBATCH --nodelist=JobNode02 - #SBATCH --gres=gpu:a6000:1 # Specify device on JobNode02 - #SBATCH --job-name=sllm-worker-store - #SBATCH --output=sllm_worker.out - #SBATCH --error=sllm_worker.err - #SBATCH --gres=gpu:1 # Request 1 GPU - #SBATCH --cpus-per-task=4 # Request 4 CPU cores - #SBATCH --mem=16G # Request 16GB of RAM - - cd /path/to/ServerlessLLM - - conda activate sllm-worker - - HEAD_NODE_IP= - - export CUDA_HOME=/opt/cuda-12.5.0 # replace with your CUDA path - export PATH=$CUDA_HOME/bin:$PATH - export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH - - ray start --address=$HEAD_NODE_IP:6379 --num-cpus=4 --num-gpus=1 \ - --resources='{"worker_node": 1, "worker_id_0": 1}' --block & - - sllm-store start & - - wait - ``` - - Read the HPC's documentation to find out which partition you can use. Replace ```your_partition``` in the script with that partition name. - - Replace ```/path/to/ServerlessLLM``` with the path to the ServerlessLLM installation directory. - - Replace `````` with the IP address of the head node. - - Replace ```/opt/cuda-12.5.0``` with the path to your CUDA path. - -2. **Find the CUDA path** - - Some slurm-based HPCs have a module system, you can use ```module avail cuda``` to find the CUDA module. - - If it does not work, read the HPC's documentation carefully to find the CUDA path. For example, my doc said CUDA is in ```\opt```. Then you can use ```srun``` command to start an interactive session on the node, such as ```srun --pty -t 00:30:00 -p your_partition --gres=gpu:1 /bin/bash```. A pseudo-terminal will be started for you to find the path. - - Find it and replace ```/opt/cuda-12.5.0``` with the path to your CUDA path. -3. **Submit the script on the other node** - - Use ```sbatch start_worker_node.sh``` to submit the script to certain idle node (here we assume it is ```JobNode02```). In addition, We recommend that you place the head and worker on different nodes so that the Serve can start smoothly later, rather than queuing up for resource allocation. -4. **Expected output** - - In ```sllm_worker.out```, you will see the following output: - - - The worker node expected output: - ```shell - Local node IP: xxx.xxx.xx.xx - -------------------- - Ray runtime started. - -------------------- - ``` - - The store expected output: - ```shell - I20241030 11:52:54.719007 1321560 checkpoint_store.cpp:41] Number of GPUs: 1 - I20241030 11:52:54.773468 1321560 checkpoint_store.cpp:43] I/O threads: 4, chunk size: 32MB - I20241030 11:52:54.773548 1321560 checkpoint_store.cpp:45] Storage path: "./models/" - I20241030 11:52:55.060559 1321560 checkpoint_store.cpp:71] GPU 0 UUID: 52b01995-4fa9-c8c3-a2f2-a1fda7e46cb2 - I20241030 11:52:55.060798 1321560 pinned_memory_pool.cpp:29] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes - I20241030 11:52:57.258795 1321560 checkpoint_store.cpp:83] Memory pool created with 4GB - I20241030 11:52:57.262835 1321560 server.cpp:306] Server listening on 0.0.0.0:8073 - ``` -### Step 3: Start the Serve on the Head Node -1. **Activate the ```sllm``` environment and start the serve.** - - Here is the example script, named```start_serve.sh```. - ```shell - #!/bin/sh - #SBATCH --partition=your_partition - #SBATCH --nodelist=JobNode01 # This node should be the same as head - #SBATCH --output=serve.log - - cd /path/to/ServerlessLLM - - conda activate sllm - - sllm start --host - # sllm start --host --port # if you have changed the port - ``` - - Replace `your_partition` in the script as before. - - Replace `/path/to/ServerlessLLM` as before. - - Replace `` you have found in Step 1 (if port 8343 is not available). -2. **Submit the script on the head node** - - Use ```sbatch start_serve.sh``` to submit the script to the head node (```JobNode01```). - -3. **Expected output** - ```shell - -- Connecting to existing Ray cluster at address: xxx.xxx.xx.xx:6379... - -- Connected to Ray cluster. - INFO: Started server process [1339357] - INFO: Waiting for application startup. - INFO: Application startup complete. - INFO: Uvicorn running on http://xxx.xxx.xx.xx:8343 (Press CTRL+C to quit) - ``` -### Step 4: Use sllm to manage models -1. **You can do this step on login node, and set the ```LLM_SERVER_URL``` environment variable:** - ```shell - $ conda activate sllm - (sllm)$ export LLM_SERVER_URL=http://:8343 - ``` - - Replace `` with the actual IP address of the head node. - - Replace ```8343``` with the actual port number (`` in Step1) if you have changed it. -2. **Deploy a Model Using ```sllm```** - ```shell - (sllm)$ sllm deploy --model facebook/opt-1.3b - ``` -### Step 5: Query the Model Using OpenAI API Client - **You can use the following command to query the model:** - ```shell - curl $LLM_SERVER_URL/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "facebook/opt-1.3b", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ] - }' - ``` - - Replace `````` with the actual IP address of the head node. - - Replace ```8343``` with the actual port number (`` in Step 1) if you have changed it. -### Step 6: Stop Jobs -On the SLURM cluster, we usually use the ```scancel``` command to stop the job. Firstly, list all jobs you have submitted (replace ```your_username``` with your username): -```shell -$ squeue -u your_username -JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) - 1234 compute sllm-head your_username R 0:01 1 JobNode01 - 1235 compute sllm-worker-store your_username R 0:01 1 JobNode02 - 1236 compute sllm-serve your_username R 0:01 1 JobNode01 -``` -Then, use ```scancel``` to stop the job (```1234```, ```1235``` and ```1236``` are JOBIDs): -```shell -$ scancel 1234 1235 1236 -``` diff --git a/docs/stable/developer/_category_.json b/docs/stable/developer/_category_.json deleted file mode 100644 index 89a7abc..0000000 --- a/docs/stable/developer/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "Developer Guide", - "position": 6 -} diff --git a/docs/stable/developer/supporting_a_new_hardware.md b/docs/stable/developer/supporting_a_new_hardware.md deleted file mode 100644 index 312cb6c..0000000 --- a/docs/stable/developer/supporting_a_new_hardware.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -sidebar_position: 0 ---- - -# Supporting a New Hardware - -ServerlessLLM actively expands support for new hardware configurations to meet diverse deployment needs. - -## Support Standards -Hardware is considered supported by ServerlessLLM if: -1. Any of the inference backends used (e.g., Transformers, vLLM) can run model inference on the hardware. -2. ServerlessLLM Store can successfully load model checkpoints on the hardware. - -## Steps to Support a New Hardware -1. **Check Inference Backend Compatibility**: Refer to the specific inference backend documentation (e.g., for vLLM, Transformers) for hardware support. -2. **ServerlessLLM Store Configuration**: - - If the hardware provides CUDA-compatible APIs (e.g., ROCm), adjust the build script (`CMakeLists.txt`) by adding necessary compiler flags. - - For non-CUDA-compatible APIs, implementing a custom checkpoint loading function might be required. - -## Verifying Hardware Support in ServerlessLLM Store -The hardware support is verified if it successfully completes the [Quick Start Guide](https://serverlessllm.github.io/docs/stable/getting_started/) examples, showcasing checkpoint loading and inference functionality without errors. - -If the hardware is not publicly available (i.e., can't be tested by the ServerlessLLM team), a screenshot or output log of the successful execution of the Quick Start Guide examples is required to verify hardware support. - -If you encounter any issues or have questions, please reach out to the ServerlessLLM team by raising an issue on the [GitHub repository](https://github.com/ServerlessLLM/ServerlessLLM/issues). \ No newline at end of file diff --git a/docs/stable/features/_category_.json b/docs/stable/features/_category_.json deleted file mode 100644 index 56e0bf0..0000000 --- a/docs/stable/features/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "Features", - "position": 2 -} \ No newline at end of file diff --git a/docs/stable/features/live_migration.md b/docs/stable/features/live_migration.md deleted file mode 100644 index 2db4b1d..0000000 --- a/docs/stable/features/live_migration.md +++ /dev/null @@ -1,211 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Live Migration of Inference Instances - -This example illustrates the live migration of inference instances in a ServerlessLLM cluster by constructing a scenario where two models are deployed to the cluster. Model `Qwen2.5-3B` is stored on both nodes, while model `Qwen2.5-1.5B` is only stored on node 0 (e.g., due to being less popular). This example will show a locality-contention scenario where `Qwen2.5-3B` is being served on node 0 but `Qwen2.5-1.5B` is requested to be served on the same node for optimal locality. We will find that: - -- **Without migration**, `Qwen2.5-1.5B` would have to wait for the completion of the ongoing inference instance of `Qwen2.5-3B` on node 0. -- **With live migration**, the ongoing inference instance of `Qwen2.5-3B` is migrated to node 1, and `Qwen2.5-1.5B` is allocated to node 0, thus can be served immediately. - -## Prerequisites - -To run this example, we will use Docker Compose to set up a ServerlessLLM cluster. Before proceeding, please ensure you have read the [Quickstart Guide](../getting_started.md). - -**Requirements:** - -- **Two GPUs** are required to illustrate the live migration of inference instances. -- **At least 20 GB of host memory** (this can be adjusted by using smaller models). -- **ServerlessLLM version 0.6**: Ensure you have `sllm==0.6` and `sllm-store==0.6` installed. - -## Usage - -Start a local Docker-based ray cluster using Docker Compose. - -### Clone the ServerlessLLM Repository - -If you haven't already, clone the ServerlessLLM repository: - -```bash -git clone https://github.com/ServerlessLLM/ServerlessLLM.git -cd ServerlessLLM/examples/live_migration -``` - -### Configure the Model Directory - -Create a directory on your host machine where models will be stored, and set the MODEL_FOLDER environment variable to point to this directory: - -```bash -export MODEL_FOLDER=/path/to/your/models -``` - -Replace `/path/to/your/models` with the actual path where you want to store the models. - -The Docker Compose configuration is already located in the `examples/live_migration` directory. - -## Test ServerlessLLM Without Live Migration - -1. **Start the ServerlessLLM Services Using Docker Compose** - -```bash -docker compose up -d -``` - -This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. - -:::tip -Use the following command to monitor the logs of the head node: - -```bash -docker logs -f sllm_head -``` -::: - -2. **Deploy Models with the Placement Spec Files** - -Activate the ServerlessLLM environment and set the server URL: -```bash -conda activate sllm -export LLM_SERVER_URL=http://127.0.0.1:8343 -``` - -Deploy the models: -```bash -sllm deploy --config config-qwen-1.5b.json -sllm deploy --config config-qwen-3b.json -``` - -3. **Verify the Deployment** - -Start two inference requests in parallel. The first request is for `Qwen2.5-3B`, and the second request, sent shortly after, is for `Qwen2.5-1.5B`. The `sleep` command is used to introduce a short interval between the two requests: - -```bash -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "Qwen/Qwen2.5-3B-Instruct", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Could you share a story of the history of Computer Science?"} - ], - "max_tokens": 1024 - }' & - -sleep 3 - -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "Qwen/Qwen2.5-1.5B-Instruct", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ], - "max_tokens": 64 - }' -``` - -Since `Qwen2.5-3B` is requested first, `Qwen2.5-1.5B` must wait for the ongoing inference instance of `Qwen2.5-3B` to complete on node 0 before it can start processing. - - -4. Clean up. - -```bash -docker compose down -``` - -## Test ServerlessLLM With Live Migration - -1. **Start the ServerlessLLM Services with Live Migration Enabled** - -Use the following command to start the ServerlessLLM services with live migration enabled. This configuration includes the `enable-migration.yml` file: - -```bash -docker compose -f docker-compose.yml -f enable-migration.yml up -d -``` - -This command will start the Ray head node and two worker nodes, enabling the live migration feature. - -2. **Deploy Models with the Placement Spec Files** - -Activate the ServerlessLLM environment and set the server URL: - -```bash -conda activate sllm -export LLM_SERVER_URL=http://127.0.0.1:8343 -``` - -Deploy the models: - -```bash -sllm deploy --config config-qwen-1.5b.json -sllm deploy --config config-qwen-3b.json -``` - -3. **Verify the Deployment** - -Start two inference requests in parallel. The first request is for `Qwen2.5-3B`, and the second request, sent shortly after, is for `Qwen2.5-1.5B`. The `sleep` command is used to introduce a short interval between the two requests: - -```bash -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "Qwen/Qwen2.5-3B-Instruct", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Could you share a story of the history of Computer Science?"} - ], - "max_tokens": 1024 - }' & - -sleep 3 - -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "Qwen/Qwen2.5-1.5B-Instruct", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ], - "max_tokens": 64 - }' -``` - -According to the response, you should observe that `Qwen2.5-1.5B` completes ahead of `Qwen2.5-3B`. This is because the ongoing inference instance of `Qwen2.5-3B` is live-migrated from node 0 to node 1, allowing `Qwen2.5-1.5B` to be served immediately on node 0. - -As shown in the log message, the ongoing inference instance of the model `Qwen/Qwen2.5-3B-Instruct` is live-migrated from node 0 to node 1. And model `Qwen/Qwen2.5-1.5B-Instruct` is allocated to node 0. - -```bash -(MigrationRouter pid=1724) INFO 12-10 22:05:02 migration_router.py:106] Executing migration plan: MigrationPlan(target_node_id='1', source_instance=InstanceStatus(instance_id='Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab', node_id='0', num_gpu=1, concurrency=0, model_name='Qwen/Qwen2.5-3B-Instruct', num_current_tokens=0)) -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:164] Initialized backend for instance Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 for model Qwen/Qwen2.5-3B-Instruct -# Start multi-round live migration -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:178] Migration iteration 0 -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:183] Number of tokens: 353, delta: 353 -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:198] Migration iteration 0 completed -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:178] Migration iteration 1 -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:183] Number of tokens: 14, delta: 14 -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:188] Migration completed: remained 14 tokens -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:201] Migrated instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab to Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 -# Finish multi-round live migration -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:215] Instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab removed -(MigrationRouter pid=1724) DEBUG 12-10 22:05:13 migration_router.py:77] Preempted request: ... -# Resume the instance on target node -(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:83] Resuming request on target instance: Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 -# Qwen/Qwen2.5-1.5B is allocated to node 0 -(StoreManager pid=1459) INFO 12-10 22:05:14 store_manager.py:344] Loading Qwen/Qwen2.5-1.5B-Instruct to node 0 -(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:92] Deallocating model Qwen/Qwen2.5-3B-Instruct instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab -(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:103] Node 0 deallocated 1 GPUs -(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:108] Model Qwen/Qwen2.5-3B-Instruct instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab deallocated -(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 storage_aware_scheduler.py:188] Migrated instance Qwen/Qwen2.5-3B-Instruct to node 1 instance Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 -(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 storage_aware_scheduler.py:195] Allocated node 0 for model Qwen/Qwen2.5-1.5B-Instruct -``` - -4. Clean up. - -```bash -docker compose down -``` - - diff --git a/docs/stable/features/peft_lora_fine_tuning.md b/docs/stable/features/peft_lora_fine_tuning.md deleted file mode 100644 index af07fdb..0000000 --- a/docs/stable/features/peft_lora_fine_tuning.md +++ /dev/null @@ -1,328 +0,0 @@ ---- -sidebar_position: 4 ---- -# PEFT LoRA Fine-tuning - -This feature introduces a dedicated fine-tuning backend (`ft_backend`) for handling LoRA (Low-Rank Adaptation) fine-tuning jobs in ServerlessLLM. This implementation provides isolated fine-tuning instances with specialized resource management and lifecycle control. - -## Prerequisites - -Before using the fine-tuning feature, ensure you have: - -1. **Base Model**: A base model must be saved using the transformers backend -2. **Docker Setup**: ServerlessLLM cluster running via Docker Compose -3. **Storage**: Adequate storage space for fine-tuned adapters - -## Usage - -### Step 1. **Start the ServerlessLLM Services Using Docker Compose** - -```bash -docker compose up -d -``` - -This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. - -:::tip -Use the following command to monitor the logs of the head node: - -```bash -docker logs -f sllm_head -``` -::: - -### Step 2: Submit Fine-tuning Job - -Submit a fine-tuning job using the REST API: - -```bash -curl -X POST $LLM_SERVER_URL/v1/fine-tuning/jobs \ - -H "Content-Type: application/json" \ - -d @examples/fine_tuning/fine_tuning_config.json -``` - -#### Fine-tuning Configuration - -Create a configuration file (`fine_tuning_config.json`) with the following structure: - -```json -{ - "model": "facebook/opt-125m", - "ft_backend": "peft_lora", - "num_gpus": 1, - "num_cpus": 1, - "timeout": 3600, - "backend_config": { - "output_dir": "facebook/adapters/opt-125m_adapter_test", - "dataset_config": { - "dataset_source": "hf_hub", - "hf_dataset_name": "fka/awesome-chatgpt-prompts", - "tokenization_field": "prompt", - "split": "train", - "data_files": "", - "extension_type": "" - }, - "lora_config": { - "r": 4, - "lora_alpha": 32, - "lora_dropout": 0.05, - "bias": "none", - "task_type": "CAUSAL_LM" - }, - "training_config": { - "auto_find_batch_size": true, - "save_strategy": "no", - "num_train_epochs": 2, - "learning_rate": 0.0001, - "use_cpu": false - } - } -} -``` - -#### Configuration Parameters - -**Job Configuration:** -- `model`: Base model name -- `ft_backend`: Fine-tuning backend type (currently supports "peft_lora") -- `num_cpus`: Number of CPU cores required -- `num_gpus`: Number of GPUs required -- `timeout`: Maximum execution time in seconds - -**Dataset Configuration:** -- `dataset_source`: Source type ("hf_hub" or "local") -- `hf_dataset_name`: HuggingFace dataset name (for hf_hub) -- `data_files`: Local file paths (for local) -- `extension_type`: File extension type (for local) -- `tokenization_field`: Field name for tokenization -- `split`: Dataset split to use -- More dataset config parameters could be found in [huggingface datasets documentation](https://huggingface.co/docs/datasets/en/loading#load) - -**LoRA Configuration:** -- `r`: LoRA rank -- `lora_alpha`: LoRA alpha parameter -- `target_modules`: Target modules for LoRA adaptation -- `lora_dropout`: Dropout rate -- `bias`: Bias handling strategy -- `task_type`: Task type for PEFT -- More LoraConfig parameters could be found in [huggingface documentation](https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraConfig) - -**Training Configuration:** -- `num_train_epochs`: Number of training epochs -- `per_device_train_batch_size`: Batch size per device -- `gradient_accumulation_steps`: Gradient accumulation steps -- `learning_rate`: Learning rate -- `warmup_steps`: Number of warmup steps -- `logging_steps`: Logging frequency -- `save_steps`: Model saving frequency -- `eval_steps`: Evaluation frequency -- More training arguments could be found in [huggingface documentation](https://huggingface.co/docs/transformers/v4.53.3/en/main_classes/trainer#transformers.TrainingArguments) - -### Step 3: Expected Response - -Upon successful job submission, you'll receive a response with the job ID: - -```json -{ - "job_id": "job-123" -} -``` - -### Step 4: Monitor Job Status - -Check the status of your fine-tuning job: - -```bash -curl -X GET "$LLM_SERVER_URL/v1/fine_tuning/jobs/job-123" -``` - -#### Status Response - -```json -{ - "id": "job-123", - "object": "fine_tuning.job", - "status": { - "config": { - "model": "facebook/opt-125m", - "ft_backend": "peft_lora", - "num_gpus": 1, - "num_cpus": 1, - "timeout": 3600, - "backend_config": { - "output_dir": "facebook/adapters/opt-125m_adapter_test", - "dataset_config": { - "dataset_source": "hf_hub", - "hf_dataset_name": "fka/awesome-chatgpt-prompts", - "tokenization_field": "prompt", - "split": "train", - "data_files": "", - "extension_type": "" - }, - "lora_config": { - "r": 4, - "lora_alpha": 32, - "lora_dropout": 0.05, - "bias": "none", - "task_type": "CAUSAL_LM" - }, - "training_config": { - "auto_find_batch_size": true, - "save_strategy": "no", - "num_train_epochs": 2, - "learning_rate": 0.0001, - "use_cpu": false - } - } - }, - "status": "running", - "created_time": "2025-08-26T04:18:11.155785", - "updated_time": "2025-08-26T04:18:11.155791", - "priority": 0 - } -} -``` - -**Possible Status Values:** -- `pending`: Job is waiting for resources -- `running`: Job is currently executing -- `completed`: Job completed successfully -- `failed`: Job failed with an error -- `cancelled`: Job was cancelled by user - -### Step 5: Cancel Job (Optional) - -If needed, you can cancel a running job: - -```bash -curl -X POST "$LLM_SERVER_URL/v1/fine_tuning/jobs/job-123/cancel" -``` - -## Job Management - -### Resource Allocation - -Fine-tuning jobs are allocated resources based on the specified requirements: - -- **CPU**: Number of CPU cores specified in `num_cpus` -- **GPU**: Number of GPUs specified in `num_gpus` -- **Memory**: Automatically managed based on model size and batch size - -### Priority System - -Jobs are processed based on priority and creation time: - -1. **Higher Priority**: Jobs with higher priority values are processed first -2. **FIFO**: Jobs with the same priority are processed in order of creation -3. **Resource Availability**: Jobs wait until sufficient resources are available - -### Timeout Handling - -Jobs have configurable timeout limits: - -- **Default Timeout**: 3600 seconds (1 hour) -- **Configurable**: Set via `timeout` parameter in job configuration -- **Automatic Cleanup**: Jobs are automatically marked as failed if they exceed the timeout - -## Output and Storage - -### LoRA Adapter Storage - -Fine-tuned LoRA adapters are automatically saved to the `output_dir` path you config in the `fine_tuning_config.json`, like: - -``` -{STORAGE_PATH}/transformers/facebook/adapters/opt-125m_adapter_test -``` - -### Adapter Contents - -The saved adapter includes: - -- **LoRA Weights**: Fine-tuned LoRA parameters -- **Configuration**: LoRA configuration file -- **Metadata**: Training metadata and statistics - -## Integration with Serving - -### Using Fine-tuned Adapters - -After successful fine-tuning, the LoRA adapter can be used for inference: - -```bash -# Deploy model with fine-tuned adapter -sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "my_adapter=ft_facebook/opt-125m_adapter" - -# Use the adapter for inference -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-125m", - "messages": [ - {"role": "user", "content": "Hello, how are you?"} - ], - "lora_adapter_name": "my_adapter" -}' -``` - -For more details about PEFT LoRA Serving, please see the [documentation](./peft_lora_serving.md) -## Troubleshooting - -### Common Issues - -1. **Job Stuck in Pending**: Check resource availability and job priority -2. **Dataset Loading Failures**: Verify dataset configuration and accessibility -3. **Training Failures**: Check GPU memory and batch size settings -4. **Timeout Errors**: Increase timeout or optimize training configuration - -## API Reference - -### Endpoints - -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/v1/fine-tuning/jobs` | POST | Submit a fine-tuning job | -| `/v1/fine_tuning/jobs/{fine_tuning_job_id}` | GET | Get job status | -| `/v1/fine_tuning/jobs/{fine_tuning_job_id}/cancel` | POST | Cancel a running job | - -### Response Codes - -| Code | Description | -|------|-------------| -| 200 | Success | -| 400 | Bad Request | -| 404 | Job not found | -| 500 | Internal Server Error | - -## Examples - -### Complete Fine-tuning Workflow - -```bash -# 1. Save base model -sllm-store save --model facebook/opt-125m --backend transformers - -# 2. Start the ServerlessLLM cluster with docker compose -cd examples/docker -docker compose up -d --build - -# 3. Submit fine-tuning job -cd .. && cd .. -curl -X POST $LLM_SERVER_URL/v1/fine-tuning/jobs \ - -H "Content-Type: application/json" \ - -d @examples/fine_tuning/fine_tuning_config.json - -# 4. Monitor job status -curl -X GET "$LLM_SERVER_URL/v1/fine_tuning/jobs/job-123" - -# 5. Deploy base model with fine-tuned adapter -sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "my_adapter=ft_facebook/opt-125m_adapter" - -# 5. Use for inference -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-125m", - "messages": [{"role": "user", "content": "Hello"}], - "lora_adapter_name": "my_adapter" -}' -``` diff --git a/docs/stable/features/peft_lora_serving.md b/docs/stable/features/peft_lora_serving.md deleted file mode 100644 index 4c7d3a6..0000000 --- a/docs/stable/features/peft_lora_serving.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -sidebar_position: 2 ---- -# PEFT LoRA Serving - -This example illustrates the process of deploying and serving a base large language model enhanced with LoRA (Low-Rank Adaptation) adapters in a ServerlessLLM cluster. It demonstrates how to start the cluster, deploy a base model with multiple LoRA adapters, perform inference using different adapters, and update or remove the adapters dynamically. - -## Pre-requisites - -To run this example, we will use Docker Compose to set up a ServerlessLLM cluster. Before proceeding, please ensure you have read the [Quickstart Guide](../getting_started.md). - -We will use the following example base model & LoRA adapters -- Base model: `facebook/opt-125m` -- LoRA adapters: - - `peft-internal-testing/opt-125m-dummy-lora` - - `monsterapi/opt125M_alpaca` - - `edbeeching/opt-125m-lora` - - `Hagatiana/opt-125m-lora` - -## Usage - -Start a local Docker-based ray cluster using Docker Compose. - -### Step 1: Download the Docker Compose File - -Download the `docker-compose.yml` file from the ServerlessLLM repository: -```bash -# Create a directory for the ServerlessLLM Docker setup -mkdir serverless-llm-docker && cd serverless-llm-docker - -# Download the docker-compose.yml file -curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml - -# Alternatively, you can use wget: -# wget https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml -``` - -### Step 2: Configuration - -Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: - -```bash -export MODEL_FOLDER=/path/to/your/models -``` - -Replace `/path/to/your/models` with the actual path where you want to store the models. - -### Step 3: Start the Services - -Start the ServerlessLLM services using Docker Compose: - -```bash -docker compose up -d -``` - -This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. - -:::tip -Use the following command to monitor the logs of the head node: - -```bash -docker logs -f sllm_head -``` -::: - -### Step 4: Deploy Models with LoRA Adapters -1. Configuration -```bash -conda activate sllm -export LLM_SERVER_URL=http://127.0.0.1:8343 -``` -2. Deploy models with specified lora adapters. -```bash -sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "demo_lora1=peft-internal-testing/opt-125m-dummy-lora demo_lora2=monsterapi/opt125M_alpaca" -``` -3. Verify the deployment. -```bash -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-125m", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ], - "lora_adapter_name": "demo_lora1" - }' -``` -If no lora adapters specified, the system will use the base model to do inference -```bash -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-125m", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ] - }' -``` -### Step 5: Update LoRA Adapters -If you wish to switch to a different set of LoRA adapters, you can still use `sllm deploy` command with updated adapter configurations. ServerlessLLM will automatically reload the new adapters without restarting the backend. -```bash -sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "demo-lora1=edbeeching/opt-125m-lora demo-lora2=Hagatiana/opt-125m-lora" -``` - -### Step 6: Clean Up - -Delete the lora adapters by running the following command (this command will only delete lora adapters, the base model won't be deleted): -```bash -sllm delete facebook/opt-125m --lora-adapters "demo-lora1 demo-lora2" -``` -If you need to stop and remove the containers, you can use the following commands: -```bash -docker compose down -``` \ No newline at end of file diff --git a/docs/stable/features/quantized_models.md b/docs/stable/features/quantized_models.md deleted file mode 100644 index df09751..0000000 --- a/docs/stable/features/quantized_models.md +++ /dev/null @@ -1,175 +0,0 @@ ---- -sidebar_position: 3 ---- - -# Quantization - -This example demonstrates the use of quantization within the ServerlessLLM framework to optimize model serving. Quantization is a technique used to reduce the memory footprint and computational requirements of a large language model by representing its weights with lower-precision data types, such as 8-bit integers (int8). This example will showcase how to deploy and serve a quantized model in a ServerlessLLM cluster. - -## Pre-requisites - -We will use Docker Compose to run a ServerlessLLM cluster in this example. Therefore, please make sure you have read the Quickstart Guide before proceeding. - -## Usage -Start a local Docker-based ray cluster using Docker Compose. - -## Step 1: Set up the Environment - -Create a directory for this example and download the `docker-compose.yml` file. - -```bash -mkdir sllm-quantization-example && cd sllm-quantization-example -curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml - -## Step 2: Configuration - -Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: - -```bash -export MODEL_FOLDER=/path/to/your/models -``` - -Replace `/path/to/your/models` with the actual path where you want to store the models. - -## Step 3: Start the Services - -Start the ServerlessLLM services using Docker Compose: - -```bash -docker compose up -d -``` - -This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. - -:::tip -Use the following command to monitor the logs of the head node: - -```bash -docker logs -f sllm_head -``` -::: - -## Step 4: Create Quantization and Deployment Configurations - -First, we'll generate a standard Hugging Face BitsAndBytesConfig and save it to a JSON file. Then, we'll create a deployment configuration file with these quantization settings embedded in it. - -1. Generate the Quantization Config - -Create a Python script named `get_config.py` in the current directory with the following content: -```python -# get_config.py -from transformers import BitsAndBytesConfig - -quantization_config = BitsAndBytesConfig(load_in_4bit=True) -quantization_config.to_json_file("quantization_config.json") - -``` - -Run the script to generate `quantization_config.json`: -```bash -python get_config.py -``` - - -2. Create the Deployment Config - -Now, create a file named `quantized_deploy_config.json`. This file tells ServerlessLLM which model to deploy and instructs the backend to use the quantization settings. You should copy the contents of `quantization_config.json` into the `quantization_config` field below. A template can be found in `sllm/cli/default_config.json`. - -```json -{ - "model": "facebook/opt-1.3b", - "backend": "transformers", - "num_gpus": 1, - "auto_scaling_config": { - "metric": "concurrency", - "target": 1, - "min_instances": 0, - "max_instances": 10, - "keep_alive": 0 - }, - "backend_config": { - "pretrained_model_name_or_path": "", - "device_map": "auto", - "torch_dtype": "float16", - "hf_model_class": "AutoModelForCausalLM", - "quantization_config": { - "_load_in_4bit": true, - "_load_in_8bit": false, - "bnb_4bit_compute_dtype": "float32", - "bnb_4bit_quant_storage": "uint8", - "bnb_4bit_quant_type": "fp4", - "bnb_4bit_use_double_quant": false, - "llm_int8_enable_fp32_cpu_offload": false, - "llm_int8_has_fp16_weight": false, - "llm_int8_skip_modules": null, - "llm_int8_threshold": 6.0, - "load_in_4bit": true, - "load_in_8bit": false, - "quant_method": "bitsandbytes" - } - } -} - -``` - -> Note: Quantization currently only supports the "transformers" backend. Support for other backends will come soon. - -## Step 5: Deploy the Quantized Model -With the configuration files in place, deploy the model using the `sllm-cli`. - -```bash -conda activate sllm -export LLM_SERVER_URL=http://127.0.0.1:8343 - -sllm-cli deploy --config quantized_deploy_config.json -``` - -## Step 6: Verify the deployment. -Send an inference to the server to query the model: - -```bash -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-1.3b", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ] - }' -``` - -To verify the model is being loaded in the desired precision, check the logs (`docker logs sllm_head`). You should see that the model is indeed being loaded in `fp4`. - - -```log -(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:321] load config takes 0.0030286312103271484 seconds -(RoundRobinRouter pid=481) INFO 07-02 20:01:49 roundrobin_router.py:272] [] -(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:331] load model takes 0.2806234359741211 seconds -(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:338] device_map: OrderedDict([('', 0)]) -(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:345] compute_device_placement takes 0.18753838539123535 seconds -(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:376] allocate_cuda_memory takes 0.0020012855529785156 seconds -(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 client.py:72] load_into_gpu: transformers/facebook/opt-1.3b, 70b42a05-4faa-4eaf-bb73-512c6453e7fa -(TransformersBackend pid=352, ip=172.18.0.2) INFO 07-02 20:01:49 client.py:113] Model loaded: transformers/facebook/opt-1.3b, 70b42a05-4faa-4eaf-bb73-512c6453e7fa -(TransformersBackend pid=352, ip=172.18.0.2) INFO 07-02 20:01:49 transformers.py:398] restore state_dict takes 0.0007319450378417969 seconds -(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:411] using precision: fp4 -(TransformersBackend pid=352, ip=172.18.0.2) INFO 07-02 20:01:50 client.py:117] confirm_model_loaded: transformers/facebook/opt-1.3b, 70b42a05-4faa-4eaf-bb73-512c6453e7fa -``` - -You should receive a successful JSON response from the model. - -## Step 7: Clean Up - -Delete the model deployment by running the following command: - -```bash -sllm-cli delete facebook/opt-1.3b -``` - -If you need to stop and remove the containers, you can use the following commands: - -```bash -docker compose down -``` - - diff --git a/docs/stable/features/storage_aware_scheduling.md b/docs/stable/features/storage_aware_scheduling.md deleted file mode 100644 index 81723f9..0000000 --- a/docs/stable/features/storage_aware_scheduling.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -sidebar_position: 0 ---- - -# Storage Aware Scheduling with Docker Compose - -## Pre-requisites - -We will use Docker Compose to run a ServerlessLLM cluster in this example. Therefore, please make sure you have read the [Quickstart Guide](../getting_started.md) before proceeding. - -## Usage - -Start a local Docker-based ray cluster using Docker Compose. - -### Step 1: Clone the ServerlessLLM Repository - -If you haven't already, clone the ServerlessLLM repository: - -```bash -git clone https://github.com/ServerlessLLM/ServerlessLLM.git -cd ServerlessLLM/examples/storage_aware_scheduling -``` - -### Step 2: Configuration - -Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: - -```bash -export MODEL_FOLDER=/path/to/your/models -``` - -Replace `/path/to/your/models` with the actual path where you want to store the models. - -### Step 3: Enable Storage Aware Scheduling in Docker Compose - -The Docker Compose configuration is already located in the `examples/storage_aware_scheduling` directory. To activate storage-aware scheduling, ensure the `docker-compose.yml` file includes the necessary configurations(`sllm_head` service should include the `--enable-storage-aware` command). - -:::tip -Recommend to adjust the number of GPUs and `mem_pool_size` based on the resources available on your machine. -::: - - -### Step 4: Start the Services - -Start the ServerlessLLM services using Docker Compose: - -```bash -docker compose up -d -``` - -This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. - -:::tip -Use the following command to monitor the logs of the head node: - -```bash -docker logs -f sllm_head -``` -::: - -### Step 5: Deploy Models with Placement Spec - -In the `examples/storage_aware_scheduling` directory, the example configuration files (`config-opt-2.7b.json` and `config-opt-1.3b.json`) are already given. - -> Note: Storage aware scheduling currently only supports the "transformers" backend. Support for other backends will come soon. - -2. Deploy models with the placement spec files. - -```bash -conda activate sllm -export LLM_SERVER_URL=http://127.0.0.1:8343 - -sllm deploy --config config-opt-2.7b.json -sllm deploy --config config-opt-1.3b.json -``` - -3. Verify the deployment. - -```bash -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-2.7b", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ] - }' - -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-1.3b", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ] - }' -``` - -As shown in the log message, the model "facebook/opt-2.7b" is scheduled on server 0, while the model "facebook/opt-1.3b" is scheduled on server 1. - -```log -(StorageAwareScheduler pid=1543) INFO 11-12 23:48:27 storage_aware_scheduler.py:137] Sorted scheduling options: [('0', 4.583079601378258)] -(StorageAwareScheduler pid=1543) INFO 11-12 23:48:27 storage_aware_scheduler.py:144] Allocated node 0 for model facebook/opt-2.7b -(StorageAwareScheduler pid=1543) INFO 11-12 23:48:38 storage_aware_scheduler.py:137] Sorted scheduling options: [('1', 2.266678696047572)] -(StorageAwareScheduler pid=1543) INFO 11-12 23:48:38 storage_aware_scheduler.py:144] Allocated node 1 for model facebook/opt-1.3b -``` - -### Step 6: Clean Up - -Delete the model deployment by running the following command: - -```bash -sllm delete facebook/opt-1.3b facebook/opt-2.7b -``` - -If you need to stop and remove the containers, you can use the following commands: - -```bash -docker compose down -``` - diff --git a/docs/stable/getting_started.md b/docs/stable/getting_started.md deleted file mode 100644 index f98364d..0000000 --- a/docs/stable/getting_started.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Getting Started - -This guide demonstrates how to quickly set up a local ServerlessLLM cluster using Docker Compose on a single machine. We will initialize a minimal cluster, consisting of a head node and a single worker node. Then, we'll deploy a model using the `sllm` and query the deployment through an OpenAI-compatible API. - -:::note -We strongly recommend using Docker (Compose) to manage your ServerlessLLM cluster, whether you are using ServerlessLLM for testing or development. However, if Docker is not a viable option for you, please refer to the [deploy from scratch guide](./deployment/single_machine.md). -::: - -## Prerequisites - -Before you begin, ensure you have the following installed and configured: - -1. **Docker**: Installed on your system. You can download it from [here](https://docs.docker.com/get-docker/). -2. **ServerlessLLM CLI**: Installed on your system. Install it using `pip install serverless-llm`. -3. **GPUs**: At least one NVIDIA GPU is required. If you have multiple GPUs, you can adjust the `docker-compose.yml` file accordingly. -4. **NVIDIA Docker Toolkit**: This enables Docker to utilize NVIDIA GPUs. Follow the installation guide [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). - -## Start the ServerlessLLM Cluster - -We will use Docker Compose to simplify the ServerlessLLM setup process. - -### Step 1: Download the Docker Compose File - -Download the `docker-compose.yml` file from the ServerlessLLM repository: - -```bash -# Create a directory for the ServerlessLLM Docker setup -mkdir serverless-llm-docker && cd serverless-llm-docker - -# Download the docker-compose.yml file -curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml - -# Alternatively, you can use wget: -# wget https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml -``` - -### Step 2: Configuration - -Create a directory on your host machine to store models. Then, set the `MODEL_FOLDER` environment variable to point to this directory: - -```bash -export MODEL_FOLDER=/path/to/your/models -``` - -Replace `/path/to/your/models` with the actual path where you intend to store the models. This directory will be mounted into the Docker containers. - -### Step 3: Start the Services - -Start the ServerlessLLM services using Docker Compose: - -```bash -docker compose up -d -``` - -This command will start the Ray head node and a worker node as defined in the `docker-compose.yml` file. - -Verify that the services are ready: - -```bash -docker logs sllm_head -``` - -Ensure the services are ready before proceeding. You should see output similar to the following: - -```bash -... -(SllmController pid=1435) INFO 05-26 15:40:49 controller.py:68] Starting scheduler -INFO: Started server process [1] -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://0.0.0.0:8343 (Press CTRL+C to quit) -(FcfsScheduler pid=1604) INFO 05-26 15:40:49 fcfs_scheduler.py:54] Starting FCFS scheduler -(FcfsScheduler pid=1604) INFO 05-26 15:40:49 fcfs_scheduler.py:111] Starting control loop -``` - -## Deploy a Model Using sllm - -Set the `LLM_SERVER_URL` environment variable: - -```bash -export LLM_SERVER_URL=http://127.0.0.1:8343 -``` - -Deploy a model to the ServerlessLLM cluster using the `sllm`: - -```bash -sllm deploy --model facebook/opt-1.3b -``` -> Note: This command will take some time to download the model from the Hugging Face Model Hub. -> You can use any model from the [Hugging Face Model Hub](https://huggingface.co/models) by specifying its name in the `--model` argument. - -Expected output: - -```plaintext -INFO 08-01 07:38:12 deploy.py:36] Deploying model facebook/opt-1.3b with default configuration. -INFO 08-01 07:39:00 deploy.py:49] Model registered successfully. -``` - -## Query the Model - -You can now query the model using any OpenAI API client. For example, use the following `curl` command: -```bash -curl $LLM_SERVER_URL/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "model": "facebook/opt-1.3b", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is your name?"} - ] - }' -``` - -Expected output: - -```plaintext -{"id":"chatcmpl-8b4773e9-a98b-41db-8163-018ed3dc65e2","object":"chat.completion","created":1720183759,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}}% -``` - -## Clean Up -To delete a deployed model, execute the following command: - -```bash -sllm delete facebook/opt-1.3b -``` - -This command removes the specified model from the ServerlessLLM server. - -To stop the ServerlessLLM services, use the following command: -```bash -docker compose down -``` \ No newline at end of file diff --git a/docs/stable/intro.md b/docs/stable/intro.md deleted file mode 100644 index 8786a40..0000000 --- a/docs/stable/intro.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -sidebar_position: 0 ---- - -# Serverless LLM - - -ServerlessLLM - -ServerlessLLM is a **fast** and **easy-to-use** serving system designed for **affordable** multi-LLM serving, also known as LLM-as-a-Service. ServerlessLLM is ideal for environments with multiple LLMs that need to be served on limited GPU resources, as it enables efficient dynamic loading of LLMs onto GPUs. By elastically scaling model instances and multiplexing GPUs, ServerlessLLM can significantly reduce costs compared to traditional GPU-dedicated serving systems while still providing low-latency (Time-to-First-Token, TTFT) LLM completions. - -ServerlessLLM now supports NVIDIA and AMD GPUs, including following hardware: -* NVIDIA GPUs: Compute Capability 7.0+ (e.g, V100, A100, RTX A6000, GeForce RTX 3060) -* AMD GPUs: ROCm 6.2.0+ (tested on MI100s and MI200s) - -## Documentation - -### Getting Started - -- [Quickstart](./getting_started.md) -- [Single Machine Deployment (From Scratch)](./deployment/single_machine.md) -- [Multi-machine Deployment](./deployment/multi_machine.md) -- [SLURM Cluster Deployment](./deployment/slurm_cluster.md) - -### Advanced Features - -- [Storage-Aware Scheduler](./features/storage_aware_scheduling.md) -- [Live Migration](./features/live_migration.md) -- [PEFT LoRA Serving](./features/peft_lora_serving.md) - -### ServerlessLLM Store - -- [Quickstart](./store/quickstart.md) -- [ROCm Quickstart](./store/rocm_quickstart.md) - -### ServerlessLLM CLI - -- [ServerlessLLM CLI API](../api/cli.md) diff --git a/docs/stable/models/_category_.json b/docs/stable/models/_category_.json deleted file mode 100644 index 67c0cfe..0000000 --- a/docs/stable/models/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "Models", - "position": 7 -} diff --git a/docs/stable/models/supported_models.md b/docs/stable/models/supported_models.md deleted file mode 100644 index 6077615..0000000 --- a/docs/stable/models/supported_models.md +++ /dev/null @@ -1,13 +0,0 @@ -# Supported Models - -ServerlessLLM supports a plethora of language models from [Huggingface (HF) Transformers](https://huggingface.co/models). This page lists the models and model architectures currently supported by ServerlessLLM. - -To test a model, simply add it to the `supported_models.json` inside `/ServerlessLLM/tests/inference_tests` and the Github Actions will automatically test whether not it is supported. - -## Text-only Language Models - -Architecture |Models |Example HF Models |vLLM |Transformers -------------------|--------------|--------------------|-----|------------- -`OPTForCausalLM` |OPT, OPT-IML |`facebook/opt-6.7b` |✅ |✅ - - diff --git a/docs/stable/store/_category_.json b/docs/stable/store/_category_.json deleted file mode 100644 index 78b547f..0000000 --- a/docs/stable/store/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "ServerlessLLM Store", - "position": 5 -} diff --git a/docs/stable/store/quantization.md b/docs/stable/store/quantization.md deleted file mode 100644 index 3450410..0000000 --- a/docs/stable/store/quantization.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -sidebar_position: 2 ---- - -# Quantization - -> Note: Quantization is currently experimental, especially on multi-GPU machines. You may encounter issues when using this feature in multi-GPU environments. - -ServerlessLLM currently supports `bitsandbytes` quantization, which reduces model memory usage by converting weights to lower-precision data types. You can configure this by passing a `BitsAndBytesConfig` object when loading a model. - -Available precisions include: -- `int8` -- `fp4` -- `nf4` - -> Note: CPU offloading and dequantization is not currently supported. - -## 8-bit Quantization (`int8`) - -8-bit quantization halves the memory usage compared to 16-bit precision with minimal impact on model accuracy. It is a robust and recommended starting point for quantization. - -```python -from transformers import AutoModelForCausalLM, BitsAndBytesConfig - -# Configure 8-bit quantization -quantization_config = BitsAndBytesConfig( - load_in_8bit=True -) - -# Load the model with the config -model_8bit = AutoModelForCausalLM.from_pretrained( - "facebook/opt-1.3b", - quantization_config=quantization_config, - device_map="auto", -) -``` - -## 4-bit Quantization (`fp4`) -FP4 (4-bit Floating Point) quantization offers more aggressive memory savings than 8-bit. It is a good option for running very large models on consumer-grade hardware. - -```python -from transformers import AutoModelForCausalLM, BitsAndBytesConfig - -# Configure 4-bit FP4 quantization -quantization_config = BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_quant_type="fp4" -) - -# Load the model with the config -model_fp4 = AutoModelForCausalLM.from_pretrained( - "facebook/opt-1.3b", - quantization_config=quantization_config, - device_map="auto", -) -``` - -## 4-bit Quantization (`nf4`) -NF4 (4-bit NormalFloat) is an advanced data type optimized for models whose weights follow a normal distribution. NF4 is generally the recommended 4-bit option as it often yields better model accuracy compared to FP4. - -```python -import torch -from transformers import AutoModelForCausalLM, BitsAndBytesConfig - -# Configure 4-bit NF4 quantization -quantization_config = BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_quant_type="nf4" -) - -# Load the model with the config -model_nf4 = AutoModelForCausalLM.from_pretrained( - "facebook/opt-1.3b", - quantization_config=quantization_config, - device_map="auto", -) -``` - -## `torch_dtype` (Data Type for Unquantized Layers) -The `torch_dtype` parameter sets the data type for model layers that are not quantized (e.g. `LayerNorm`). Setting this to `torch.float16` or `torch.bfloat16` can further reduce memory usage. If unspecified, these layers default to `torch.float16`. - -```python -import torch -from transformers import AutoModelForCausalLM, BitsAndBytesConfig - -# Configure 4-bit NF4 quantization -quantization_config = BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_quant_type="nf4" -) - -# Load model, casting non-quantized layers to float16 -model_mixed_precision = AutoModelForCausalLM.from_pretrained( - "facebook/opt-1.3b", - quantization_config=quantization_config, - torch_dtype=torch.float16, - device_map="auto", -) -``` - -For further information, consult the [HuggingFace Documentation for BitsAndBytes](https://huggingface.co/docs/transformers/main/en/quantization/bitsandbytes). - diff --git a/docs/stable/store/quickstart.md b/docs/stable/store/quickstart.md deleted file mode 100644 index 83492ea..0000000 --- a/docs/stable/store/quickstart.md +++ /dev/null @@ -1,245 +0,0 @@ ---- -sidebar_position: 0 ---- - -# Quickstart Guide - -ServerlessLLM Store (`sllm-store`) is a Python library that supports fast model checkpoint loading from multi-tier storage (i.e., DRAM, SSD, HDD) into GPUs. - -ServerlessLLM Store provides a model manager and two key functions: -- `save_model`: Convert a HuggingFace model into a loading-optimized format and save it to a local path. -- `load_model`: Load a model into given GPUs. - - -## Requirements -- OS: Ubuntu 20.04 -- Python: 3.10 -- GPU: compute capability 7.0 or higher - -## Installations - -### Create a virtual environment -```bash -conda create -n sllm-store python=3.10 -y -conda activate sllm-store -``` - -### Install with pip -```bash -pip install serverless-llm-store -``` - -### Install from source -1. Clone the repository and enter the `store` directory - -``` bash -git clone https://github.com/ServerlessLLM/ServerlessLLM.git -cd ServerlessLLM/sllm_store -``` - -2. Install the package from source - -```bash -rm -rf build -pip install . -``` - -## Usage Examples -:::tip -We highly recommend using a fast storage device (e.g., NVMe SSD) to store the model files for the best experience. -For example, create a directory `models` on the NVMe SSD and link it to the local path. -```bash -mkdir -p /mnt/nvme/models # Replace '/mnt/nvme' with your NVMe SSD path. -ln -s /mnt/nvme/models ./models -``` -::: - -1. Convert a model to ServerlessLLM format and save it to a local path: -```python -from sllm_store.transformers import save_model - -# Load a model from HuggingFace model hub. -import torch -from transformers import AutoModelForCausalLM -model = AutoModelForCausalLM.from_pretrained('facebook/opt-1.3b', torch_dtype=torch.float16) - -# Replace './models' with your local path. -save_model(model, './models/facebook/opt-1.3b') -``` - -2. Launch the checkpoint store server in a separate process: -```bash -# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. -sllm-store start --storage-path $PWD/models --mem-pool-size 4GB -``` - - - -3. Load model in your project and make inference: -```python -import time -import torch -from sllm_store.transformers import load_model - -# warm up the GPU -num_gpus = torch.cuda.device_count() -for i in range(num_gpus): - torch.ones(1).to(f"cuda:{i}") - torch.cuda.synchronize() - -start = time.time() -model = load_model("facebook/opt-1.3b", device_map="auto", torch_dtype=torch.float16, storage_path="./models/", fully_parallel=True) -# Please note the loading time depends on the model size and the hardware bandwidth. -print(f"Model loading time: {time.time() - start:.2f}s") - -from transformers import AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained('facebook/opt-1.3b') -inputs = tokenizer('Hello, my dog is cute', return_tensors='pt').to("cuda") -outputs = model.generate(**inputs) -print(tokenizer.decode(outputs[0], skip_special_tokens=True)) -``` - -4. Clean up by "Ctrl+C" the server process. - -## Usage with vLLM - -ServerlessLLM integrates with vLLM to provide fast model loading capabilities. Follow these steps to set up and use ServerlessLLM with vLLM. - -### Prerequisites - -Before using ServerlessLLM with vLLM, you need to apply a compatibility patch to your vLLM installation. This patch has been tested with vLLM version `0.9.0.1`. - -### Apply the vLLM Patch - -1. **Check patch status** (optional): - ```bash - ./sllm_store/vllm_patch/check_patch.sh - ``` - -2. **Apply the patch**: - ```bash - ./sllm_store/vllm_patch/patch.sh - ``` - -3. **Remove the patch** (if needed): - ```bash - ./sllm_store/vllm_patch/remove_patch.sh - ``` - -:::note -The patch file is located at `sllm_store/vllm_patch/sllm_load.patch` in the ServerlessLLM repository. -::: - - -Our api aims to be compatible with the `sharded_state` load format in vLLM. Thus, due to the model modifications about the model architecture done by vLLM, the model format for vLLM is **not** the same as we used in transformers. Thus, the `ServerlessLLM format` mentioned in the subsequent sections means the format integrated with vLLM, which is different from the `ServerlessLLM format` used in the previous sections. - -Thus, for fist-time users, you have to load the model from other backends and then converted it to the ServerlessLLM format. - -1. Download the model from HuggingFace and save it in the ServerlessLLM format: -``` bash -python3 examples/sllm_store/save_vllm_model.py --model-name facebook/opt-1.3b --storage-path $PWD/models --tensor-parallel-size 1 - -``` - -You can also transfer the model from the local path compared to download it from network by passing the `--local-model-path` argument. - -After downloading the model, you can launch the checkpoint store server and load the model in vLLM through `sllm` load format. - -2. Launch the checkpoint store server in a separate process: -```bash -# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. -sllm-store start --storage-path $PWD/models --mem-pool-size 4GB -``` - -3. Load the model in vLLM: -```python -from vllm import LLM, SamplingParams - -import os - -storage_path = os.getenv("STORAGE_PATH", "./models") -model_name = "facebook/opt-1.3b" -model_path = os.path.join(storage_path, model_name) - -llm = LLM( - model=model_path, - load_format="serverless_llm", - dtype="float16" -) - -prompts = [ - "Hello, my name is", - "The president of the United States is", - "The capital of France is", - "The future of AI is", -] - -sampling_params = SamplingParams(temperature=0.8, top_p=0.95) -outputs = llm.generate(prompts, sampling_params) - -# Print the outputs. -for output in outputs: - prompt = output.prompt - generated_text = output.outputs[0].text - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") -``` - -# Fine-tuning -ServerlessLLM currently supports LoRA fine-tuning using peft through the Hugging Face Transformers PEFT. - -ServerlessLLM Store provides a model manager and two key functions: -- save_lora: Convert an LoRA adapter into a loading-optimized format and save it to a local path. -- load_lora: Load an adapter into loaded model. - -> Note: Fine-tuning is currently experimental, especially on multi-GPU machines. You may encounter issues when using this feature in multi-GPU environments. - -## Usage Examples - -1. Convert an adapter to ServerlessLLM format and save it to a local path: -``` -from sllm_store.transformers import save_lora - -# TODO: Load an adapter from HuggingFace model hub. - - -# Replace './models' with your local path. -save_lora(adapter, './models/facebook/opt-1.3b') -``` - -2. Launch the checkpoint store server in a separate process: -``` -# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. -sllm-store start --storage-path $PWD/models --mem-pool-size 4GB -``` - -3. Load the adapter on your model and make inference: -``` -import time -import torch -from sllm_store.transformers import load_model, load_lora - -model = load_model("facebook/opt-1.3b", device_map="auto", torch_dtype=torch.float16, storage_path="./models/", fully_parallel=True) - -model = load_lora("facebook/opt-1.3b", adapter_name="demo_lora", adapter_path="ft_facebook/opt-1.3b_adapter1", device_map="auto", torch_dtype=torch.float16, storage_path="./models/") - -# Please note the loading time depends on the base model size and the hardware bandwidth. -print(f"Model loading time: {time.time() - start:.2f}s") - -from transformers import AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained('facebook/opt-1.3b') -inputs = tokenizer('Hello, my dog is cute', return_tensors='pt').to("cuda") -outputs = model.generate(**inputs) -print(tokenizer.decode(outputs[0], skip_special_tokens=True)) -``` - -4. Clean up by `Ctrl+C` the server process. diff --git a/docs/stable/store/rocm_quickstart.md b/docs/stable/store/rocm_quickstart.md deleted file mode 100644 index 7807301..0000000 --- a/docs/stable/store/rocm_quickstart.md +++ /dev/null @@ -1,164 +0,0 @@ ---- -sidebar_position: 1 ---- - -# ROCm Quick Start - -ServerlessLLM Store (`sllm-store`) currently supports ROCm platform. However, there are no pre-built wheels for ROCm. - -Due to an internal bug in ROCm, serverless-llm-store may face a GPU memory leak in ROCm before version 6.2.0, as noted in [issue](https://github.com/ROCm/HIP/issues/3580). - -1. Clone the repository and enter the `store` directory: - -```bash -git clone https://github.com/ServerlessLLM/ServerlessLLM.git -cd ServerlessLLM/sllm_store -``` -After that, you may either use the Docker image or build the `sllm-store` wheel from source and install it in your environment. - -## Use the Docker image - -We provide a Dockerfile with ROCm support. Currently, it's built on base image `rocm/vllm-dev:base_ROCm-6.3.1_20250528_tuned_20250530` - -2. Build the Docker image: - -``` bash -docker build -t sllm_store_rocm -f Dockerfile.rocm . -``` - -3. Start the Docker container: - -:::tip -If you want to run inference outside the Docker container, you need to pass the port to the host machine. For example, `-p 8073:8073`. You can also get the wheel from the Docker container after starting it via `docker cp sllm_store_server:/app/dist .`. -::: - -``` bash -docker run --name sllm_store_server --rm -it \ - --device /dev/kfd --device /dev/dri \ - --security-opt seccomp=unconfined \ - -v $(pwd)/models:/models \ - sllm_store_rocm -``` - -Expected output: - -``` bash -INFO 06-05 12:59:07 cli.py:76] Starting gRPC server -INFO 06-05 12:59:07 server.py:40] StorageServicer: storage_path=/models, mem_pool_size=4294967296, num_thread=4, chunk_size=33554432, registration_required=False -WARNING: Logging before InitGoogleLogging() is written to STDERR -I20250605 12:59:11.141070 1 checkpoint_store_hip.cpp:42] Number of GPUs: 1 -I20250605 12:59:11.141098 1 checkpoint_store_hip.cpp:44] I/O threads: 4, chunk size: 32MB -I20250605 12:59:11.141103 1 checkpoint_store_hip.cpp:46] Storage path: "/models" -I20250605 12:59:11.141119 1 checkpoint_store_hip.cpp:72] GPU 0 UUID: 61363865-3865-3038-3831-366132376261 -I20250605 12:59:11.519277 1 pinned_memory_pool_hip.cpp:30] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes -I20250605 12:59:12.487957 1 checkpoint_store_hip.cpp:84] Memory pool created with 4GB -INFO 06-05 12:59:12 server.py:231] Starting gRPC server on 0.0.0.0:8073 - -``` - -After starting the Docker container, you can enter the container and run the following command to test the installation. - -``` bash -docker exec -it sllm_store_server /bin/bash -``` - -Try to save and load a transformer model: - -``` bash -python3 examples/save_transformers_model.py --model-name "facebook/opt-1.3b" --storage-path "/models" -python3 examples/load_transformers_model.py --model-name "facebook/opt-1.3b" --storage-path "/models" -``` -Expected output: - -``` bash -DEBUG 06-05 13:01:01 transformers.py:203] load_dict_non_blocking takes 0.0071375370025634766 seconds -DEBUG 06-05 13:01:01 transformers.py:213] load config takes 0.003943443298339844 seconds -DEBUG 06-05 13:01:01 torch.py:137] allocate_cuda_memory takes 0.0012660026550292969 seconds -DEBUG 06-05 13:01:01 client.py:72] load_into_gpu: facebook/opt-1.3b, 93b1932e-4b43-42cb-b82d-7228ef21810b -INFO 06-05 13:01:01 client.py:113] Model loaded: facebook/opt-1.3b, 93b1932e-4b43-42cb-b82d-7228ef21810b -INFO 06-05 13:01:01 torch.py:160] restore state_dict takes 0.0004298686981201172 seconds -DEBUG 06-05 13:01:02 transformers.py:224] load model takes 0.9706132411956787 seconds -INFO 06-05 13:01:02 client.py:117] confirm_model_loaded: facebook/opt-1.3b, 93b1932e-4b43-42cb-b82d-7228ef21810b -INFO 06-05 13:01:06 client.py:125] Model loaded -Model loading time: 5.28s -tokenizer_config.json: 100%|██████████████████████████████| 685/685 [00:00<00:00, 6.68MB/s] -vocab.json: 100%|███████████████████████████████████████| 899k/899k [00:00<00:00, 4.05MB/s] -merges.txt: 100%|███████████████████████████████████████| 456k/456k [00:00<00:00, 3.05MB/s] -special_tokens_map.json: 100%|████████████████████████████| 441/441 [00:00<00:00, 4.10MB/s] -/usr/local/lib/python3.12/dist-packages/torch/nn/modules/linear.py:125: UserWarning: Failed validator: GCN_ARCH_NAME (Triggered internally at /app/pytorch/aten/src/ATen/hip/tunable/Tunable.cpp:366.) - return F.linear(input, self.weight, self.bias) -Hello, my dog is cute and I want to give him a good home. I have a lot of experience with dogs and I -``` - -Try to save and load a model in vLLM: - -``` bash -python3 examples/save_vllm_model.py --model-name "facebook/opt-125m" --storage-path "/models" -python3 examples/load_vllm_model.py --model-name "facebook/opt-125m" --storage-path "/models" -``` -Expected output: - -``` bash -INFO 06-05 13:02:51 [__init__.py:243] Automatically detected platform rocm. -INFO 06-05 13:02:52 [__init__.py:31] Available plugins for group vllm.general_plugins: -INFO 06-05 13:02:52 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver -INFO 06-05 13:02:52 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load. -INFO 06-05 13:03:00 [config.py:793] This model supports multiple tasks: {'reward', 'embed', 'generate', 'classify', 'score'}. Defaulting to 'generate'. -INFO 06-05 13:03:00 [arg_utils.py:1594] rocm is experimental on VLLM_USE_V1=1. Falling back to V0 Engine. -INFO 06-05 13:03:04 [config.py:1910] Disabled the custom all-reduce kernel because it is not supported on current platform. -INFO 06-05 13:03:04 [llm_engine.py:230] Initializing a V0 LLM engine (v0.9.0.1) with config: model='/models/facebook/opt-125m', speculative_config=None, tokenizer='/models/facebook/opt-125m', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config={}, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=2048, download_dir=None, load_format=LoadFormat.SERVERLESS_LLM, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, decoding_config=DecodingConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_backend=''), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None), seed=0, served_model_name=/models/facebook/opt-125m, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=None, chunked_prefill_enabled=False, use_async_output_proc=True, pooler_config=None, compilation_config={"compile_sizes": [], "inductor_compile_config": {"enable_auto_functionalized_v2": false}, "cudagraph_capture_sizes": [256, 248, 240, 232, 224, 216, 208, 200, 192, 184, 176, 168, 160, 152, 144, 136, 128, 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 4, 2, 1], "max_capture_size": 256}, use_cached_outputs=False, -INFO 06-05 13:03:04 [rocm.py:208] None is not supported in AMD GPUs. -INFO 06-05 13:03:04 [rocm.py:209] Using ROCmFlashAttention backend. -INFO 06-05 13:03:05 [parallel_state.py:1064] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0, EP rank 0 -INFO 06-05 13:03:05 [model_runner.py:1170] Starting to load model /models/facebook/opt-125m... -DEBUG 06-05 13:03:05 torch.py:137] allocate_cuda_memory takes 0.0004763603210449219 seconds -DEBUG 06-05 13:03:05 client.py:72] load_into_gpu: facebook/opt-125m/rank_0, e8e7d900-652d-4822-8992-ad22f734b9c8 -INFO 06-05 13:03:05 client.py:113] Model loaded: facebook/opt-125m/rank_0, e8e7d900-652d-4822-8992-ad22f734b9c8 -INFO 06-05 13:03:05 torch.py:160] restore state_dict takes 0.00021338462829589844 seconds -INFO 06-05 13:03:05 client.py:117] confirm_model_loaded: facebook/opt-125m/rank_0, e8e7d900-652d-4822-8992-ad22f734b9c8 -INFO 06-05 13:03:05 client.py:125] Model loaded -INFO 06-05 13:03:05 [model_runner.py:1202] Model loading took 0.2363 GiB and 0.711783 seconds -/app/third_party/vllm/vllm/model_executor/layers/utils.py:80: UserWarning: Failed validator: GCN_ARCH_NAME (Triggered internally at /app/pytorch/aten/src/ATen/hip/tunable/Tunable.cpp:366.) - return torch.nn.functional.linear(x, weight, bias) -INFO 06-05 13:03:17 [worker.py:303] Memory profiling takes 11.68 seconds -INFO 06-05 13:03:17 [worker.py:303] the current vLLM instance can use total_gpu_memory (23.98GiB) x gpu_memory_utilization (0.90) = 21.59GiB -INFO 06-05 13:03:17 [worker.py:303] model weights take 0.24GiB; non_torch_memory takes 0.53GiB; PyTorch activation peak memory takes 0.49GiB; the rest of the memory reserved for KV Cache is 20.33GiB. -INFO 06-05 13:03:17 [executor_base.py:112] # rocm blocks: 37011, # CPU blocks: 7281 -INFO 06-05 13:03:17 [executor_base.py:117] Maximum concurrency for 2048 tokens per request: 289.15x -INFO 06-05 13:03:18 [model_runner.py:1526] Capturing cudagraphs for decoding. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI. If out-of-memory error occurs during cudagraph capture, consider decreasing `gpu_memory_utilization` or switching to eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage. -Capturing CUDA graph shapes: 100%|█████████████████████████| 35/35 [00:09<00:00, 3.55it/s] -INFO 06-05 13:03:28 [model_runner.py:1684] Graph capturing finished in 10 secs, took 0.13 GiB -INFO 06-05 13:03:28 [llm_engine.py:428] init engine (profile, create kv cache, warmup model) took 22.81 seconds -Adding requests: 100%|█████████████████████████████████████| 4/4 [00:00<00:00, 2079.22it/s] -Processed prompts: 100%|█| 4/4 [00:00<00:00, 6.71it/s, est. speed input: 43.59 toks/s, out -Prompt: 'Hello, my name is', Generated text: ' Joel, my dad is my friend and we are in a relationship. I am' -Prompt: 'The president of the United States is', Generated text: ' speaking out against the release of some State Department documents which show the Russians were involved' -Prompt: 'The capital of France is', Generated text: ' a worldwide knowledge center. What better place to learn about the history and culture of' -Prompt: 'The future of AI is', Generated text: " here: it's the future of everything\nIf you want to test your minds" -[rank0]:[W605 13:03:30.532018298 ProcessGroupNCCL.cpp:1476] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -``` - -## Build the wheel from source and install - -Currently, `pip install .` does not work with ROCm. We suggest you build `sllm-store` wheel and manually install it in your environment. - - - -If there's a customized PyTorch version installed, you may need to run the following command to modify the `torch` version in `requirements.txt`: - -```bash -python3 using_existing_torch.py -``` - -2. Build the wheel: - -```bash -python setup.py sdist bdist_wheel -``` - -## Known issues - -1. GPU memory leak in ROCm before version 6.2.0. - -This issue is due to an internal bug in ROCm. After the inference instance is completed, the GPU memory is still occupied and not released. For more information, please refer to [issue](https://github.com/ROCm/HIP/issues/3580). - diff --git a/docusaurus.config.js b/docusaurus.config.js index ce5082b..42fea8b 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -45,6 +45,22 @@ const config = { // Remove this to remove the "edit this page" links. editUrl: 'https://github.com/ServerlessLLM/serverlessllm.github.io/tree/main/', + // Versioning configuration + lastVersion: '0.7.0', // Make stable version the default + versions: { + current: { + label: 'Latest (dev)', + path: 'latest', + banner: 'unreleased', + badge: true, + }, + '0.7.0': { + label: '0.7.0 (stable)', + path: '/', // Stable version at root path + banner: 'none', + badge: false, + }, + }, }, blog: { showReadingTime: true, @@ -102,7 +118,7 @@ const config = { items: [ { label: 'Documents', - to: '/docs/stable/intro', + to: '/docs/intro', } ], }, diff --git a/sidebars.js b/sidebars.js index 196d255..0f8d482 100644 --- a/sidebars.js +++ b/sidebars.js @@ -14,13 +14,19 @@ /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ const sidebars = { // By default, Docusaurus generates a sidebar from the docs folder structure + // Excludes 'api' directory which has its own sidebar tutorialSidebar: [ { type: 'autogenerated', - dirName: 'stable', + dirName: '.', + }, + ], + apiSidebar: [ + { + type: 'autogenerated', + dirName: 'api', }, ], - apiSidebar: [{type: 'autogenerated', dirName: 'api'}], // But you can create a sidebar manually /* diff --git a/src/pages/index.js b/src/pages/index.js index b41695f..76403e1 100644 --- a/src/pages/index.js +++ b/src/pages/index.js @@ -21,7 +21,7 @@ function HomepageHeader() {
+ to="/docs/intro"> Let's go to ServerlessLLM documents!
From ab66037d878772a0579f83184208a20baa362072 Mon Sep 17 00:00:00 2001 From: future-xy Date: Thu, 6 Nov 2025 18:59:57 +0000 Subject: [PATCH 2/3] git commit -m "feat: add documentation versioning with 0.8.0 and 0.7.0 - Add Docusaurus versioning infrastructure - Create version snapshots for 0.8.0 (stable) and 0.7.0 - Set up Latest (dev) documentation track - Add GitHub Actions workflows for automated doc syncing - Add custom version banner components - Update configuration and sidebar structure --- VERSIONING.md | 309 +++++++++++++ docs/README.md | 2 +- docs/api/sllm-store-cli.md | 2 +- docs/community/_category_.json | 4 + docs/community/meetups.md | 11 + docs/community/talks.md | 8 + docs/deployment/_category_.json | 4 + docs/deployment/multi_machine.md | 296 +++++++++++++ docs/deployment/single_machine.md | 217 +++++++++ docs/deployment/slurm_cluster.md | 411 ++++++++++++++++++ docs/developer/_category_.json | 4 + docs/developer/supporting_a_new_hardware.md | 25 ++ docs/features/_category_.json | 4 + docs/features/live_migration.md | 211 +++++++++ docs/features/peft_lora_fine_tuning.md | 328 ++++++++++++++ docs/features/peft_lora_serving.md | 116 +++++ docs/features/quantized_models.md | 175 ++++++++ docs/features/storage_aware_scheduling.md | 123 ++++++ docs/getting_started.md | 136 ++++++ docs/intro.md | 38 ++ docs/models/_category_.json | 4 + docs/models/supported_models.md | 13 + docs/store/_category_.json | 4 + docs/store/quantization.md | 102 +++++ docs/store/quickstart.md | 245 +++++++++++ docs/store/rocm_quickstart.md | 164 +++++++ docusaurus.config.js | 21 +- sidebars.js | 64 ++- src/theme/DocVersionBanner/index.js | 110 +++++ versioned_docs/version-0.7.0/README.md | 39 ++ versioned_docs/version-0.7.0/api/cli.md | 409 +++++++++++++++++ versioned_docs/version-0.7.0/api/intro.md | 9 + .../version-0.7.0/community/_category_.json | 4 + .../version-0.7.0/community/meetups.md | 11 + .../version-0.7.0/community/talks.md | 8 + .../version-0.7.0/deployment/_category_.json | 4 + .../version-0.7.0/deployment/multi_machine.md | 296 +++++++++++++ .../deployment/single_machine.md | 217 +++++++++ .../version-0.7.0/deployment/slurm_cluster.md | 411 ++++++++++++++++++ .../version-0.7.0/developer/_category_.json | 4 + .../developer/supporting_a_new_hardware.md | 25 ++ .../version-0.7.0/features/_category_.json | 4 + .../version-0.7.0/features/live_migration.md | 211 +++++++++ .../features/peft_lora_serving.md | 105 +++++ .../features/storage_aware_scheduling.md | 123 ++++++ .../version-0.7.0/getting_started.md | 136 ++++++ .../version-0.7.0/images/favicon.ico | Bin 0 -> 46169 bytes versioned_docs/version-0.7.0/images/logo.svg | 302 +++++++++++++ .../version-0.7.0/images/serverlessllm.jpg | Bin 0 -> 102673 bytes .../version-0.7.0/images/wechat.png | Bin 0 -> 7417 bytes versioned_docs/version-0.7.0/intro.md | 38 ++ .../version-0.7.0/models/_category_.json | 4 + .../version-0.7.0/models/supported_models.md | 13 + .../version-0.7.0/store/_category_.json | 4 + .../version-0.7.0/store/quickstart.md | 294 +++++++++++++ .../version-0.7.0/store/rocm_quickstart.md | 174 ++++++++ versioned_docs/version-0.8.0/README.md | 39 ++ versioned_docs/version-0.8.0/api/cli.md | 323 ++++++++++++++ versioned_docs/version-0.8.0/api/intro.md | 9 + .../version-0.8.0/api/sllm-store-cli.md | 242 +++++++++++ .../version-0.8.0/community/_category_.json | 4 + .../version-0.8.0/community/meetups.md | 11 + .../version-0.8.0/community/talks.md | 8 + .../version-0.8.0/deployment/_category_.json | 4 + .../version-0.8.0/deployment/multi_machine.md | 296 +++++++++++++ .../deployment/single_machine.md | 217 +++++++++ .../version-0.8.0/deployment/slurm_cluster.md | 411 ++++++++++++++++++ .../version-0.8.0/developer/_category_.json | 4 + .../developer/supporting_a_new_hardware.md | 25 ++ .../version-0.8.0/features/_category_.json | 4 + .../version-0.8.0/features/live_migration.md | 211 +++++++++ .../features/peft_lora_fine_tuning.md | 328 ++++++++++++++ .../features/peft_lora_serving.md | 116 +++++ .../features/quantized_models.md | 175 ++++++++ .../features/storage_aware_scheduling.md | 123 ++++++ .../version-0.8.0/getting_started.md | 136 ++++++ .../version-0.8.0/images/favicon.ico | Bin 0 -> 46169 bytes versioned_docs/version-0.8.0/images/logo.svg | 302 +++++++++++++ .../version-0.8.0/images/serverlessllm.jpg | Bin 0 -> 102673 bytes .../version-0.8.0/images/wechat.png | Bin 0 -> 13949 bytes versioned_docs/version-0.8.0/intro.md | 38 ++ .../version-0.8.0/models/_category_.json | 4 + .../version-0.8.0/models/supported_models.md | 13 + .../version-0.8.0/store/_category_.json | 4 + .../version-0.8.0/store/quantization.md | 102 +++++ .../version-0.8.0/store/quickstart.md | 245 +++++++++++ .../version-0.8.0/store/rocm_quickstart.md | 164 +++++++ .../version-0.7.0-sidebars.json | 72 +++ .../version-0.8.0-sidebars.json | 72 +++ versions.json | 1 + 90 files changed, 9689 insertions(+), 10 deletions(-) create mode 100644 VERSIONING.md create mode 100644 docs/community/_category_.json create mode 100644 docs/community/meetups.md create mode 100644 docs/community/talks.md create mode 100644 docs/deployment/_category_.json create mode 100644 docs/deployment/multi_machine.md create mode 100644 docs/deployment/single_machine.md create mode 100644 docs/deployment/slurm_cluster.md create mode 100644 docs/developer/_category_.json create mode 100644 docs/developer/supporting_a_new_hardware.md create mode 100644 docs/features/_category_.json create mode 100644 docs/features/live_migration.md create mode 100644 docs/features/peft_lora_fine_tuning.md create mode 100644 docs/features/peft_lora_serving.md create mode 100644 docs/features/quantized_models.md create mode 100644 docs/features/storage_aware_scheduling.md create mode 100644 docs/getting_started.md create mode 100644 docs/intro.md create mode 100644 docs/models/_category_.json create mode 100644 docs/models/supported_models.md create mode 100644 docs/store/_category_.json create mode 100644 docs/store/quantization.md create mode 100644 docs/store/quickstart.md create mode 100644 docs/store/rocm_quickstart.md create mode 100644 src/theme/DocVersionBanner/index.js create mode 100644 versioned_docs/version-0.7.0/README.md create mode 100644 versioned_docs/version-0.7.0/api/cli.md create mode 100644 versioned_docs/version-0.7.0/api/intro.md create mode 100644 versioned_docs/version-0.7.0/community/_category_.json create mode 100644 versioned_docs/version-0.7.0/community/meetups.md create mode 100644 versioned_docs/version-0.7.0/community/talks.md create mode 100644 versioned_docs/version-0.7.0/deployment/_category_.json create mode 100644 versioned_docs/version-0.7.0/deployment/multi_machine.md create mode 100644 versioned_docs/version-0.7.0/deployment/single_machine.md create mode 100644 versioned_docs/version-0.7.0/deployment/slurm_cluster.md create mode 100644 versioned_docs/version-0.7.0/developer/_category_.json create mode 100644 versioned_docs/version-0.7.0/developer/supporting_a_new_hardware.md create mode 100644 versioned_docs/version-0.7.0/features/_category_.json create mode 100644 versioned_docs/version-0.7.0/features/live_migration.md create mode 100644 versioned_docs/version-0.7.0/features/peft_lora_serving.md create mode 100644 versioned_docs/version-0.7.0/features/storage_aware_scheduling.md create mode 100644 versioned_docs/version-0.7.0/getting_started.md create mode 100644 versioned_docs/version-0.7.0/images/favicon.ico create mode 100644 versioned_docs/version-0.7.0/images/logo.svg create mode 100644 versioned_docs/version-0.7.0/images/serverlessllm.jpg create mode 100644 versioned_docs/version-0.7.0/images/wechat.png create mode 100644 versioned_docs/version-0.7.0/intro.md create mode 100644 versioned_docs/version-0.7.0/models/_category_.json create mode 100644 versioned_docs/version-0.7.0/models/supported_models.md create mode 100644 versioned_docs/version-0.7.0/store/_category_.json create mode 100644 versioned_docs/version-0.7.0/store/quickstart.md create mode 100644 versioned_docs/version-0.7.0/store/rocm_quickstart.md create mode 100644 versioned_docs/version-0.8.0/README.md create mode 100644 versioned_docs/version-0.8.0/api/cli.md create mode 100644 versioned_docs/version-0.8.0/api/intro.md create mode 100644 versioned_docs/version-0.8.0/api/sllm-store-cli.md create mode 100644 versioned_docs/version-0.8.0/community/_category_.json create mode 100644 versioned_docs/version-0.8.0/community/meetups.md create mode 100644 versioned_docs/version-0.8.0/community/talks.md create mode 100644 versioned_docs/version-0.8.0/deployment/_category_.json create mode 100644 versioned_docs/version-0.8.0/deployment/multi_machine.md create mode 100644 versioned_docs/version-0.8.0/deployment/single_machine.md create mode 100644 versioned_docs/version-0.8.0/deployment/slurm_cluster.md create mode 100644 versioned_docs/version-0.8.0/developer/_category_.json create mode 100644 versioned_docs/version-0.8.0/developer/supporting_a_new_hardware.md create mode 100644 versioned_docs/version-0.8.0/features/_category_.json create mode 100644 versioned_docs/version-0.8.0/features/live_migration.md create mode 100644 versioned_docs/version-0.8.0/features/peft_lora_fine_tuning.md create mode 100644 versioned_docs/version-0.8.0/features/peft_lora_serving.md create mode 100644 versioned_docs/version-0.8.0/features/quantized_models.md create mode 100644 versioned_docs/version-0.8.0/features/storage_aware_scheduling.md create mode 100644 versioned_docs/version-0.8.0/getting_started.md create mode 100644 versioned_docs/version-0.8.0/images/favicon.ico create mode 100644 versioned_docs/version-0.8.0/images/logo.svg create mode 100644 versioned_docs/version-0.8.0/images/serverlessllm.jpg create mode 100644 versioned_docs/version-0.8.0/images/wechat.png create mode 100644 versioned_docs/version-0.8.0/intro.md create mode 100644 versioned_docs/version-0.8.0/models/_category_.json create mode 100644 versioned_docs/version-0.8.0/models/supported_models.md create mode 100644 versioned_docs/version-0.8.0/store/_category_.json create mode 100644 versioned_docs/version-0.8.0/store/quantization.md create mode 100644 versioned_docs/version-0.8.0/store/quickstart.md create mode 100644 versioned_docs/version-0.8.0/store/rocm_quickstart.md create mode 100644 versioned_sidebars/version-0.7.0-sidebars.json create mode 100644 versioned_sidebars/version-0.8.0-sidebars.json create mode 100644 versions.json diff --git a/VERSIONING.md b/VERSIONING.md new file mode 100644 index 0000000..83c8e3b --- /dev/null +++ b/VERSIONING.md @@ -0,0 +1,309 @@ +# ServerlessLLM Documentation Versioning Guide + +## Overview + +The ServerlessLLM documentation now supports versioning with two documentation tracks: + +1. **Stable versions** (0.7.0, 0.8.0, etc.) - Released, tested documentation for production use +2. **Latest (dev)** - Bleeding-edge documentation synced from the main branch + +## URL Structure + +- `/docs/` or `/docs/intro` → **0.7.0 (stable)** - Default for users +- `/docs/latest/` or `/docs/latest/intro` → **Latest (dev)** - For developers +- `/docs/0.7.0/` → Specific stable version +- `/docs/api/` → API documentation (same for all versions) + +## Version Dropdown + +Users can switch between versions using the dropdown in the navigation bar: +- **0.7.0 (stable)** - Current stable release (default) +- **Latest (dev)** - Development documentation with unreleased features +- Older versions (0.6.0, 0.5.0, etc.) as they are added + +## Directory Structure + +``` +serverlessllm.github.io/ +├── docs/ # Latest (dev) - synced from main branch +│ ├── intro.md +│ ├── getting_started.md +│ ├── deployment/ +│ ├── features/ +│ ├── store/ +│ ├── api/ # API docs (shared across versions) +│ └── ... +├── versioned_docs/ # Stable version snapshots +│ └── version-0.7.0/ # 0.7.0 stable release +│ ├── intro.md +│ ├── getting_started.md +│ └── ... +├── versioned_sidebars/ # Sidebar configs for each version +│ └── version-0.7.0-sidebars.json +└── versions.json # List of all versions: ["0.7.0"] +``` + +## How Versioning Works + +### For Latest (Dev) Documentation + +1. Developer commits to `main` branch in ServerlessLLM repository +2. Sync workflow triggers +3. Documentation in `docs/` folder is updated +4. Users accessing `/docs/latest/` see the new content + +### For Stable Version Documentation + +1. New release is published (e.g., v0.8.0) +2. Version snapshot workflow triggers +3. Creates `versioned_docs/version-0.8.0/` with frozen documentation +4. Updates `versions.json` to include 0.8.0 +5. Updates config to make 0.8.0 the new default +6. Users accessing `/docs/` now see 0.8.0 by default + +## Sync Workflows + +### 1. Sync Latest Docs (Continuous) + +**Location:** `.github/workflows/sync-latest-docs.yml` + +**Trigger:** Repository dispatch event `sync-latest-docs` + +**Purpose:** Updates the "Latest (dev)" documentation from main branch + +**Testing:** +```bash +gh workflow run sync-latest-docs.yml +``` + +### 2. Create Version Snapshot (On Release) + +**Location:** `.github/workflows/create-version-snapshot.yml` + +**Trigger:** Repository dispatch event `create-version-snapshot` or manual dispatch + +**Purpose:** Creates a versioned snapshot when a new release is published + +**Testing:** +```bash +gh workflow run create-version-snapshot.yml \ + -f version=0.8.0 \ + -f tag=v0.8.0 +``` + +## Main Repository Setup + +The main ServerlessLLM repository needs workflow files to trigger documentation syncing. + +See `.github/MAIN_REPO_WORKFLOWS.md` for detailed instructions on: +- Setting up sync workflows in the main repo +- Creating GitHub tokens +- Configuring repository dispatch events + +## Creating a New Version + +### Manual Process + +When releasing a new version (e.g., 0.8.0): + +1. **Sync docs from the release tag:** +```bash +# Clone the main repo at the specific tag +git clone --depth 1 --branch v0.8.0 https://github.com/ServerlessLLM/ServerlessLLM.git temp_repo + +# Remove current docs (keep api/) +find docs/ -mindepth 1 -maxdepth 1 ! -name 'api' ! -name 'images' ! -name 'README.md' -exec rm -rf {} + + +# Copy docs from release +cp -r temp_repo/docs/* docs/ + +# Clean up +rm -rf temp_repo +``` + +2. **Create version snapshot:** +```bash +npm run docusaurus docs:version 0.8.0 +``` + +3. **Update docusaurus.config.js:** +```javascript +docs: { + lastVersion: '0.8.0', // Update to new stable version + versions: { + current: { + label: 'Latest (dev)', + path: 'latest', + banner: 'unreleased', + }, + '0.8.0': { + label: '0.8.0 (stable)', + path: '/', + banner: 'none', + }, + '0.7.0': { + label: '0.7.0', + // Older version, no longer default + }, + }, +} +``` + +4. **Commit and push:** +```bash +git add . +git commit -m "docs: create version 0.8.0 snapshot" +git push +``` + +### Automated Process (Recommended) + +Use the `create-version-snapshot.yml` workflow: + +```bash +# Trigger from main ServerlessLLM repo on release +# OR manually trigger from this repo: +gh workflow run create-version-snapshot.yml \ + -f version=0.8.0 \ + -f tag=v0.8.0 +``` + +## Version Configuration + +### versions.json + +Lists all stable versions: +```json +[ + "0.8.0", + "0.7.0", + "0.6.0" +] +``` + +### docusaurus.config.js + +Configure version behavior: + +```javascript +docs: { + lastVersion: '0.8.0', // Default version for /docs/ + versions: { + current: { + label: 'Latest (dev)', // Label in dropdown + path: 'latest', // URL path + banner: 'unreleased', // Warning banner + badge: true, // Show badge + }, + '0.8.0': { + label: '0.8.0 (stable)', + path: '/', // Root path (default) + banner: 'none', + }, + '0.7.0': { + label: '0.7.0', + // Uses default path: /docs/0.7.0/ + }, + }, +} +``` + +## Migration Summary + +The migration from the old structure involved: + +1. ✅ Moved `docs/stable/*` → `docs/*` +2. ✅ Updated `sidebars.js` to use root directory +3. ✅ Created initial version 0.7.0 snapshot +4. ✅ Configured versioning in `docusaurus.config.js` +5. ✅ Fixed broken internal links +6. ✅ Updated homepage link +7. ✅ Created sync workflows + +## Breaking Changes + +### URL Changes + +Old URL structure: +- `/docs/stable/intro` → Documentation + +New URL structure: +- `/docs/intro` → Stable documentation (0.7.0) +- `/docs/latest/intro` → Latest development docs +- `/docs/0.7.0/intro` → Specific version + +**Impact:** External links to `/docs/stable/*` will break and need updating. + +### Recommendation + +Set up redirects for old URLs: +```javascript +// In docusaurus.config.js +plugins: [ + [ + '@docusaurus/plugin-client-redirects', + { + redirects: [ + { + from: '/docs/stable/:path', + to: '/docs/:path', + }, + ], + }, + ], +], +``` + +## Troubleshooting + +### Build Errors + +**Issue:** Broken links after restructuring + +**Solution:** Update all internal links: +- `../stable/path.md` → `../path.md` +- Ensure cross-references between `docs/` and `docs/api/` use correct relative paths + +### Version Not Showing in Dropdown + +**Issue:** New version not visible + +**Solution:** +1. Check `versions.json` includes the version +2. Verify `versioned_docs/version-X.X.X/` exists +3. Rebuild: `npm run build` + +### Wrong Default Version + +**Issue:** Latest (dev) shows by default instead of stable + +**Solution:** Ensure `lastVersion` in config points to stable version: +```javascript +docs: { + lastVersion: '0.7.0', // Should be stable, not 'current' +} +``` + +## Best Practices + +1. **Always sync from tagged releases** for stable versions, not from branches +2. **Test versioned docs locally** before deploying +3. **Update version labels** in config when creating new versions +4. **Keep version history** - don't delete old versions unless necessary +5. **Document breaking changes** between versions in release notes + +## Future Enhancements + +Potential improvements to consider: + +1. **Automatic version creation** on GitHub releases +2. **Version deprecation banners** for outdated versions +3. **Version-specific search** to filter results by version +4. **Changelog integration** linking versions to release notes +5. **Version comparison tools** to see changes between versions + +## Support + +For issues or questions about versioning: +- Check Docusaurus versioning docs: https://docusaurus.io/docs/versioning +- Report issues: https://github.com/ServerlessLLM/serverlessllm.github.io/issues diff --git a/docs/README.md b/docs/README.md index 0ef88bf..4a1397a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # ServerlessLLM documents -Please find our documents in [ServerlessLLM](https://serverlessllm.github.io/docs/stable/getting_started). +Please find our documents in [ServerlessLLM](https://serverlessllm.github.io/docs/getting_started). ## How to build ServerlessLLM Docs diff --git a/docs/api/sllm-store-cli.md b/docs/api/sllm-store-cli.md index 5ab0900..1efe935 100644 --- a/docs/api/sllm-store-cli.md +++ b/docs/api/sllm-store-cli.md @@ -208,7 +208,7 @@ sllm-store load [OPTIONS] - Adapter name to save. Must be a Hugging Face pretrained LoRA adapter name. - `--precision ` - - Precision to use when loading the model (`transformers` backend only). For more info on quantization in ServerlessLLM, visit [here](https://serverlessllm.github.io/docs/stable/store/quickstart#quantization). + - Precision to use when loading the model (`transformers` backend only). For more info on quantization in ServerlessLLM, visit [here](https://serverlessllm.github.io/docs/store/quickstart#quantization). - `--storage-path ` - Location where the model will be loaded from. diff --git a/docs/community/_category_.json b/docs/community/_category_.json new file mode 100644 index 0000000..8a81fd9 --- /dev/null +++ b/docs/community/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Community", + "position": 8 +} diff --git a/docs/community/meetups.md b/docs/community/meetups.md new file mode 100644 index 0000000..8e3811a --- /dev/null +++ b/docs/community/meetups.md @@ -0,0 +1,11 @@ +# ServerlessLLM Meetups + +We host regular biweekly developer meetings online. We will share project updates from the ServerlessLLM developer team presented during these meetings. Please find the materials of our previous meetups below: + +Date |Topic |Slides +---------------|-------------|--------- +February 21st 2025 | Fine Tuning | [Slides](https://docs.google.com/presentation/d/1rnw3mieAAbMabDIoIGS-ciMGc3hJ7AICYSaNJp-Fk4s/edit?usp=sharing) +March 7th 2025 |Quantization |[Slides](https://docs.google.com/presentation/d/1uSbP-LzGbbvPsemIAE6jCFsggYm_ATxQguCHDmdwoXE/edit?usp=sharing) + +We are always looking for contributors to join us on the developer team. If you are interested in contributing, consult our [job board](https://github.com/orgs/ServerlessLLM/projects/2) and claim a feature. For any other questions, please contact us on [this email](mailto:Y.Fu@ed.ac.uk) or on [our Discord server](https://discord.gg/AEF8Gduvm8). + diff --git a/docs/community/talks.md b/docs/community/talks.md new file mode 100644 index 0000000..66a2531 --- /dev/null +++ b/docs/community/talks.md @@ -0,0 +1,8 @@ +# ServerlessLLM Talks + +Materials for ServerlessLLM talks will be listed here. + +Topic |Location |Date |Links +-------------|----------------|---------------|------------------------------------ +Efficient Sharing of AI Infrastructures with Specialized Serverless Computing | University of Pennsylvania |January 29th 2025 |[Slides](https://drive.google.com/file/d/17GwXsqaDDS7Xw8nX_-RaKiwpaPQgu9WD/view) \| [Event](https://asset.seas.upenn.edu/event/yao-fu-university-of-edinburgh/) +ServerlessLLM Tutorial | SESAME'25 | March 31st 2025 |[Slides](https://docs.google.com/presentation/d/1ioGCVpsg0x3oCxX19EiE820aMiY22X5MG6jgImZ1W18/edit?usp=sharing) \| [Event](https://sesame25.github.io/) diff --git a/docs/deployment/_category_.json b/docs/deployment/_category_.json new file mode 100644 index 0000000..534be1d --- /dev/null +++ b/docs/deployment/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Deployment", + "position": 3 +} diff --git a/docs/deployment/multi_machine.md b/docs/deployment/multi_machine.md new file mode 100644 index 0000000..21583c3 --- /dev/null +++ b/docs/deployment/multi_machine.md @@ -0,0 +1,296 @@ +--- +sidebar_position: 2 +--- + +# Multi-machine + +This guide will help you get started with running ServerlessLLM on multiple machines using Docker containers. You'll learn how to set up a head node on one machine and connect worker nodes from different machines using Docker, ensuring proper network communication between the containers. You can extend this setup to use as many nodes as you need. + +## Prerequisites + +This guide requires **two machines**: +- One machine for the head node (no GPU required) +- One machine with an NVIDIA GPU to serve as the worker node + +You can add more worker machines with GPUs as needed to scale out your deployment. + +### For All Machines + +Ensure you have the following installed and configured on all machines (both head node and worker machines): + +1. **Docker**: Installed on your system. You can download it from [here](https://docs.docker.com/get-docker/). +2. **Network connectivity**: Ensure all machines can communicate with each other on the required ports (6379 for Ray, 8343 for ServerlessLLM API, and 8073 for storage service). + +:::tip +The **ServerlessLLM CLI** (`pip install serverless-llm`) can be installed on any machine that needs to manage model deployments. This could be your local computer or any machine within the cluster that can connect to the head node. +::: + +### For Worker Machines Only + +These requirements are only necessary for the worker machines that will run the models: + +1. **GPUs**: At least one NVIDIA GPU is required on each worker machine. If you have multiple GPUs, you can adjust the Docker configuration accordingly. +2. **NVIDIA Docker Toolkit**: This enables Docker to utilize NVIDIA GPUs. Follow the installation guide [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). + +## Multi-Machine Setup + +We'll start a head node on one machine using Docker, then add a worker node from another machine using Docker containers with host networking. + +### Step 1: Start the Head Node + +1. **Start the head node using Docker:** + +```bash +# Get the machine's IP address that will be accessible to other machines +export HEAD_IP=$(hostname -I | awk '{print $1}') +echo "Head node IP address: $HEAD_IP" + +docker run -d \ + --name sllm_head \ + --network host \ + -e MODE=HEAD \ + -e RAY_NODE_IP=$HEAD_IP \ + serverlessllm/sllm:latest +``` + +:::important +For multi-machine setups, setting the `RAY_NODE_IP` is critical. It should be set to an IP address that is accessible from all worker machines. The command above attempts to automatically determine your machine's primary IP, but in complex network environments, you may need to specify it manually. + +If your machine has multiple network interfaces, ensure you use the IP that other machines in your network can access. +::: + +:::tip +If you don't have the ServerlessLLM Docker image locally, Docker will automatically pull it from the registry. You can also adjust the CPU and resource allocations by setting additional environment variables like `RAY_NUM_CPUS` and `RAY_RESOURCES`. +::: + +2. **Verify the head node is running and note the external IP:** + +```bash +docker logs sllm_head +``` + +Expected output should include: + +```bash +> docker logs sllm_head +... +2025-05-29 14:29:46,211 INFO scripts.py:744 -- Local node IP: 129.215.164.107 +... +(SllmController pid=380) INFO 05-29 14:29:53 controller.py:59] Starting store manager +(SllmController pid=380) INFO 05-29 14:29:56 controller.py:68] Starting scheduler +(StoreManager pid=417) INFO 05-29 14:29:56 store_manager.py:226] Initializing store manager +(StoreManager pid=417) INFO 05-29 14:29:56 store_manager.py:237] Initializing cluster and collecting hardware info +(StoreManager pid=417) ERROR 05-29 14:29:56 store_manager.py:242] No worker nodes found +INFO: Started server process [1] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:8343 (Press CTRL+C to quit) +(FcfsScheduler pid=456) INFO 05-29 14:29:56 fcfs_scheduler.py:54] Starting FCFS scheduler +(FcfsScheduler pid=456) INFO 05-29 14:29:56 fcfs_scheduler.py:111] Starting control loop +``` + +Make note of the IP address shown in the logs. This is the address that worker nodes will use to connect to the head node. + +### Step 2: Start Worker Node on a Different Machine + +:::tip +You can adjust the memory pool size and other parameters based on the resources available on your worker machine. +::: + +1. **On the worker machine, create a directory for model storage:** + +```bash +mkdir -p /path/to/your/models +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +2. **Start the worker node:** + +```bash +# Replace with the actual IP address of the head node from the previous step +# DO NOT copy-paste this line directly - update with your actual head node IP +export HEAD_IP= +``` + +```bash +# Get the worker machine's IP address that will be accessible to the head node +export WORKER_IP=$(hostname -I | awk '{print $1}') +echo "Worker node IP address: $WORKER_IP" + +docker run -d \ + --name sllm_worker_0 \ + --network host \ + --gpus '"device=0"' \ + -e WORKER_ID=0 \ + -e STORAGE_PATH=/models \ + -e MODE=WORKER \ + -e RAY_HEAD_ADDRESS=${HEAD_IP}:6379 \ + -e RAY_NODE_IP=$WORKER_IP \ + -v ${MODEL_FOLDER}:/models \ + serverlessllm/sllm:latest \ + --mem-pool-size 4GB --registration-required true +``` + +:::important +For multi-machine setups, setting the `RAY_NODE_IP` on worker nodes is just as critical as on the head node. It should be set to an IP address that is accessible from the head node. Without this, workers might report internal Docker IPs that aren't accessible across machines. + +Make sure to replace `192.168.1.100` with the actual IP address of your head node that you noted earlier. +::: + +3. **Verify worker node is connected:** + +On the worker machine, check if the worker has properly connected to the Ray cluster: + +```bash +docker exec -it sllm_worker_0 bash -c "source /opt/conda/etc/profile.d/conda.sh && conda activate worker && ray status" +``` + +Expected output should include both the head node and worker node resources: + +```bash +> docker exec -it sllm_worker_0 bash -c "source /opt/conda/etc/profile.d/conda.sh && conda activate worker && ray status" +======== Autoscaler status: 2025-05-29 14:42:30.434645 ======== +Node status +--------------------------------------------------------------- +Active: + 1 node_f0a8e97ca64c64cebd551f469a38d0d66ce304f7cc1cc9696fe33cf3 + 1 node_3b7db178afb8bdb16460d0cb6463dc7b9b3afbcc00753c3be110c9b3 +Pending: + (no pending nodes) +Recent failures: + (no failures) + +Resources +--------------------------------------------------------------- +Usage: + 3.0/52.0 CPU + 0.0/1.0 GPU + 0.30000000000000004/1.0 control_node + 0B/526.36GiB memory + 0B/18.63GiB object_store_memory + 0.0/1.0 worker_id_0 + 0.0/1.0 worker_node + +Demands: + (no resource demands) +``` + +This output confirms that both the head node and worker node are properly connected and their resources are recognized by the Ray cluster. + +:::tip +**Adding more worker nodes:** You can add more worker nodes by repeating Step 2 on additional machines with GPUs. Just make sure to: +1. Use a unique `WORKER_ID` for each worker (1, 2, 3, etc.) +2. Point each worker to the same head node IP address +3. Ensure each worker has its own `RAY_NODE_IP` set correctly +::: + +### Step 3: Use `sllm` to manage models + +#### Configure the Environment + +**On any machine with `sllm` installed, set the `LLM_SERVER_URL` environment variable:** + +> Replace `` with the actual IP address of the head node. + +```bash +export LLM_SERVER_URL=http://:8343 +``` + +#### Deploy a Model Using `sllm` + +```bash +sllm deploy --model facebook/opt-1.3b +``` + +> Note: This command will spend some time downloading the model from the Hugging Face Model Hub. You can use any model from the [Hugging Face Model Hub](https://huggingface.co/models) by specifying the model name in the `--model` argument. + +Expected output: + +```bash +INFO 07-24 06:51:32 deploy.py:83] Model registered successfully. +``` + +### Step 4: Query the Model Using OpenAI API Client + +**You can query the model using any OpenAI API client. For example, use the following command:** + +**Make sure the model is successfully deployed before querying.** + +> Replace `` with the actual IP address of the head node. + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +Expected output: + +```json +{"id":"chatcmpl-23d3c0e5-70a0-4771-acaf-bcb2851c6ea6","object":"chat.completion","created":1721706121,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} +``` + +#### Delete a Deployed Model Using `sllm` + +When you're done using a model, you can delete it: + +```bash +sllm delete facebook/opt-1.3b +``` + +This will remove the specified model from the ServerlessLLM server. + +## Clean Up + +To stop and remove all ServerlessLLM containers: + +1. **Stop all containers:** + +```bash +# On head node machine +docker stop sllm_head +docker rm sllm_head + +# On each worker machine +docker stop sllm_worker_0 # Use appropriate container name (sllm_worker_1, sllm_worker_2, etc.) +docker rm sllm_worker_0 +``` + +2. **Optional: Remove the Docker image:** + +```bash +docker rmi serverlessllm/sllm:latest +``` + +:::tip +If you don't have the ServerlessLLM Docker image locally, Docker will automatically pull it from the registry. You can also adjust the CPU and resource allocations by setting additional environment variables like `RAY_NUM_CPUS` and `RAY_RESOURCES`. +::: + +## Troubleshooting + +### Network Issues + +1. **Connection refused errors**: Ensure that firewalls on all machines allow traffic on ports 6379, 8343, and 8073. + +2. **Ray cluster connection issues**: + - Verify that the head node IP address is correct and that the Ray port (6379) is accessible from worker machines + - Ensure both head and worker nodes have their `RAY_NODE_IP` set to an IP address that is accessible from other machines + - Check that you're not using private Docker network IPs (typically 172.x.x.x) which aren't accessible across machines + +3. **Workers can't connect to head node**: + - Make sure the `RAY_HEAD_ADDRESS` points to the external IP of the head node, not localhost or an internal Docker IP + - Verify network connectivity with `ping` or `telnet` from worker machines to the head node IP on port 6379 + +4. **GPU access issues**: Make sure the NVIDIA Docker toolkit is properly installed and that the `--gpus` flag is used for worker containers. + +### Container Management + +- **View running containers**: `docker ps` \ No newline at end of file diff --git a/docs/deployment/single_machine.md b/docs/deployment/single_machine.md new file mode 100644 index 0000000..7e064a2 --- /dev/null +++ b/docs/deployment/single_machine.md @@ -0,0 +1,217 @@ +--- +sidebar_position: 1 +--- + +# Single machine (from scratch) + +This guide provides instructions for setting up ServerlessLLM from scratch on a single machine. This 'from scratch' approach means you will manually initialize and manage the Ray cluster components. It involves using multiple terminal sessions, each configured with a distinct Conda environment, to run the head and worker processes on the same physical machine, effectively simulating a multi-node deployment locally. + +:::note +We strongly recommend using Docker (Compose) as detailed in the [Docker Compose guide](../getting_started.md). Docker provides a smoother and generally easier setup process. Follow this guide only if Docker is not a suitable option for your environment. +::: + +## Installation + +### Requirements + +Ensure your system meets the following prerequisites: + +- **OS**: Ubuntu 20.04 +- **Python**: 3.10 +- **GPU**: NVIDIA GPU with compute capability 7.0 or higher + +### Installing with pip + +Follow these steps to install ServerlessLLM using pip: + +**Create the head environment:** + +```bash +# Create and activate a conda environment +conda create -n sllm python=3.10 -y +conda activate sllm + +# Install ServerlessLLM and its store component +pip install serverless-llm serverless-llm-store +``` + +**Create the worker environment:** + +```bash +# Create and activate a conda environment +conda create -n sllm-worker python=3.10 -y +conda activate sllm-worker + +# Install ServerlessLLM (worker version) and its store component +pip install "serverless-llm[worker]" serverless-llm-store +``` + +:::note +If you plan to integrate vLLM with ServerlessLLM, a patch needs to be applied to the vLLM repository. For detailed instructions, please refer to the [vLLM Patch](#vllm-patch) section. +::: + +### Installing from Source + +To install ServerlessLLM from source, follow these steps: + +1. Clone the repository: + ```bash + git clone https://github.com/ServerlessLLM/ServerlessLLM.git + cd ServerlessLLM + ``` + +2. Create the head environment: + ```bash + # Create and activate a conda environment + conda create -n sllm python=3.10 -y + conda activate sllm + + # Install sllm_store (pip install is recommended for speed) + cd sllm_store && rm -rf build + pip install . + cd .. + + # Install ServerlessLLM + pip install . + ``` + +3. Create the worker environment: + ```bash + # Create and activate a conda environment + conda create -n sllm-worker python=3.10 -y + conda activate sllm-worker + + # Install sllm_store (pip install is recommended for speed) + cd sllm_store && rm -rf build + pip install . + cd .. + + # Install ServerlessLLM (worker version) + pip install ".[worker]" + ``` + +### vLLM Patch + +To use vLLM with ServerlessLLM, you must apply a patch. The patch file is located at `sllm_store/vllm_patch/sllm_load.patch` within the ServerlessLLM repository. This patch has been tested with vLLM version `0.9.0.1`. + +Apply the patch using the following script: + +```bash +conda activate sllm-worker +./sllm_store/vllm_patch/patch.sh +``` + +## Running ServerlessLLM Locally + +These steps describe how to run ServerlessLLM on your local machine. + +### 1. Start a Local Ray Cluster + +First, initiate a local Ray cluster. This cluster will consist of one head node and one worker node (on the same machine). + +**Start the head node:** + +Open a new terminal and run: + +```bash +conda activate sllm +ray start --head --port=6379 --num-cpus=4 --num-gpus=0 \ + --resources='{"control_node": 1}' --block +``` + +**Start the worker node:** + +Open another new terminal and run: + +```bash +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 # Or your desired GPU ID +ray start --address=0.0.0.0:6379 --num-cpus=4 --num-gpus=1 \ + --resources='{"worker_node": 1, "worker_id_0": 1}' --block +``` + +### 2. Start the ServerlessLLM Store Server + +Next, start the ServerlessLLM Store server. By default, it uses `./models` as the storage path. + +Open a new terminal and run: + +```bash +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 # Or your desired GPU ID +sllm-store start +``` + +Expected output: + +```log +$ sllm-store start +INFO 12-31 17:13:23 cli.py:58] Starting gRPC server +INFO 12-31 17:13:23 server.py:34] StorageServicer: storage_path=./models, mem_pool_size=4294967296, num_thread=4, chunk_size=33554432, registration_required=False +WARNING: Logging before InitGoogleLogging() is written to STDERR +I20241231 17:13:23.947276 2165054 checkpoint_store.cpp:41] Number of GPUs: 1 +I20241231 17:13:23.947299 2165054 checkpoint_store.cpp:43] I/O threads: 4, chunk size: 32MB +I20241231 17:13:23.947309 2165054 checkpoint_store.cpp:45] Storage path: "./models" +I20241231 17:13:24.038651 2165054 checkpoint_store.cpp:71] GPU 0 UUID: c9938b31-33b0-e02f-24c5-88bd6fbe19ad +I20241231 17:13:24.038700 2165054 pinned_memory_pool.cpp:29] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes +I20241231 17:13:25.557906 2165054 checkpoint_store.cpp:83] Memory pool created with 4GB +INFO 12-31 17:13:25 server.py:243] Starting gRPC server on 0.0.0.0:8073 +``` + +### 3. Start ServerlessLLM + +Now, start the ServerlessLLM service process using `sllm start`. + + +Open a new terminal and run: + +```bash +sllm start +``` + +At this point, you should have four terminals open: one for the Ray head node, one for the Ray worker node, one for the ServerlessLLM Store server, and one for the ServerlessLLM service (started via `sllm start`). + +### 4. Deploy a Model + +With all services running, you can deploy a model. + +Open a new terminal and run: + +```bash +conda activate sllm +sllm deploy --model facebook/opt-1.3b +``` + +This command downloads the specified model from Hugging Face Hub. To load a model from a local path, you can use a `config.json` file. Refer to the [CLI API documentation](../api/cli.md#example-configuration-file-configjson) for details. + +### 5. Query the Model + +Once the model is deployed, you can query it using any OpenAI API-compatible client. For example, use the following `curl` command: + +```bash +curl http://127.0.0.1:8343/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +Expected output: + +```json +{"id":"chatcmpl-9f812a40-6b96-4ef9-8584-0b8149892cb9","object":"chat.completion","created":1720021153,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} +``` + +## Clean Up + +To delete a deployed model, use the following command: + +```bash +sllm delete facebook/opt-1.3b +``` + +This command removes the specified model from the ServerlessLLM server. \ No newline at end of file diff --git a/docs/deployment/slurm_cluster.md b/docs/deployment/slurm_cluster.md new file mode 100644 index 0000000..5d3a476 --- /dev/null +++ b/docs/deployment/slurm_cluster.md @@ -0,0 +1,411 @@ +--- +sidebar_position: 3 +--- + +# SLURM cluster + +This guide will help you get started with running ServerlessLLM on SLURM cluster. It provides two deployment methods, based on `sbatch` and `srun`. If you are in development, we recommend using `srun`, as it is easier to debug than `sbatch`, and if you are in production mode, `sbatch` is recommended. Please make sure you have installed the ServerlessLLM following the [installation guide](./single_machine.md#installation) on all machines. + +## Pre-requisites +Before you begin, make sure you have checked the following: +### Some Tips about Installation +- If 'not enough disk space' is reported when `pip install` on the login node, you can submit it to a job node for execution + ```shell + #!/bin/bash + #SBATCH --partition=Teach-Standard + #SBATCH --job-name=ray-head + #SBATCH --output=sllm_pip.out + #SBATCH --error=sllm_pip.err + #SBATCH --nodes=1 + #SBATCH --ntasks=1 + #SBATCH --cpus-per-task=4 + #SBATCH --gpus-per-task=0 + + # Identify which conda you are using, here is an example that conda is in /opt/conda + source /opt/conda/bin/activate + + conda create -n sllm python=3.10 -y + conda activate sllm + pip install serverless-llm + pip install serverless-llm-store + + conda deactivate sllm + + conda create -n sllm-worker python=3.10 -y + conda activate sllm-worker + pip install serverless-llm[worker] + pip install serverless-llm-store + ``` + +### Command for Querying GPU Resource Information +Run the following commands in the cluster to check GPU resource information. +```shell +sinfo -O partition,nodelist,gres +``` +**Expected Output** +```shell +PARTITION NODELIST GRES +Partition1 JobNode[01,03] gpu:gtx_1060:8 +Partition2 JobNode[04-17] gpu:a6000:2,gpu:gtx_ +``` + +### Identify an idle node +Use `sinfo -p ` to identify some idle nodes + +**Expected Output** +```shell +$ sinfo -p compute +PARTITION AVAIL NODES STATE TIMELIMIT NODELIST +compute up 10 idle infinite JobNode[01-10] +compute up 5 alloc infinite JobNode[11-15] +compute up 2 down infinite JobNode[16-17] +``` + +### Job Nodes Setup +**`srun` Node Selection** + +Only one JobNode is enough. + +**`sbatch` Node Selection** +Let's start a head on the main job node (`JobNode01`) and add the worker on other job node (`JobNode02`). The head and the worker should be on different job nodes to avoid resource contention. The `sllm-store` should be started on the job node that runs worker (`JobNode02`), for passing the model weights, and the `sllm start` should be started on the main job node (`JobNode01`), finally you can use `sllm` to manage the models on the login node. + + +Note: `JobNode02` requires GPU, but `JobNode01` does not. +- **Head**: JobNode01 +- **Worker**: JobNode02 +- **sllm-store**: JobNode02 +- **sllm-serve**: JobNode01 +- **sllm**: Login Node + +--- +## SRUN +If you are in development, we recommend using `srun` to start ServerlessLLM, as it is easier to debug than `sbatch` +### Step 1: Use `srun` enter the JobNode +To start an interactive session on the specified compute node (JobNode), use: +``` +srun --partition --nodelist --gres :1 --pty bash +``` +This command requests a session on the specified node and provides an interactive shell. `--gres :1` specifies the GPU device you will use, for example: `--gres gpu:gtx_1060:1` + +### Step 2: Install ServerlessLLM +Firstly, please make sure CUDA driver available on the node. Here are some commands to check it. +```shell +nvidia-smi + +which nvcc +``` +If `nvidia-smi` has listed GPU information, but `which nvcc` has no output. Then use the following commands to load `nvcc`. Here is an example that cuda is located at `/opt/cuda-12.2.0` +```shell +export PATH=/opt/cuda-12.2.0/bin:$PATH +export LD_LIBRARY_PATH=/opt/cuda-12.2.0/lib64:$LD_LIBRARY_PATH +``` +Then, following the [installation guide](./single_machine.md#installation) to install ServerlessLLM. +### Step 3: Prepare multiple windows with `tmux` +Since srun provides a single interactive shell, you can use tmux to create multiple windows. Start a tmux session: +```shell +tmux +``` +This creates a new tmux session + +**Create multiple windows** +- Use `Ctrl+B` → `C` to start a new window +- Repeat the shortcut 4 more times to create a total of 5 windows. + +**What if `Ctrl+B` does not work?** + +If `Ctrl + B` is unresponsive, reset tmux key bindings: +```shell +tmux unbind C-b +tmux set-option -g prefix C-b +tmux bind C-b send-prefix +``` + +**Command to switch windows** + +Once multiple windows are created, you can switch between them using: + +`Ctrl + B` → `N` (Next window) +`Ctrl + B` → `P` (Previous window) +`Ctrl + B` → `W` (List all windows and select) +`Ctrl + B` → [Number] (Switch to a specific window, e.g., Ctrl + B → 1) + +### Step 4: Run ServerlessLLM on the JobNode +First find ports that are already occupied. Then pick your favourite number from the remaining ports to replace the following placeholder ``. For example: `6379` + +It should also be said that certain slurm system is a bit slow, **so please be patient and wait for the system to output**. + +In the first window, start a local ray cluster with 1 head node and 1 worker node: +```shell +source /opt/conda/bin/activate +conda activate sllm +ray start --head --port= --num-cpus=4 --num-gpus=0 --resources='{"control_node": 1}' --block +``` +In the second window, start the worker node: +```shell +source /opt/conda/bin/activate +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 +ray start --address=0.0.0.0: --num-cpus=4 --num-gpus=1 --resources='{"worker_node": 1, "worker_id_0": 1}' --block +``` +In the third window, start ServerlessLLM Store server: +```shell +source /opt/conda/bin/activate +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 +sllm-store start +``` +In the 4th window, start ServerlessLLM Serve: +```shell +source /opt/conda/bin/activate +conda activate sllm +sllm-serve start +``` +Everything is set! + + +In the 5th window, let's deploy a model to the ServerlessLLM server. You can deploy a model by running the following command: +```shell +source /opt/conda/bin/activate +conda activate sllm +sllm deploy --model facebook/opt-1.3b --backend transformers +``` +This will download the model from HuggingFace transformers. After deploying, you can query the model by any OpenAI API client. For example, you can use the following Python code to query the model: +```shell +curl http://127.0.0.1:8343/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` +Expected output: +```shell +{"id":"chatcmpl-9f812a40-6b96-4ef9-8584-0b8149892cb9","object":"chat.completion","created":1720021153,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} +``` + +### Step 5: Clean up +To delete a deployed model, use the following command: +```shell +sllm delete facebook/opt-1.3b +``` +This will remove the specified model from the ServerlessLLM server. + +In each window, use `Ctrl + c` to stop server and `exit` to exit current `tmux` session. + +--- +## SBATCH +### Step 1: Start the Head Node +Since the head node does not require a gpu, you can find a low-computing capacity node to deploy the head node. +1. **Activate the `sllm` environment and start the head node:** + + Here is the example script, named `start_head_node.sh`. + ```shell + #!/bin/bash + #SBATCH --partition=your-partition # Specify the partition + #SBATCH --nodelist=JobNode01 # Specify an idle node + #SBATCH --job-name=ray-head + #SBATCH --output=sllm_head.out + #SBATCH --error=sllm_head.err + #SBATCH --nodes=1 + #SBATCH --ntasks=1 + #SBATCH --cpus-per-task=12 + #SBATCH --gpus-per-task=0 + + cd /path/to/ServerlessLLM + + source /opt/conda/bin/activate # make sure conda will be loaded correctly + conda activate sllm + + ray start --head --port=6379 --num-cpus=12 --num-gpus=0 --resources='{"control_node": 1}' --block + ``` + - Replace `your-partition`, `JobNode01` and `/path/to/ServerlessLLM` + +2. **Submit the script** + + Use ```sbatch start_head_node.sh``` to submit the script to certain idle node. + +3. **Expected output** + + In `sllm_head.out`, you will see the following output: + + ```shell + Local node IP: + -------------------- + Ray runtime started. + -------------------- + ``` + **Remember the IP address**, denoted ``````, you will need it in following steps. + +4. **Find an available port for serve** + - Some HPCs have a firewall that blocks port 8343. You can use `nc -zv 8343` to check if the port is accessible. + - If it is not accessible, find an available port and replace `available_port` in the following script. + - Here is an example script, named `find_port.sh` + + ```shell + #!/bin/bash + #SBATCH --partition=your-partition + #SBATCH --nodelist=JobNode01 + #SBATCH --job-name=find_port + #SBATCH --output=find_port.log + #SBATCH --time=00:05:00 + #SBATCH --mem=1G + + echo "Finding available port on $(hostname)" + + python -c " + import socket + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) + print(f'Available port: {s.getsockname()[1]}') + " + ``` + Use `sbatch find_port.sh` to submit the script to JobNode01, and in `find_port.log`, you will see the following output: + ``` + Finding available port on JobNode01 + Available port: + ``` + Remember this ``, you will use it in Step 4 + +### Step 2: Start the Worker Node & Store +We will start the worker node and store in the same script. Because the server loads the model weights onto the GPU and uses shared GPU memory to pass the pointer to the client. If you submit another script with ```#SBATCH --gpres=gpu:1```, it will be possibly set to use a different GPU, as specified by different ```CUDA_VISIBLE_DEVICES``` settings. Thus, they cannot pass the model weights. +1. **Activate the ```sllm-worker``` environment and start the worker node.** + + Here is the example script, named```start_worker_node.sh```. + ```shell + #!/bin/sh + #SBATCH --partition=your_partition + #SBATCH --nodelist=JobNode02 + #SBATCH --gres=gpu:a6000:1 # Specify device on JobNode02 + #SBATCH --job-name=sllm-worker-store + #SBATCH --output=sllm_worker.out + #SBATCH --error=sllm_worker.err + #SBATCH --gres=gpu:1 # Request 1 GPU + #SBATCH --cpus-per-task=4 # Request 4 CPU cores + #SBATCH --mem=16G # Request 16GB of RAM + + cd /path/to/ServerlessLLM + + conda activate sllm-worker + + HEAD_NODE_IP= + + export CUDA_HOME=/opt/cuda-12.5.0 # replace with your CUDA path + export PATH=$CUDA_HOME/bin:$PATH + export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH + + ray start --address=$HEAD_NODE_IP:6379 --num-cpus=4 --num-gpus=1 \ + --resources='{"worker_node": 1, "worker_id_0": 1}' --block & + + sllm-store start & + + wait + ``` + - Read the HPC's documentation to find out which partition you can use. Replace ```your_partition``` in the script with that partition name. + - Replace ```/path/to/ServerlessLLM``` with the path to the ServerlessLLM installation directory. + - Replace `````` with the IP address of the head node. + - Replace ```/opt/cuda-12.5.0``` with the path to your CUDA path. + +2. **Find the CUDA path** + - Some slurm-based HPCs have a module system, you can use ```module avail cuda``` to find the CUDA module. + - If it does not work, read the HPC's documentation carefully to find the CUDA path. For example, my doc said CUDA is in ```\opt```. Then you can use ```srun``` command to start an interactive session on the node, such as ```srun --pty -t 00:30:00 -p your_partition --gres=gpu:1 /bin/bash```. A pseudo-terminal will be started for you to find the path. + - Find it and replace ```/opt/cuda-12.5.0``` with the path to your CUDA path. +3. **Submit the script on the other node** + + Use ```sbatch start_worker_node.sh``` to submit the script to certain idle node (here we assume it is ```JobNode02```). In addition, We recommend that you place the head and worker on different nodes so that the Serve can start smoothly later, rather than queuing up for resource allocation. +4. **Expected output** + + In ```sllm_worker.out```, you will see the following output: + + - The worker node expected output: + ```shell + Local node IP: xxx.xxx.xx.xx + -------------------- + Ray runtime started. + -------------------- + ``` + - The store expected output: + ```shell + I20241030 11:52:54.719007 1321560 checkpoint_store.cpp:41] Number of GPUs: 1 + I20241030 11:52:54.773468 1321560 checkpoint_store.cpp:43] I/O threads: 4, chunk size: 32MB + I20241030 11:52:54.773548 1321560 checkpoint_store.cpp:45] Storage path: "./models/" + I20241030 11:52:55.060559 1321560 checkpoint_store.cpp:71] GPU 0 UUID: 52b01995-4fa9-c8c3-a2f2-a1fda7e46cb2 + I20241030 11:52:55.060798 1321560 pinned_memory_pool.cpp:29] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes + I20241030 11:52:57.258795 1321560 checkpoint_store.cpp:83] Memory pool created with 4GB + I20241030 11:52:57.262835 1321560 server.cpp:306] Server listening on 0.0.0.0:8073 + ``` +### Step 3: Start the Serve on the Head Node +1. **Activate the ```sllm``` environment and start the serve.** + + Here is the example script, named```start_serve.sh```. + ```shell + #!/bin/sh + #SBATCH --partition=your_partition + #SBATCH --nodelist=JobNode01 # This node should be the same as head + #SBATCH --output=serve.log + + cd /path/to/ServerlessLLM + + conda activate sllm + + sllm start --host + # sllm start --host --port # if you have changed the port + ``` + - Replace `your_partition` in the script as before. + - Replace `/path/to/ServerlessLLM` as before. + - Replace `` you have found in Step 1 (if port 8343 is not available). +2. **Submit the script on the head node** + + Use ```sbatch start_serve.sh``` to submit the script to the head node (```JobNode01```). + +3. **Expected output** + ```shell + -- Connecting to existing Ray cluster at address: xxx.xxx.xx.xx:6379... + -- Connected to Ray cluster. + INFO: Started server process [1339357] + INFO: Waiting for application startup. + INFO: Application startup complete. + INFO: Uvicorn running on http://xxx.xxx.xx.xx:8343 (Press CTRL+C to quit) + ``` +### Step 4: Use sllm to manage models +1. **You can do this step on login node, and set the ```LLM_SERVER_URL``` environment variable:** + ```shell + $ conda activate sllm + (sllm)$ export LLM_SERVER_URL=http://:8343 + ``` + - Replace `` with the actual IP address of the head node. + - Replace ```8343``` with the actual port number (`` in Step1) if you have changed it. +2. **Deploy a Model Using ```sllm```** + ```shell + (sllm)$ sllm deploy --model facebook/opt-1.3b + ``` +### Step 5: Query the Model Using OpenAI API Client + **You can use the following command to query the model:** + ```shell + curl $LLM_SERVER_URL/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' + ``` + - Replace `````` with the actual IP address of the head node. + - Replace ```8343``` with the actual port number (`` in Step 1) if you have changed it. +### Step 6: Stop Jobs +On the SLURM cluster, we usually use the ```scancel``` command to stop the job. Firstly, list all jobs you have submitted (replace ```your_username``` with your username): +```shell +$ squeue -u your_username +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 1234 compute sllm-head your_username R 0:01 1 JobNode01 + 1235 compute sllm-worker-store your_username R 0:01 1 JobNode02 + 1236 compute sllm-serve your_username R 0:01 1 JobNode01 +``` +Then, use ```scancel``` to stop the job (```1234```, ```1235``` and ```1236``` are JOBIDs): +```shell +$ scancel 1234 1235 1236 +``` diff --git a/docs/developer/_category_.json b/docs/developer/_category_.json new file mode 100644 index 0000000..89a7abc --- /dev/null +++ b/docs/developer/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Developer Guide", + "position": 6 +} diff --git a/docs/developer/supporting_a_new_hardware.md b/docs/developer/supporting_a_new_hardware.md new file mode 100644 index 0000000..2dc4f0a --- /dev/null +++ b/docs/developer/supporting_a_new_hardware.md @@ -0,0 +1,25 @@ +--- +sidebar_position: 0 +--- + +# Supporting a New Hardware + +ServerlessLLM actively expands support for new hardware configurations to meet diverse deployment needs. + +## Support Standards +Hardware is considered supported by ServerlessLLM if: +1. Any of the inference backends used (e.g., Transformers, vLLM) can run model inference on the hardware. +2. ServerlessLLM Store can successfully load model checkpoints on the hardware. + +## Steps to Support a New Hardware +1. **Check Inference Backend Compatibility**: Refer to the specific inference backend documentation (e.g., for vLLM, Transformers) for hardware support. +2. **ServerlessLLM Store Configuration**: + - If the hardware provides CUDA-compatible APIs (e.g., ROCm), adjust the build script (`CMakeLists.txt`) by adding necessary compiler flags. + - For non-CUDA-compatible APIs, implementing a custom checkpoint loading function might be required. + +## Verifying Hardware Support in ServerlessLLM Store +The hardware support is verified if it successfully completes the [Quick Start Guide](https://serverlessllm.github.io/docs/getting_started/) examples, showcasing checkpoint loading and inference functionality without errors. + +If the hardware is not publicly available (i.e., can't be tested by the ServerlessLLM team), a screenshot or output log of the successful execution of the Quick Start Guide examples is required to verify hardware support. + +If you encounter any issues or have questions, please reach out to the ServerlessLLM team by raising an issue on the [GitHub repository](https://github.com/ServerlessLLM/ServerlessLLM/issues). \ No newline at end of file diff --git a/docs/features/_category_.json b/docs/features/_category_.json new file mode 100644 index 0000000..56e0bf0 --- /dev/null +++ b/docs/features/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Features", + "position": 2 +} \ No newline at end of file diff --git a/docs/features/live_migration.md b/docs/features/live_migration.md new file mode 100644 index 0000000..2db4b1d --- /dev/null +++ b/docs/features/live_migration.md @@ -0,0 +1,211 @@ +--- +sidebar_position: 1 +--- + +# Live Migration of Inference Instances + +This example illustrates the live migration of inference instances in a ServerlessLLM cluster by constructing a scenario where two models are deployed to the cluster. Model `Qwen2.5-3B` is stored on both nodes, while model `Qwen2.5-1.5B` is only stored on node 0 (e.g., due to being less popular). This example will show a locality-contention scenario where `Qwen2.5-3B` is being served on node 0 but `Qwen2.5-1.5B` is requested to be served on the same node for optimal locality. We will find that: + +- **Without migration**, `Qwen2.5-1.5B` would have to wait for the completion of the ongoing inference instance of `Qwen2.5-3B` on node 0. +- **With live migration**, the ongoing inference instance of `Qwen2.5-3B` is migrated to node 1, and `Qwen2.5-1.5B` is allocated to node 0, thus can be served immediately. + +## Prerequisites + +To run this example, we will use Docker Compose to set up a ServerlessLLM cluster. Before proceeding, please ensure you have read the [Quickstart Guide](../getting_started.md). + +**Requirements:** + +- **Two GPUs** are required to illustrate the live migration of inference instances. +- **At least 20 GB of host memory** (this can be adjusted by using smaller models). +- **ServerlessLLM version 0.6**: Ensure you have `sllm==0.6` and `sllm-store==0.6` installed. + +## Usage + +Start a local Docker-based ray cluster using Docker Compose. + +### Clone the ServerlessLLM Repository + +If you haven't already, clone the ServerlessLLM repository: + +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/examples/live_migration +``` + +### Configure the Model Directory + +Create a directory on your host machine where models will be stored, and set the MODEL_FOLDER environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +The Docker Compose configuration is already located in the `examples/live_migration` directory. + +## Test ServerlessLLM Without Live Migration + +1. **Start the ServerlessLLM Services Using Docker Compose** + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +2. **Deploy Models with the Placement Spec Files** + +Activate the ServerlessLLM environment and set the server URL: +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` + +Deploy the models: +```bash +sllm deploy --config config-qwen-1.5b.json +sllm deploy --config config-qwen-3b.json +``` + +3. **Verify the Deployment** + +Start two inference requests in parallel. The first request is for `Qwen2.5-3B`, and the second request, sent shortly after, is for `Qwen2.5-1.5B`. The `sleep` command is used to introduce a short interval between the two requests: + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-3B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Could you share a story of the history of Computer Science?"} + ], + "max_tokens": 1024 + }' & + +sleep 3 + +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-1.5B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ], + "max_tokens": 64 + }' +``` + +Since `Qwen2.5-3B` is requested first, `Qwen2.5-1.5B` must wait for the ongoing inference instance of `Qwen2.5-3B` to complete on node 0 before it can start processing. + + +4. Clean up. + +```bash +docker compose down +``` + +## Test ServerlessLLM With Live Migration + +1. **Start the ServerlessLLM Services with Live Migration Enabled** + +Use the following command to start the ServerlessLLM services with live migration enabled. This configuration includes the `enable-migration.yml` file: + +```bash +docker compose -f docker-compose.yml -f enable-migration.yml up -d +``` + +This command will start the Ray head node and two worker nodes, enabling the live migration feature. + +2. **Deploy Models with the Placement Spec Files** + +Activate the ServerlessLLM environment and set the server URL: + +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` + +Deploy the models: + +```bash +sllm deploy --config config-qwen-1.5b.json +sllm deploy --config config-qwen-3b.json +``` + +3. **Verify the Deployment** + +Start two inference requests in parallel. The first request is for `Qwen2.5-3B`, and the second request, sent shortly after, is for `Qwen2.5-1.5B`. The `sleep` command is used to introduce a short interval between the two requests: + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-3B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Could you share a story of the history of Computer Science?"} + ], + "max_tokens": 1024 + }' & + +sleep 3 + +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-1.5B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ], + "max_tokens": 64 + }' +``` + +According to the response, you should observe that `Qwen2.5-1.5B` completes ahead of `Qwen2.5-3B`. This is because the ongoing inference instance of `Qwen2.5-3B` is live-migrated from node 0 to node 1, allowing `Qwen2.5-1.5B` to be served immediately on node 0. + +As shown in the log message, the ongoing inference instance of the model `Qwen/Qwen2.5-3B-Instruct` is live-migrated from node 0 to node 1. And model `Qwen/Qwen2.5-1.5B-Instruct` is allocated to node 0. + +```bash +(MigrationRouter pid=1724) INFO 12-10 22:05:02 migration_router.py:106] Executing migration plan: MigrationPlan(target_node_id='1', source_instance=InstanceStatus(instance_id='Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab', node_id='0', num_gpu=1, concurrency=0, model_name='Qwen/Qwen2.5-3B-Instruct', num_current_tokens=0)) +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:164] Initialized backend for instance Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 for model Qwen/Qwen2.5-3B-Instruct +# Start multi-round live migration +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:178] Migration iteration 0 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:183] Number of tokens: 353, delta: 353 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:198] Migration iteration 0 completed +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:178] Migration iteration 1 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:183] Number of tokens: 14, delta: 14 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:188] Migration completed: remained 14 tokens +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:201] Migrated instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab to Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 +# Finish multi-round live migration +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:215] Instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab removed +(MigrationRouter pid=1724) DEBUG 12-10 22:05:13 migration_router.py:77] Preempted request: ... +# Resume the instance on target node +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:83] Resuming request on target instance: Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 +# Qwen/Qwen2.5-1.5B is allocated to node 0 +(StoreManager pid=1459) INFO 12-10 22:05:14 store_manager.py:344] Loading Qwen/Qwen2.5-1.5B-Instruct to node 0 +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:92] Deallocating model Qwen/Qwen2.5-3B-Instruct instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:103] Node 0 deallocated 1 GPUs +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:108] Model Qwen/Qwen2.5-3B-Instruct instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab deallocated +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 storage_aware_scheduler.py:188] Migrated instance Qwen/Qwen2.5-3B-Instruct to node 1 instance Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 storage_aware_scheduler.py:195] Allocated node 0 for model Qwen/Qwen2.5-1.5B-Instruct +``` + +4. Clean up. + +```bash +docker compose down +``` + + diff --git a/docs/features/peft_lora_fine_tuning.md b/docs/features/peft_lora_fine_tuning.md new file mode 100644 index 0000000..af07fdb --- /dev/null +++ b/docs/features/peft_lora_fine_tuning.md @@ -0,0 +1,328 @@ +--- +sidebar_position: 4 +--- +# PEFT LoRA Fine-tuning + +This feature introduces a dedicated fine-tuning backend (`ft_backend`) for handling LoRA (Low-Rank Adaptation) fine-tuning jobs in ServerlessLLM. This implementation provides isolated fine-tuning instances with specialized resource management and lifecycle control. + +## Prerequisites + +Before using the fine-tuning feature, ensure you have: + +1. **Base Model**: A base model must be saved using the transformers backend +2. **Docker Setup**: ServerlessLLM cluster running via Docker Compose +3. **Storage**: Adequate storage space for fine-tuned adapters + +## Usage + +### Step 1. **Start the ServerlessLLM Services Using Docker Compose** + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +### Step 2: Submit Fine-tuning Job + +Submit a fine-tuning job using the REST API: + +```bash +curl -X POST $LLM_SERVER_URL/v1/fine-tuning/jobs \ + -H "Content-Type: application/json" \ + -d @examples/fine_tuning/fine_tuning_config.json +``` + +#### Fine-tuning Configuration + +Create a configuration file (`fine_tuning_config.json`) with the following structure: + +```json +{ + "model": "facebook/opt-125m", + "ft_backend": "peft_lora", + "num_gpus": 1, + "num_cpus": 1, + "timeout": 3600, + "backend_config": { + "output_dir": "facebook/adapters/opt-125m_adapter_test", + "dataset_config": { + "dataset_source": "hf_hub", + "hf_dataset_name": "fka/awesome-chatgpt-prompts", + "tokenization_field": "prompt", + "split": "train", + "data_files": "", + "extension_type": "" + }, + "lora_config": { + "r": 4, + "lora_alpha": 32, + "lora_dropout": 0.05, + "bias": "none", + "task_type": "CAUSAL_LM" + }, + "training_config": { + "auto_find_batch_size": true, + "save_strategy": "no", + "num_train_epochs": 2, + "learning_rate": 0.0001, + "use_cpu": false + } + } +} +``` + +#### Configuration Parameters + +**Job Configuration:** +- `model`: Base model name +- `ft_backend`: Fine-tuning backend type (currently supports "peft_lora") +- `num_cpus`: Number of CPU cores required +- `num_gpus`: Number of GPUs required +- `timeout`: Maximum execution time in seconds + +**Dataset Configuration:** +- `dataset_source`: Source type ("hf_hub" or "local") +- `hf_dataset_name`: HuggingFace dataset name (for hf_hub) +- `data_files`: Local file paths (for local) +- `extension_type`: File extension type (for local) +- `tokenization_field`: Field name for tokenization +- `split`: Dataset split to use +- More dataset config parameters could be found in [huggingface datasets documentation](https://huggingface.co/docs/datasets/en/loading#load) + +**LoRA Configuration:** +- `r`: LoRA rank +- `lora_alpha`: LoRA alpha parameter +- `target_modules`: Target modules for LoRA adaptation +- `lora_dropout`: Dropout rate +- `bias`: Bias handling strategy +- `task_type`: Task type for PEFT +- More LoraConfig parameters could be found in [huggingface documentation](https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraConfig) + +**Training Configuration:** +- `num_train_epochs`: Number of training epochs +- `per_device_train_batch_size`: Batch size per device +- `gradient_accumulation_steps`: Gradient accumulation steps +- `learning_rate`: Learning rate +- `warmup_steps`: Number of warmup steps +- `logging_steps`: Logging frequency +- `save_steps`: Model saving frequency +- `eval_steps`: Evaluation frequency +- More training arguments could be found in [huggingface documentation](https://huggingface.co/docs/transformers/v4.53.3/en/main_classes/trainer#transformers.TrainingArguments) + +### Step 3: Expected Response + +Upon successful job submission, you'll receive a response with the job ID: + +```json +{ + "job_id": "job-123" +} +``` + +### Step 4: Monitor Job Status + +Check the status of your fine-tuning job: + +```bash +curl -X GET "$LLM_SERVER_URL/v1/fine_tuning/jobs/job-123" +``` + +#### Status Response + +```json +{ + "id": "job-123", + "object": "fine_tuning.job", + "status": { + "config": { + "model": "facebook/opt-125m", + "ft_backend": "peft_lora", + "num_gpus": 1, + "num_cpus": 1, + "timeout": 3600, + "backend_config": { + "output_dir": "facebook/adapters/opt-125m_adapter_test", + "dataset_config": { + "dataset_source": "hf_hub", + "hf_dataset_name": "fka/awesome-chatgpt-prompts", + "tokenization_field": "prompt", + "split": "train", + "data_files": "", + "extension_type": "" + }, + "lora_config": { + "r": 4, + "lora_alpha": 32, + "lora_dropout": 0.05, + "bias": "none", + "task_type": "CAUSAL_LM" + }, + "training_config": { + "auto_find_batch_size": true, + "save_strategy": "no", + "num_train_epochs": 2, + "learning_rate": 0.0001, + "use_cpu": false + } + } + }, + "status": "running", + "created_time": "2025-08-26T04:18:11.155785", + "updated_time": "2025-08-26T04:18:11.155791", + "priority": 0 + } +} +``` + +**Possible Status Values:** +- `pending`: Job is waiting for resources +- `running`: Job is currently executing +- `completed`: Job completed successfully +- `failed`: Job failed with an error +- `cancelled`: Job was cancelled by user + +### Step 5: Cancel Job (Optional) + +If needed, you can cancel a running job: + +```bash +curl -X POST "$LLM_SERVER_URL/v1/fine_tuning/jobs/job-123/cancel" +``` + +## Job Management + +### Resource Allocation + +Fine-tuning jobs are allocated resources based on the specified requirements: + +- **CPU**: Number of CPU cores specified in `num_cpus` +- **GPU**: Number of GPUs specified in `num_gpus` +- **Memory**: Automatically managed based on model size and batch size + +### Priority System + +Jobs are processed based on priority and creation time: + +1. **Higher Priority**: Jobs with higher priority values are processed first +2. **FIFO**: Jobs with the same priority are processed in order of creation +3. **Resource Availability**: Jobs wait until sufficient resources are available + +### Timeout Handling + +Jobs have configurable timeout limits: + +- **Default Timeout**: 3600 seconds (1 hour) +- **Configurable**: Set via `timeout` parameter in job configuration +- **Automatic Cleanup**: Jobs are automatically marked as failed if they exceed the timeout + +## Output and Storage + +### LoRA Adapter Storage + +Fine-tuned LoRA adapters are automatically saved to the `output_dir` path you config in the `fine_tuning_config.json`, like: + +``` +{STORAGE_PATH}/transformers/facebook/adapters/opt-125m_adapter_test +``` + +### Adapter Contents + +The saved adapter includes: + +- **LoRA Weights**: Fine-tuned LoRA parameters +- **Configuration**: LoRA configuration file +- **Metadata**: Training metadata and statistics + +## Integration with Serving + +### Using Fine-tuned Adapters + +After successful fine-tuning, the LoRA adapter can be used for inference: + +```bash +# Deploy model with fine-tuned adapter +sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "my_adapter=ft_facebook/opt-125m_adapter" + +# Use the adapter for inference +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "user", "content": "Hello, how are you?"} + ], + "lora_adapter_name": "my_adapter" +}' +``` + +For more details about PEFT LoRA Serving, please see the [documentation](./peft_lora_serving.md) +## Troubleshooting + +### Common Issues + +1. **Job Stuck in Pending**: Check resource availability and job priority +2. **Dataset Loading Failures**: Verify dataset configuration and accessibility +3. **Training Failures**: Check GPU memory and batch size settings +4. **Timeout Errors**: Increase timeout or optimize training configuration + +## API Reference + +### Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/v1/fine-tuning/jobs` | POST | Submit a fine-tuning job | +| `/v1/fine_tuning/jobs/{fine_tuning_job_id}` | GET | Get job status | +| `/v1/fine_tuning/jobs/{fine_tuning_job_id}/cancel` | POST | Cancel a running job | + +### Response Codes + +| Code | Description | +|------|-------------| +| 200 | Success | +| 400 | Bad Request | +| 404 | Job not found | +| 500 | Internal Server Error | + +## Examples + +### Complete Fine-tuning Workflow + +```bash +# 1. Save base model +sllm-store save --model facebook/opt-125m --backend transformers + +# 2. Start the ServerlessLLM cluster with docker compose +cd examples/docker +docker compose up -d --build + +# 3. Submit fine-tuning job +cd .. && cd .. +curl -X POST $LLM_SERVER_URL/v1/fine-tuning/jobs \ + -H "Content-Type: application/json" \ + -d @examples/fine_tuning/fine_tuning_config.json + +# 4. Monitor job status +curl -X GET "$LLM_SERVER_URL/v1/fine_tuning/jobs/job-123" + +# 5. Deploy base model with fine-tuned adapter +sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "my_adapter=ft_facebook/opt-125m_adapter" + +# 5. Use for inference +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [{"role": "user", "content": "Hello"}], + "lora_adapter_name": "my_adapter" +}' +``` diff --git a/docs/features/peft_lora_serving.md b/docs/features/peft_lora_serving.md new file mode 100644 index 0000000..4c7d3a6 --- /dev/null +++ b/docs/features/peft_lora_serving.md @@ -0,0 +1,116 @@ +--- +sidebar_position: 2 +--- +# PEFT LoRA Serving + +This example illustrates the process of deploying and serving a base large language model enhanced with LoRA (Low-Rank Adaptation) adapters in a ServerlessLLM cluster. It demonstrates how to start the cluster, deploy a base model with multiple LoRA adapters, perform inference using different adapters, and update or remove the adapters dynamically. + +## Pre-requisites + +To run this example, we will use Docker Compose to set up a ServerlessLLM cluster. Before proceeding, please ensure you have read the [Quickstart Guide](../getting_started.md). + +We will use the following example base model & LoRA adapters +- Base model: `facebook/opt-125m` +- LoRA adapters: + - `peft-internal-testing/opt-125m-dummy-lora` + - `monsterapi/opt125M_alpaca` + - `edbeeching/opt-125m-lora` + - `Hagatiana/opt-125m-lora` + +## Usage + +Start a local Docker-based ray cluster using Docker Compose. + +### Step 1: Download the Docker Compose File + +Download the `docker-compose.yml` file from the ServerlessLLM repository: +```bash +# Create a directory for the ServerlessLLM Docker setup +mkdir serverless-llm-docker && cd serverless-llm-docker + +# Download the docker-compose.yml file +curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml + +# Alternatively, you can use wget: +# wget https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml +``` + +### Step 2: Configuration + +Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +### Step 3: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +### Step 4: Deploy Models with LoRA Adapters +1. Configuration +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` +2. Deploy models with specified lora adapters. +```bash +sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "demo_lora1=peft-internal-testing/opt-125m-dummy-lora demo_lora2=monsterapi/opt125M_alpaca" +``` +3. Verify the deployment. +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ], + "lora_adapter_name": "demo_lora1" + }' +``` +If no lora adapters specified, the system will use the base model to do inference +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` +### Step 5: Update LoRA Adapters +If you wish to switch to a different set of LoRA adapters, you can still use `sllm deploy` command with updated adapter configurations. ServerlessLLM will automatically reload the new adapters without restarting the backend. +```bash +sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "demo-lora1=edbeeching/opt-125m-lora demo-lora2=Hagatiana/opt-125m-lora" +``` + +### Step 6: Clean Up + +Delete the lora adapters by running the following command (this command will only delete lora adapters, the base model won't be deleted): +```bash +sllm delete facebook/opt-125m --lora-adapters "demo-lora1 demo-lora2" +``` +If you need to stop and remove the containers, you can use the following commands: +```bash +docker compose down +``` \ No newline at end of file diff --git a/docs/features/quantized_models.md b/docs/features/quantized_models.md new file mode 100644 index 0000000..df09751 --- /dev/null +++ b/docs/features/quantized_models.md @@ -0,0 +1,175 @@ +--- +sidebar_position: 3 +--- + +# Quantization + +This example demonstrates the use of quantization within the ServerlessLLM framework to optimize model serving. Quantization is a technique used to reduce the memory footprint and computational requirements of a large language model by representing its weights with lower-precision data types, such as 8-bit integers (int8). This example will showcase how to deploy and serve a quantized model in a ServerlessLLM cluster. + +## Pre-requisites + +We will use Docker Compose to run a ServerlessLLM cluster in this example. Therefore, please make sure you have read the Quickstart Guide before proceeding. + +## Usage +Start a local Docker-based ray cluster using Docker Compose. + +## Step 1: Set up the Environment + +Create a directory for this example and download the `docker-compose.yml` file. + +```bash +mkdir sllm-quantization-example && cd sllm-quantization-example +curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml + +## Step 2: Configuration + +Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +## Step 3: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +## Step 4: Create Quantization and Deployment Configurations + +First, we'll generate a standard Hugging Face BitsAndBytesConfig and save it to a JSON file. Then, we'll create a deployment configuration file with these quantization settings embedded in it. + +1. Generate the Quantization Config + +Create a Python script named `get_config.py` in the current directory with the following content: +```python +# get_config.py +from transformers import BitsAndBytesConfig + +quantization_config = BitsAndBytesConfig(load_in_4bit=True) +quantization_config.to_json_file("quantization_config.json") + +``` + +Run the script to generate `quantization_config.json`: +```bash +python get_config.py +``` + + +2. Create the Deployment Config + +Now, create a file named `quantized_deploy_config.json`. This file tells ServerlessLLM which model to deploy and instructs the backend to use the quantization settings. You should copy the contents of `quantization_config.json` into the `quantization_config` field below. A template can be found in `sllm/cli/default_config.json`. + +```json +{ + "model": "facebook/opt-1.3b", + "backend": "transformers", + "num_gpus": 1, + "auto_scaling_config": { + "metric": "concurrency", + "target": 1, + "min_instances": 0, + "max_instances": 10, + "keep_alive": 0 + }, + "backend_config": { + "pretrained_model_name_or_path": "", + "device_map": "auto", + "torch_dtype": "float16", + "hf_model_class": "AutoModelForCausalLM", + "quantization_config": { + "_load_in_4bit": true, + "_load_in_8bit": false, + "bnb_4bit_compute_dtype": "float32", + "bnb_4bit_quant_storage": "uint8", + "bnb_4bit_quant_type": "fp4", + "bnb_4bit_use_double_quant": false, + "llm_int8_enable_fp32_cpu_offload": false, + "llm_int8_has_fp16_weight": false, + "llm_int8_skip_modules": null, + "llm_int8_threshold": 6.0, + "load_in_4bit": true, + "load_in_8bit": false, + "quant_method": "bitsandbytes" + } + } +} + +``` + +> Note: Quantization currently only supports the "transformers" backend. Support for other backends will come soon. + +## Step 5: Deploy the Quantized Model +With the configuration files in place, deploy the model using the `sllm-cli`. + +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 + +sllm-cli deploy --config quantized_deploy_config.json +``` + +## Step 6: Verify the deployment. +Send an inference to the server to query the model: + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +To verify the model is being loaded in the desired precision, check the logs (`docker logs sllm_head`). You should see that the model is indeed being loaded in `fp4`. + + +```log +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:321] load config takes 0.0030286312103271484 seconds +(RoundRobinRouter pid=481) INFO 07-02 20:01:49 roundrobin_router.py:272] [] +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:331] load model takes 0.2806234359741211 seconds +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:338] device_map: OrderedDict([('', 0)]) +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:345] compute_device_placement takes 0.18753838539123535 seconds +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:376] allocate_cuda_memory takes 0.0020012855529785156 seconds +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 client.py:72] load_into_gpu: transformers/facebook/opt-1.3b, 70b42a05-4faa-4eaf-bb73-512c6453e7fa +(TransformersBackend pid=352, ip=172.18.0.2) INFO 07-02 20:01:49 client.py:113] Model loaded: transformers/facebook/opt-1.3b, 70b42a05-4faa-4eaf-bb73-512c6453e7fa +(TransformersBackend pid=352, ip=172.18.0.2) INFO 07-02 20:01:49 transformers.py:398] restore state_dict takes 0.0007319450378417969 seconds +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:411] using precision: fp4 +(TransformersBackend pid=352, ip=172.18.0.2) INFO 07-02 20:01:50 client.py:117] confirm_model_loaded: transformers/facebook/opt-1.3b, 70b42a05-4faa-4eaf-bb73-512c6453e7fa +``` + +You should receive a successful JSON response from the model. + +## Step 7: Clean Up + +Delete the model deployment by running the following command: + +```bash +sllm-cli delete facebook/opt-1.3b +``` + +If you need to stop and remove the containers, you can use the following commands: + +```bash +docker compose down +``` + + diff --git a/docs/features/storage_aware_scheduling.md b/docs/features/storage_aware_scheduling.md new file mode 100644 index 0000000..81723f9 --- /dev/null +++ b/docs/features/storage_aware_scheduling.md @@ -0,0 +1,123 @@ +--- +sidebar_position: 0 +--- + +# Storage Aware Scheduling with Docker Compose + +## Pre-requisites + +We will use Docker Compose to run a ServerlessLLM cluster in this example. Therefore, please make sure you have read the [Quickstart Guide](../getting_started.md) before proceeding. + +## Usage + +Start a local Docker-based ray cluster using Docker Compose. + +### Step 1: Clone the ServerlessLLM Repository + +If you haven't already, clone the ServerlessLLM repository: + +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/examples/storage_aware_scheduling +``` + +### Step 2: Configuration + +Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +### Step 3: Enable Storage Aware Scheduling in Docker Compose + +The Docker Compose configuration is already located in the `examples/storage_aware_scheduling` directory. To activate storage-aware scheduling, ensure the `docker-compose.yml` file includes the necessary configurations(`sllm_head` service should include the `--enable-storage-aware` command). + +:::tip +Recommend to adjust the number of GPUs and `mem_pool_size` based on the resources available on your machine. +::: + + +### Step 4: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +### Step 5: Deploy Models with Placement Spec + +In the `examples/storage_aware_scheduling` directory, the example configuration files (`config-opt-2.7b.json` and `config-opt-1.3b.json`) are already given. + +> Note: Storage aware scheduling currently only supports the "transformers" backend. Support for other backends will come soon. + +2. Deploy models with the placement spec files. + +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 + +sllm deploy --config config-opt-2.7b.json +sllm deploy --config config-opt-1.3b.json +``` + +3. Verify the deployment. + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-2.7b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' + +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +As shown in the log message, the model "facebook/opt-2.7b" is scheduled on server 0, while the model "facebook/opt-1.3b" is scheduled on server 1. + +```log +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:27 storage_aware_scheduler.py:137] Sorted scheduling options: [('0', 4.583079601378258)] +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:27 storage_aware_scheduler.py:144] Allocated node 0 for model facebook/opt-2.7b +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:38 storage_aware_scheduler.py:137] Sorted scheduling options: [('1', 2.266678696047572)] +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:38 storage_aware_scheduler.py:144] Allocated node 1 for model facebook/opt-1.3b +``` + +### Step 6: Clean Up + +Delete the model deployment by running the following command: + +```bash +sllm delete facebook/opt-1.3b facebook/opt-2.7b +``` + +If you need to stop and remove the containers, you can use the following commands: + +```bash +docker compose down +``` + diff --git a/docs/getting_started.md b/docs/getting_started.md new file mode 100644 index 0000000..f98364d --- /dev/null +++ b/docs/getting_started.md @@ -0,0 +1,136 @@ +--- +sidebar_position: 1 +--- + +# Getting Started + +This guide demonstrates how to quickly set up a local ServerlessLLM cluster using Docker Compose on a single machine. We will initialize a minimal cluster, consisting of a head node and a single worker node. Then, we'll deploy a model using the `sllm` and query the deployment through an OpenAI-compatible API. + +:::note +We strongly recommend using Docker (Compose) to manage your ServerlessLLM cluster, whether you are using ServerlessLLM for testing or development. However, if Docker is not a viable option for you, please refer to the [deploy from scratch guide](./deployment/single_machine.md). +::: + +## Prerequisites + +Before you begin, ensure you have the following installed and configured: + +1. **Docker**: Installed on your system. You can download it from [here](https://docs.docker.com/get-docker/). +2. **ServerlessLLM CLI**: Installed on your system. Install it using `pip install serverless-llm`. +3. **GPUs**: At least one NVIDIA GPU is required. If you have multiple GPUs, you can adjust the `docker-compose.yml` file accordingly. +4. **NVIDIA Docker Toolkit**: This enables Docker to utilize NVIDIA GPUs. Follow the installation guide [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). + +## Start the ServerlessLLM Cluster + +We will use Docker Compose to simplify the ServerlessLLM setup process. + +### Step 1: Download the Docker Compose File + +Download the `docker-compose.yml` file from the ServerlessLLM repository: + +```bash +# Create a directory for the ServerlessLLM Docker setup +mkdir serverless-llm-docker && cd serverless-llm-docker + +# Download the docker-compose.yml file +curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml + +# Alternatively, you can use wget: +# wget https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml +``` + +### Step 2: Configuration + +Create a directory on your host machine to store models. Then, set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you intend to store the models. This directory will be mounted into the Docker containers. + +### Step 3: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and a worker node as defined in the `docker-compose.yml` file. + +Verify that the services are ready: + +```bash +docker logs sllm_head +``` + +Ensure the services are ready before proceeding. You should see output similar to the following: + +```bash +... +(SllmController pid=1435) INFO 05-26 15:40:49 controller.py:68] Starting scheduler +INFO: Started server process [1] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:8343 (Press CTRL+C to quit) +(FcfsScheduler pid=1604) INFO 05-26 15:40:49 fcfs_scheduler.py:54] Starting FCFS scheduler +(FcfsScheduler pid=1604) INFO 05-26 15:40:49 fcfs_scheduler.py:111] Starting control loop +``` + +## Deploy a Model Using sllm + +Set the `LLM_SERVER_URL` environment variable: + +```bash +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` + +Deploy a model to the ServerlessLLM cluster using the `sllm`: + +```bash +sllm deploy --model facebook/opt-1.3b +``` +> Note: This command will take some time to download the model from the Hugging Face Model Hub. +> You can use any model from the [Hugging Face Model Hub](https://huggingface.co/models) by specifying its name in the `--model` argument. + +Expected output: + +```plaintext +INFO 08-01 07:38:12 deploy.py:36] Deploying model facebook/opt-1.3b with default configuration. +INFO 08-01 07:39:00 deploy.py:49] Model registered successfully. +``` + +## Query the Model + +You can now query the model using any OpenAI API client. For example, use the following `curl` command: +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +Expected output: + +```plaintext +{"id":"chatcmpl-8b4773e9-a98b-41db-8163-018ed3dc65e2","object":"chat.completion","created":1720183759,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}}% +``` + +## Clean Up +To delete a deployed model, execute the following command: + +```bash +sllm delete facebook/opt-1.3b +``` + +This command removes the specified model from the ServerlessLLM server. + +To stop the ServerlessLLM services, use the following command: +```bash +docker compose down +``` \ No newline at end of file diff --git a/docs/intro.md b/docs/intro.md new file mode 100644 index 0000000..52cae09 --- /dev/null +++ b/docs/intro.md @@ -0,0 +1,38 @@ +--- +sidebar_position: 0 +--- + +# Serverless LLM + + +ServerlessLLM + +ServerlessLLM is a **fast** and **easy-to-use** serving system designed for **affordable** multi-LLM serving, also known as LLM-as-a-Service. ServerlessLLM is ideal for environments with multiple LLMs that need to be served on limited GPU resources, as it enables efficient dynamic loading of LLMs onto GPUs. By elastically scaling model instances and multiplexing GPUs, ServerlessLLM can significantly reduce costs compared to traditional GPU-dedicated serving systems while still providing low-latency (Time-to-First-Token, TTFT) LLM completions. + +ServerlessLLM now supports NVIDIA and AMD GPUs, including following hardware: +* NVIDIA GPUs: Compute Capability 7.0+ (e.g, V100, A100, RTX A6000, GeForce RTX 3060) +* AMD GPUs: ROCm 6.2.0+ (tested on MI100s and MI200s) + +## Documentation + +### Getting Started + +- [Quickstart](./getting_started.md) +- [Single Machine Deployment (From Scratch)](./deployment/single_machine.md) +- [Multi-machine Deployment](./deployment/multi_machine.md) +- [SLURM Cluster Deployment](./deployment/slurm_cluster.md) + +### Advanced Features + +- [Storage-Aware Scheduler](./features/storage_aware_scheduling.md) +- [Live Migration](./features/live_migration.md) +- [PEFT LoRA Serving](./features/peft_lora_serving.md) + +### ServerlessLLM Store + +- [Quickstart](./store/quickstart.md) +- [ROCm Quickstart](./store/rocm_quickstart.md) + +### ServerlessLLM CLI + +- [ServerlessLLM CLI API](./api/cli.md) diff --git a/docs/models/_category_.json b/docs/models/_category_.json new file mode 100644 index 0000000..67c0cfe --- /dev/null +++ b/docs/models/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Models", + "position": 7 +} diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md new file mode 100644 index 0000000..6077615 --- /dev/null +++ b/docs/models/supported_models.md @@ -0,0 +1,13 @@ +# Supported Models + +ServerlessLLM supports a plethora of language models from [Huggingface (HF) Transformers](https://huggingface.co/models). This page lists the models and model architectures currently supported by ServerlessLLM. + +To test a model, simply add it to the `supported_models.json` inside `/ServerlessLLM/tests/inference_tests` and the Github Actions will automatically test whether not it is supported. + +## Text-only Language Models + +Architecture |Models |Example HF Models |vLLM |Transformers +------------------|--------------|--------------------|-----|------------- +`OPTForCausalLM` |OPT, OPT-IML |`facebook/opt-6.7b` |✅ |✅ + + diff --git a/docs/store/_category_.json b/docs/store/_category_.json new file mode 100644 index 0000000..78b547f --- /dev/null +++ b/docs/store/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "ServerlessLLM Store", + "position": 5 +} diff --git a/docs/store/quantization.md b/docs/store/quantization.md new file mode 100644 index 0000000..3450410 --- /dev/null +++ b/docs/store/quantization.md @@ -0,0 +1,102 @@ +--- +sidebar_position: 2 +--- + +# Quantization + +> Note: Quantization is currently experimental, especially on multi-GPU machines. You may encounter issues when using this feature in multi-GPU environments. + +ServerlessLLM currently supports `bitsandbytes` quantization, which reduces model memory usage by converting weights to lower-precision data types. You can configure this by passing a `BitsAndBytesConfig` object when loading a model. + +Available precisions include: +- `int8` +- `fp4` +- `nf4` + +> Note: CPU offloading and dequantization is not currently supported. + +## 8-bit Quantization (`int8`) + +8-bit quantization halves the memory usage compared to 16-bit precision with minimal impact on model accuracy. It is a robust and recommended starting point for quantization. + +```python +from transformers import AutoModelForCausalLM, BitsAndBytesConfig + +# Configure 8-bit quantization +quantization_config = BitsAndBytesConfig( + load_in_8bit=True +) + +# Load the model with the config +model_8bit = AutoModelForCausalLM.from_pretrained( + "facebook/opt-1.3b", + quantization_config=quantization_config, + device_map="auto", +) +``` + +## 4-bit Quantization (`fp4`) +FP4 (4-bit Floating Point) quantization offers more aggressive memory savings than 8-bit. It is a good option for running very large models on consumer-grade hardware. + +```python +from transformers import AutoModelForCausalLM, BitsAndBytesConfig + +# Configure 4-bit FP4 quantization +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="fp4" +) + +# Load the model with the config +model_fp4 = AutoModelForCausalLM.from_pretrained( + "facebook/opt-1.3b", + quantization_config=quantization_config, + device_map="auto", +) +``` + +## 4-bit Quantization (`nf4`) +NF4 (4-bit NormalFloat) is an advanced data type optimized for models whose weights follow a normal distribution. NF4 is generally the recommended 4-bit option as it often yields better model accuracy compared to FP4. + +```python +import torch +from transformers import AutoModelForCausalLM, BitsAndBytesConfig + +# Configure 4-bit NF4 quantization +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4" +) + +# Load the model with the config +model_nf4 = AutoModelForCausalLM.from_pretrained( + "facebook/opt-1.3b", + quantization_config=quantization_config, + device_map="auto", +) +``` + +## `torch_dtype` (Data Type for Unquantized Layers) +The `torch_dtype` parameter sets the data type for model layers that are not quantized (e.g. `LayerNorm`). Setting this to `torch.float16` or `torch.bfloat16` can further reduce memory usage. If unspecified, these layers default to `torch.float16`. + +```python +import torch +from transformers import AutoModelForCausalLM, BitsAndBytesConfig + +# Configure 4-bit NF4 quantization +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4" +) + +# Load model, casting non-quantized layers to float16 +model_mixed_precision = AutoModelForCausalLM.from_pretrained( + "facebook/opt-1.3b", + quantization_config=quantization_config, + torch_dtype=torch.float16, + device_map="auto", +) +``` + +For further information, consult the [HuggingFace Documentation for BitsAndBytes](https://huggingface.co/docs/transformers/main/en/quantization/bitsandbytes). + diff --git a/docs/store/quickstart.md b/docs/store/quickstart.md new file mode 100644 index 0000000..83492ea --- /dev/null +++ b/docs/store/quickstart.md @@ -0,0 +1,245 @@ +--- +sidebar_position: 0 +--- + +# Quickstart Guide + +ServerlessLLM Store (`sllm-store`) is a Python library that supports fast model checkpoint loading from multi-tier storage (i.e., DRAM, SSD, HDD) into GPUs. + +ServerlessLLM Store provides a model manager and two key functions: +- `save_model`: Convert a HuggingFace model into a loading-optimized format and save it to a local path. +- `load_model`: Load a model into given GPUs. + + +## Requirements +- OS: Ubuntu 20.04 +- Python: 3.10 +- GPU: compute capability 7.0 or higher + +## Installations + +### Create a virtual environment +```bash +conda create -n sllm-store python=3.10 -y +conda activate sllm-store +``` + +### Install with pip +```bash +pip install serverless-llm-store +``` + +### Install from source +1. Clone the repository and enter the `store` directory + +``` bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/sllm_store +``` + +2. Install the package from source + +```bash +rm -rf build +pip install . +``` + +## Usage Examples +:::tip +We highly recommend using a fast storage device (e.g., NVMe SSD) to store the model files for the best experience. +For example, create a directory `models` on the NVMe SSD and link it to the local path. +```bash +mkdir -p /mnt/nvme/models # Replace '/mnt/nvme' with your NVMe SSD path. +ln -s /mnt/nvme/models ./models +``` +::: + +1. Convert a model to ServerlessLLM format and save it to a local path: +```python +from sllm_store.transformers import save_model + +# Load a model from HuggingFace model hub. +import torch +from transformers import AutoModelForCausalLM +model = AutoModelForCausalLM.from_pretrained('facebook/opt-1.3b', torch_dtype=torch.float16) + +# Replace './models' with your local path. +save_model(model, './models/facebook/opt-1.3b') +``` + +2. Launch the checkpoint store server in a separate process: +```bash +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + + + +3. Load model in your project and make inference: +```python +import time +import torch +from sllm_store.transformers import load_model + +# warm up the GPU +num_gpus = torch.cuda.device_count() +for i in range(num_gpus): + torch.ones(1).to(f"cuda:{i}") + torch.cuda.synchronize() + +start = time.time() +model = load_model("facebook/opt-1.3b", device_map="auto", torch_dtype=torch.float16, storage_path="./models/", fully_parallel=True) +# Please note the loading time depends on the model size and the hardware bandwidth. +print(f"Model loading time: {time.time() - start:.2f}s") + +from transformers import AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained('facebook/opt-1.3b') +inputs = tokenizer('Hello, my dog is cute', return_tensors='pt').to("cuda") +outputs = model.generate(**inputs) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +4. Clean up by "Ctrl+C" the server process. + +## Usage with vLLM + +ServerlessLLM integrates with vLLM to provide fast model loading capabilities. Follow these steps to set up and use ServerlessLLM with vLLM. + +### Prerequisites + +Before using ServerlessLLM with vLLM, you need to apply a compatibility patch to your vLLM installation. This patch has been tested with vLLM version `0.9.0.1`. + +### Apply the vLLM Patch + +1. **Check patch status** (optional): + ```bash + ./sllm_store/vllm_patch/check_patch.sh + ``` + +2. **Apply the patch**: + ```bash + ./sllm_store/vllm_patch/patch.sh + ``` + +3. **Remove the patch** (if needed): + ```bash + ./sllm_store/vllm_patch/remove_patch.sh + ``` + +:::note +The patch file is located at `sllm_store/vllm_patch/sllm_load.patch` in the ServerlessLLM repository. +::: + + +Our api aims to be compatible with the `sharded_state` load format in vLLM. Thus, due to the model modifications about the model architecture done by vLLM, the model format for vLLM is **not** the same as we used in transformers. Thus, the `ServerlessLLM format` mentioned in the subsequent sections means the format integrated with vLLM, which is different from the `ServerlessLLM format` used in the previous sections. + +Thus, for fist-time users, you have to load the model from other backends and then converted it to the ServerlessLLM format. + +1. Download the model from HuggingFace and save it in the ServerlessLLM format: +``` bash +python3 examples/sllm_store/save_vllm_model.py --model-name facebook/opt-1.3b --storage-path $PWD/models --tensor-parallel-size 1 + +``` + +You can also transfer the model from the local path compared to download it from network by passing the `--local-model-path` argument. + +After downloading the model, you can launch the checkpoint store server and load the model in vLLM through `sllm` load format. + +2. Launch the checkpoint store server in a separate process: +```bash +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + +3. Load the model in vLLM: +```python +from vllm import LLM, SamplingParams + +import os + +storage_path = os.getenv("STORAGE_PATH", "./models") +model_name = "facebook/opt-1.3b" +model_path = os.path.join(storage_path, model_name) + +llm = LLM( + model=model_path, + load_format="serverless_llm", + dtype="float16" +) + +prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", +] + +sampling_params = SamplingParams(temperature=0.8, top_p=0.95) +outputs = llm.generate(prompts, sampling_params) + +# Print the outputs. +for output in outputs: + prompt = output.prompt + generated_text = output.outputs[0].text + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") +``` + +# Fine-tuning +ServerlessLLM currently supports LoRA fine-tuning using peft through the Hugging Face Transformers PEFT. + +ServerlessLLM Store provides a model manager and two key functions: +- save_lora: Convert an LoRA adapter into a loading-optimized format and save it to a local path. +- load_lora: Load an adapter into loaded model. + +> Note: Fine-tuning is currently experimental, especially on multi-GPU machines. You may encounter issues when using this feature in multi-GPU environments. + +## Usage Examples + +1. Convert an adapter to ServerlessLLM format and save it to a local path: +``` +from sllm_store.transformers import save_lora + +# TODO: Load an adapter from HuggingFace model hub. + + +# Replace './models' with your local path. +save_lora(adapter, './models/facebook/opt-1.3b') +``` + +2. Launch the checkpoint store server in a separate process: +``` +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + +3. Load the adapter on your model and make inference: +``` +import time +import torch +from sllm_store.transformers import load_model, load_lora + +model = load_model("facebook/opt-1.3b", device_map="auto", torch_dtype=torch.float16, storage_path="./models/", fully_parallel=True) + +model = load_lora("facebook/opt-1.3b", adapter_name="demo_lora", adapter_path="ft_facebook/opt-1.3b_adapter1", device_map="auto", torch_dtype=torch.float16, storage_path="./models/") + +# Please note the loading time depends on the base model size and the hardware bandwidth. +print(f"Model loading time: {time.time() - start:.2f}s") + +from transformers import AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained('facebook/opt-1.3b') +inputs = tokenizer('Hello, my dog is cute', return_tensors='pt').to("cuda") +outputs = model.generate(**inputs) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +4. Clean up by `Ctrl+C` the server process. diff --git a/docs/store/rocm_quickstart.md b/docs/store/rocm_quickstart.md new file mode 100644 index 0000000..1707703 --- /dev/null +++ b/docs/store/rocm_quickstart.md @@ -0,0 +1,164 @@ +--- +sidebar_position: 1 +--- + +# ROCm Quick Start + +ServerlessLLM Store (`sllm-store`) currently supports ROCm platform. However, there are no pre-built wheels for ROCm. + +Due to an internal bug in ROCm, serverless-llm-store may face a GPU memory leak in ROCm before version 6.2.0, as noted in [issue](https://github.com/ROCm/HIP/issues/3580). + +1. Clone the repository and enter the `store` directory: + +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/sllm_store +``` +After that, you may either use the Docker image or build the `sllm-store` wheel from source and install it in your environment. + +## Use the Docker image + +We provide a Dockerfile with ROCm support. Currently, it's built on base image `rocm/vllm-dev:base_ROCm-6.3.1_20250528_tuned_20250530` + +2. Build the Docker image: + +``` bash +docker build -t sllm_store_rocm -f Dockerfile.rocm . +``` + +3. Start the Docker container: + +:::tip +If you want to run inference outside the Docker container, you need to pass the port to the host machine. For example, `-p 8073:8073`. You can also get the wheel from the Docker container after starting it via `docker cp sllm_store_server:/app/dist .`. +::: + +``` bash +docker run --name sllm_store_server --rm -it \ + --device /dev/kfd --device /dev/dri \ + --security-opt seccomp=unconfined \ + -v $(pwd)/models:/models \ + sllm_store_rocm +``` + +Expected output: + +``` bash +INFO 06-05 12:59:07 cli.py:76] Starting gRPC server +INFO 06-05 12:59:07 server.py:40] StorageServicer: storage_path=/models, mem_pool_size=4294967296, num_thread=4, chunk_size=33554432, registration_required=False +WARNING: Logging before InitGoogleLogging() is written to STDERR +I20250605 12:59:11.141070 1 checkpoint_store_hip.cpp:42] Number of GPUs: 1 +I20250605 12:59:11.141098 1 checkpoint_store_hip.cpp:44] I/O threads: 4, chunk size: 32MB +I20250605 12:59:11.141103 1 checkpoint_store_hip.cpp:46] Storage path: "/models" +I20250605 12:59:11.141119 1 checkpoint_store_hip.cpp:72] GPU 0 UUID: 61363865-3865-3038-3831-366132376261 +I20250605 12:59:11.519277 1 pinned_memory_pool_hip.cpp:30] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes +I20250605 12:59:12.487957 1 checkpoint_store_hip.cpp:84] Memory pool created with 4GB +INFO 06-05 12:59:12 server.py:231] Starting gRPC server on 0.0.0.0:8073 + +``` + +After starting the Docker container, you can enter the container and run the following command to test the installation. + +``` bash +docker exec -it sllm_store_server /bin/bash +``` + +Try to save and load a transformer model: + +``` bash +python3 examples/save_transformers_model.py --model-name "facebook/opt-1.3b" --storage-path "/models" +python3 examples/load_transformers_model.py --model-name "facebook/opt-1.3b" --storage-path "/models" +``` +Expected output: + +``` bash +DEBUG 06-05 13:01:01 transformers.py:203] load_dict_non_blocking takes 0.0071375370025634766 seconds +DEBUG 06-05 13:01:01 transformers.py:213] load config takes 0.003943443298339844 seconds +DEBUG 06-05 13:01:01 torch.py:137] allocate_cuda_memory takes 0.0012660026550292969 seconds +DEBUG 06-05 13:01:01 client.py:72] load_into_gpu: facebook/opt-1.3b, 93b1932e-4b43-42cb-b82d-7228ef21810b +INFO 06-05 13:01:01 client.py:113] Model loaded: facebook/opt-1.3b, 93b1932e-4b43-42cb-b82d-7228ef21810b +INFO 06-05 13:01:01 torch.py:160] restore state_dict takes 0.0004298686981201172 seconds +DEBUG 06-05 13:01:02 transformers.py:224] load model takes 0.9706132411956787 seconds +INFO 06-05 13:01:02 client.py:117] confirm_model_loaded: facebook/opt-1.3b, 93b1932e-4b43-42cb-b82d-7228ef21810b +INFO 06-05 13:01:06 client.py:125] Model loaded +Model loading time: 5.28s +tokenizer_config.json: 100%|██████████████████████████████| 685/685 [00:00<00:00, 6.68MB/s] +vocab.json: 100%|███████████████████████████████████████| 899k/899k [00:00<00:00, 4.05MB/s] +merges.txt: 100%|███████████████████████████████████████| 456k/456k [00:00<00:00, 3.05MB/s] +special_tokens_map.json: 100%|████████████████████████████| 441/441 [00:00<00:00, 4.10MB/s] +/usr/local/lib/python3.12/dist-packages/torch/nn/modules/linear.py:125: UserWarning: Failed validator: GCN_ARCH_NAME (Triggered internally at /app/pytorch/aten/src/ATen/hip/tunable/Tunable.cpp:366.) + return F.linear(input, self.weight, self.bias) +Hello, my dog is cute and I want to give him a good home. I have a lot of experience with dogs and I +``` + +Try to save and load a model in vLLM: + +``` bash +python3 examples/save_vllm_model.py --model-name "facebook/opt-125m" --storage-path "/models" +python3 examples/load_vllm_model.py --model-name "facebook/opt-125m" --storage-path "/models" +``` +Expected output: + +``` bash +INFO 06-05 13:02:51 [__init__.py:243] Automatically detected platform rocm. +INFO 06-05 13:02:52 [__init__.py:31] Available plugins for group vllm.general_plugins: +INFO 06-05 13:02:52 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver +INFO 06-05 13:02:52 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load. +INFO 06-05 13:03:00 [config.py:793] This model supports multiple tasks: {'reward', 'embed', 'generate', 'classify', 'score'}. Defaulting to 'generate'. +INFO 06-05 13:03:00 [arg_utils.py:1594] rocm is experimental on VLLM_USE_V1=1. Falling back to V0 Engine. +INFO 06-05 13:03:04 [config.py:1910] Disabled the custom all-reduce kernel because it is not supported on current platform. +INFO 06-05 13:03:04 [llm_engine.py:230] Initializing a V0 LLM engine (v0.9.0.1) with config: model='/models/facebook/opt-125m', speculative_config=None, tokenizer='/models/facebook/opt-125m', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config={}, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=2048, download_dir=None, load_format=LoadFormat.SERVERLESS_LLM, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, decoding_config=DecodingConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_backend=''), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None), seed=0, served_model_name=/models/facebook/opt-125m, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=None, chunked_prefill_enabled=False, use_async_output_proc=True, pooler_config=None, compilation_config={"compile_sizes": [], "inductor_compile_config": {"enable_auto_functionalized_v2": false}, "cudagraph_capture_sizes": [256, 248, 240, 232, 224, 216, 208, 200, 192, 184, 176, 168, 160, 152, 144, 136, 128, 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 4, 2, 1], "max_capture_size": 256}, use_cached_outputs=False, +INFO 06-05 13:03:04 [rocm.py:208] None is not supported in AMD GPUs. +INFO 06-05 13:03:04 [rocm.py:209] Using ROCmFlashAttention backend. +INFO 06-05 13:03:05 [parallel_state.py:1064] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0, EP rank 0 +INFO 06-05 13:03:05 [model_runner.py:1170] Starting to load model /models/facebook/opt-125m... +DEBUG 06-05 13:03:05 torch.py:137] allocate_cuda_memory takes 0.0004763603210449219 seconds +DEBUG 06-05 13:03:05 client.py:72] load_into_gpu: facebook/opt-125m/rank_0, e8e7d900-652d-4822-8992-ad22f734b9c8 +INFO 06-05 13:03:05 client.py:113] Model loaded: facebook/opt-125m/rank_0, e8e7d900-652d-4822-8992-ad22f734b9c8 +INFO 06-05 13:03:05 torch.py:160] restore state_dict takes 0.00021338462829589844 seconds +INFO 06-05 13:03:05 client.py:117] confirm_model_loaded: facebook/opt-125m/rank_0, e8e7d900-652d-4822-8992-ad22f734b9c8 +INFO 06-05 13:03:05 client.py:125] Model loaded +INFO 06-05 13:03:05 [model_runner.py:1202] Model loading took 0.2363 GiB and 0.711783 seconds +/app/third_party/vllm/vllm/model_executor/layers/utils.py:80: UserWarning: Failed validator: GCN_ARCH_NAME (Triggered internally at /app/pytorch/aten/src/ATen/hip/tunable/Tunable.cpp:366.) + return torch.nn.functional.linear(x, weight, bias) +INFO 06-05 13:03:17 [worker.py:303] Memory profiling takes 11.68 seconds +INFO 06-05 13:03:17 [worker.py:303] the current vLLM instance can use total_gpu_memory (23.98GiB) x gpu_memory_utilization (0.90) = 21.59GiB +INFO 06-05 13:03:17 [worker.py:303] model weights take 0.24GiB; non_torch_memory takes 0.53GiB; PyTorch activation peak memory takes 0.49GiB; the rest of the memory reserved for KV Cache is 20.33GiB. +INFO 06-05 13:03:17 [executor_base.py:112] # rocm blocks: 37011, # CPU blocks: 7281 +INFO 06-05 13:03:17 [executor_base.py:117] Maximum concurrency for 2048 tokens per request: 289.15x +INFO 06-05 13:03:18 [model_runner.py:1526] Capturing cudagraphs for decoding. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI. If out-of-memory error occurs during cudagraph capture, consider decreasing `gpu_memory_utilization` or switching to eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage. +Capturing CUDA graph shapes: 100%|█████████████████████████| 35/35 [00:09<00:00, 3.55it/s] +INFO 06-05 13:03:28 [model_runner.py:1684] Graph capturing finished in 10 secs, took 0.13 GiB +INFO 06-05 13:03:28 [llm_engine.py:428] init engine (profile, create kv cache, warmup model) took 22.81 seconds +Adding requests: 100%|█████████████████████████████████████| 4/4 [00:00<00:00, 2079.22it/s] +Processed prompts: 100%|█| 4/4 [00:00<00:00, 6.71it/s, est. speed input: 43.59 toks/s, out +Prompt: 'Hello, my name is', Generated text: ' Joel, my dad is my friend and we are in a relationship. I am' +Prompt: 'The president of the United States is', Generated text: ' speaking out against the release of some State Department documents which show the Russians were involved' +Prompt: 'The capital of France is', Generated text: ' a worldwide knowledge center. What better place to learn about the history and culture of' +Prompt: 'The future of AI is', Generated text: " here: it's the future of everything\nIf you want to test your minds" +[rank0]:[W605 13:03:30.532018298 ProcessGroupNCCL.cpp:1476] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/distributed.html#shutdown (function operator()) +``` + +## Build the wheel from source and install + +Currently, `pip install .` does not work with ROCm. We suggest you build `sllm-store` wheel and manually install it in your environment. + + + +If there's a customized PyTorch version installed, you may need to run the following command to modify the `torch` version in `requirements.txt`: + +```bash +python3 using_existing_torch.py +``` + +2. Build the wheel: + +```bash +python setup.py sdist bdist_wheel +``` + +## Known issues + +1. GPU memory leak in ROCm before version 6.2.0. + +This issue is due to an internal bug in ROCm. After the inference instance is completed, the GPU memory is still occupied and not released. For more information, please refer to [issue](https://github.com/ROCm/HIP/issues/3580). + diff --git a/docusaurus.config.js b/docusaurus.config.js index 42fea8b..bbcc7b1 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -46,18 +46,24 @@ const config = { editUrl: 'https://github.com/ServerlessLLM/serverlessllm.github.io/tree/main/', // Versioning configuration - lastVersion: '0.7.0', // Make stable version the default + lastVersion: '0.8.0', // Make stable version the default at /docs/ versions: { current: { label: 'Latest (dev)', - path: 'latest', + path: 'latest', // Accessible at /docs/latest/ banner: 'unreleased', badge: true, }, - '0.7.0': { - label: '0.7.0 (stable)', - path: '/', // Stable version at root path + '0.8.0': { + label: '0.8.0 (stable)', + // Accessible at /docs/ (root) as the lastVersion banner: 'none', + badge: true, + }, + '0.7.0': { + label: '0.7.0', + // Accessible at /docs/0.7.0/ (older version) + banner: 'unmaintained', badge: false, }, }, @@ -103,6 +109,11 @@ const config = { position: 'left', label: 'API', }, + { + type: 'docsVersionDropdown', + position: 'right', + dropdownActiveClassDisabled: true, + }, { href: 'https://github.com/ServerlessLLM/ServerlessLLM', label: 'GitHub', diff --git a/sidebars.js b/sidebars.js index 0f8d482..fc6d275 100644 --- a/sidebars.js +++ b/sidebars.js @@ -14,11 +14,69 @@ /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ const sidebars = { // By default, Docusaurus generates a sidebar from the docs folder structure - // Excludes 'api' directory which has its own sidebar + // tutorialSidebar explicitly excludes 'api' directory which has its own sidebar tutorialSidebar: [ + 'intro', + 'getting_started', { - type: 'autogenerated', - dirName: '.', + type: 'category', + label: 'Features', + items: [ + { + type: 'autogenerated', + dirName: 'features', + }, + ], + }, + { + type: 'category', + label: 'Deployment', + items: [ + { + type: 'autogenerated', + dirName: 'deployment', + }, + ], + }, + { + type: 'category', + label: 'ServerlessLLM Store', + items: [ + { + type: 'autogenerated', + dirName: 'store', + }, + ], + }, + { + type: 'category', + label: 'Developer Guide', + items: [ + { + type: 'autogenerated', + dirName: 'developer', + }, + ], + }, + { + type: 'category', + label: 'Models', + items: [ + { + type: 'autogenerated', + dirName: 'models', + }, + ], + }, + { + type: 'category', + label: 'Community', + items: [ + { + type: 'autogenerated', + dirName: 'community', + }, + ], }, ], apiSidebar: [ diff --git a/src/theme/DocVersionBanner/index.js b/src/theme/DocVersionBanner/index.js new file mode 100644 index 0000000..9d80057 --- /dev/null +++ b/src/theme/DocVersionBanner/index.js @@ -0,0 +1,110 @@ +import React from 'react'; +import clsx from 'clsx'; +import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; +import Link from '@docusaurus/Link'; +import Translate from '@docusaurus/Translate'; +import { + useActivePlugin, + useDocVersionSuggestions, +} from '@docusaurus/plugin-content-docs/client'; +import {ThemeClassNames} from '@docusaurus/theme-common'; +import { + useDocsPreferredVersion, + useDocsVersion, +} from '@docusaurus/theme-common/internal'; +function UnreleasedVersionLabel({siteTitle, versionMetadata}) { + return ( + + You are viewing the latest developer preview docs. + + ); +} +function UnmaintainedVersionLabel({siteTitle, versionMetadata}) { + return ( + + You may be reading an old version of this documentation. + + ); +} +const BannerLabelComponents = { + unreleased: UnreleasedVersionLabel, + unmaintained: UnmaintainedVersionLabel, +}; +function BannerLabel(props) { + const BannerLabelComponent = + BannerLabelComponents[props.versionMetadata.banner]; + return ; +} +function LatestVersionSuggestionLabel({versionLabel, to, onClick, isUnreleased}) { + if (isUnreleased) { + return ( + + + Click here + + {' to view docs for the latest stable release '} + ({versionLabel}). + + ); + } + + // For unmaintained/old versions + return ( + + Read the{' '} + + latest stable version + + {' of this documentation '} + ({versionLabel}). + + ); +} +function DocVersionBannerEnabled({className, versionMetadata}) { + const { + siteConfig: {title: siteTitle}, + } = useDocusaurusContext(); + const {pluginId} = useActivePlugin({failfast: true}); + const getVersionMainDoc = (version) => + version.docs.find((doc) => doc.id === version.mainDocId); + const {savePreferredVersionName} = useDocsPreferredVersion(pluginId); + const {latestDocSuggestion, latestVersionSuggestion} = + useDocVersionSuggestions(pluginId); + // Try to link to same doc in latest version (not always possible), falling + // back to main doc of latest version + const latestVersionSuggestedDoc = + latestDocSuggestion ?? getVersionMainDoc(latestVersionSuggestion); + return ( +
+
+ +
+
+ savePreferredVersionName(latestVersionSuggestion.name)} + isUnreleased={versionMetadata.banner === 'unreleased'} + /> +
+
+ ); +} +export default function DocVersionBanner({className}) { + const versionMetadata = useDocsVersion(); + if (versionMetadata.banner) { + return ( + + ); + } + return null; +} diff --git a/versioned_docs/version-0.7.0/README.md b/versioned_docs/version-0.7.0/README.md new file mode 100644 index 0000000..4a1397a --- /dev/null +++ b/versioned_docs/version-0.7.0/README.md @@ -0,0 +1,39 @@ +# ServerlessLLM documents + +Please find our documents in [ServerlessLLM](https://serverlessllm.github.io/docs/getting_started). + +## How to build ServerlessLLM Docs + +This website is built using Docusaurus, a modern static website generator. + +### Installation + +To install the necessary dependencies, use the following command: + +```bash +npm install +``` + +### Local Development + +To start a local development server and open up a browser window, use the following command: + +```bash +npm run start +``` + +Most changes are reflected live without having to restart the server. + +### Build + +To generate static content into the build directory, use the following command: + +```bash +npm run build +``` + +This command generates static content into the `build` directory, which can be served using any static content hosting service. + +### About the image path + +Images are stored in `images` path. For example, we have an image called `a.jpg` in `images`. When we use this image in any position in the documents, we just use `/img/a.jpg`. (The document sync bot can copy `images` path into `img` folder in `serverlessllm.github.io` repo) diff --git a/versioned_docs/version-0.7.0/api/cli.md b/versioned_docs/version-0.7.0/api/cli.md new file mode 100644 index 0000000..2e9fb1d --- /dev/null +++ b/versioned_docs/version-0.7.0/api/cli.md @@ -0,0 +1,409 @@ +--- +sidebar_position: 2 +--- + +# CLI API + +## ServerlessLLM CLI Documentation + +### Overview +`sllm-cli` is a command-line interface (CLI) tool designed to manage and interact with ServerlessLLM models. This document provides an overview of the available commands and their usage. + +### Installation + +```bash +# Create a new environment +conda create -n sllm python=3.10 -y +conda activate sllm + +# Install ServerlessLLM +pip install serverless-llm +``` + +### Getting Started + +Before using the `sllm-cli` commands, you need to start the ServerlessLLM cluster. Follow the guides below to set up your cluster: + +- [Single Machine Deployment](../getting_started.md) +- [Single Machine Deployment (From Scratch)](../deployment/single_machine.md) +- [Multi-Machine Deployment](../deployment/multi_machine.md) +- [SLURM Cluster Deployment](../deployment/slurm_cluster.md) + +After setting up the ServerlessLLM cluster, you can use the commands listed below to manage and interact with your models. + +### Example Workflow + +1. **Deploy a Model** + > Deploy a model using the model name, which must be a HuggingFace pretrained model name. i.e. `facebook/opt-1.3b` instead of `opt-1.3b`. + ```bash + sllm-cli deploy --model facebook/opt-1.3b + ``` + +2. **Generate Output** + ```bash + echo '{ + "model": "facebook/opt-1.3b", + "messages": [ + { + "role": "user", + "content": "Please introduce yourself." + } + ], + "temperature": 0.7, + "max_tokens": 50 + }' > input.json + sllm-cli generate input.json + ``` + +3. **Delete a Model** + ```bash + sllm-cli delete facebook/opt-1.3b + ``` + +### sllm-cli deploy +Deploy a model using a configuration file or model name, with options to overwrite default configurations. The configuration file requires minimal specifications, as sensible defaults are provided for advanced configuration options. + +This command also supports [PEFT LoRA (Low-Rank Adaptation)](https://huggingface.co/docs/peft/main/en/index), allowing you to deploy adapters on top of a base model, either via CLI flags or directly in the configuration file. + +For more details on the advanced configuration options and their default values, please refer to the [Example Configuration File](#example-configuration-file-configjson) section. + +##### Usage +```bash +sllm-cli deploy [OPTIONS] +``` + +##### Options +- `--model ` + - Model name to deploy with default configuration. The model name must be a Hugging Face pretrained model name. You can find the list of available models [here](https://huggingface.co/models). + +- `--config ` + - Path to the JSON configuration file. The configuration file can be incomplete, and missing sections will be filled in by the default configuration. + +- `--backend ` + - Overwrite the backend in the default configuration. + +- `--num-gpus ` + - Overwrite the number of GPUs in the default configuration. + +- `--target ` + - Overwrite the target concurrency in the default configuration. + +- `--min-instances ` + - Overwrite the minimum instances in the default configuration. + +- `--max-instances ` + - Overwrite the maximum instances in the default configuration. + +- `--enable-lora` + - Enable LoRA adapter support for the transformers backend. Overwrite `enable_lora` in the default configuration. + +- `--lora-adapters` + - Add one or more LoRA adapters in the format `=`. Overwrite any existing `lora_adapters` in the default configuration. + +##### Examples +Deploy using a model name with default configuration: +```bash +sllm-cli deploy --model facebook/opt-1.3b +``` + +Deploy using a configuration file: +```bash +sllm-cli deploy --config /path/to/config.json +``` + +Deploy using a model name and overwrite the backend: +```bash +sllm-cli deploy --model facebook/opt-1.3b --backend transformers +``` + +Deploy using a model name and overwrite multiple configurations: +```bash +sllm-cli deploy --model facebook/opt-1.3b --num-gpus 2 --target 5 --min-instances 1 --max-instances 5 +``` + +Deploy a base model with multiple LoRA adapters: +```bash +sllm-cli deploy --model facebook/opt-1.3b --backend transformers --enable-lora --lora-adapters demo_lora1=crumb/FLAN-OPT-1.3b-LoRA demo_lora2=GrantC/alpaca-opt-1.3b-lora +``` + +##### Example Configuration File (`config.json`) +This file can be incomplete, and missing sections will be filled in by the default configuration: +```json +{ + "model": "facebook/opt-1.3b", + "backend": "transformers", + "num_gpus": 1, + "auto_scaling_config": { + "metric": "concurrency", + "target": 1, + "min_instances": 0, + "max_instances": 10, + "keep_alive": 0 + }, + "backend_config": { + "pretrained_model_name_or_path": "facebook/opt-1.3b", + "device_map": "auto", + "torch_dtype": "float16", + "hf_model_class": "AutoModelForCausalLM", + "enable_lora": true, + "lora_adapters": { + "demo_lora1": "crumb/FLAN-OPT-1.3b-LoRA", + "demo_lora2": "GrantC/alpaca-opt-1.3b-lora" + } + } +} +``` + +Below is a description of all the fields in config.json. + +| Field | Description | +| ----- | ----------- | +| model | This should be a HuggingFace model name, used to identify model instance. | +| backend | Inference engine, support `transformers` and `vllm` now. | +| num_gpus | Number of GPUs used to deploy a model instance. | +| auto_scaling_config | Config about auto scaling. | +| auto_scaling_config.metric | Metric used to decide whether to scale up or down. | +| auto_scaling_config.target | Target value of the metric. | +| auto_scaling_config.min_instances | The minimum value for model instances. | +| auto_scaling_config.max_instances | The maximum value for model instances. | +| auto_scaling_config.keep_alive | How long a model instance lasts after inference ends. For example, if keep_alive is set to 30, it will wait 30 seconds after the inference ends to see if there is another request. | +| backend_config | Config about inference backend. | +| backend_config.pretrained_model_name_or_path | The path to load the model, this can be a HuggingFace model name or a local path. | +| backend_config.device_map | Device map config used to load the model, `auto` is suitable for most scenarios. | +| backend_config.torch_dtype | Torch dtype of the model. | +| backend_config.hf_model_class | HuggingFace model class. | +| backend_config.enable_lora | Set to true to enable loading LoRA adapters during inference. | +| backend_config.lora_adapters| A dictionary of LoRA adapters in the format `{name: path}`, where each path is a local or Hugging Face-hosted LoRA adapter directory. | + +### sllm-cli delete +Delete deployed models by name, or delete specific LoRA adapters associated with a base model. + +This command supports: + - Removing deployed models + - Removing specific LoRA adapters while preserving the base model + +##### Usage +```bash +sllm-cli delete [MODELS] [OPTIONS] +``` + +##### Arguments +- `MODELS` + - Space-separated list of model names to delete. + +##### Options +- `--lora-adapters ` + - Space-separated list of LoRA adapter names to delete from the given model. If provided, the base model will not be deleted — only the specified adapters will be removed. + +##### Example +Delete multiple base models (and all their adapters): +```bash +sllm-cli delete facebook/opt-1.3b facebook/opt-2.7b meta/llama2 +``` +Delete specific LoRA adapters from a base model, keeping the base model: +```bash +sllm-cli delete facebook/opt-1.3b --lora-adapters demo_lora1 demo_lora2 +``` + +### sllm-cli generate +Generate outputs using the deployed model. + +##### Usage +```bash +sllm-cli generate [OPTIONS] +``` + +##### Options +- `-t`, `--threads ` + - Number of parallel generation processes. Default is 1. + +##### Arguments +- `input_path` + - Path to the JSON input file. + +##### Example +```bash +sllm-cli generate --threads 4 /path/to/request.json +``` + +##### Example Request File (`request.json`) +```json +{ + "model": "facebook/opt-1.3b", + "messages": [ + { + "role": "user", + "content": "Please introduce yourself." + } + ], + "temperature": 0.3, + "max_tokens": 50 +} +``` + +### sllm-cli encode (embedding) +Get the embedding using the deployed model. + +##### Usage +```bash +sllm-cli encode [OPTIONS] +``` + +##### Options +- `-t`, `--threads ` + - Number of parallel encoding processes. Default is 1. + +##### Arguments +- `input_path` + - Path to the JSON input file. + +##### Example +```bash +sllm-cli encode --threads 4 /path/to/request.json +``` + +##### Example Request File (`request.json`) +```json +{ + "model": "intfloat/e5-mistral-7b-instruct", + "task_instruct": "Given a question, retrieve passages that answer the question", + "query": [ + "Hi, how are you?" + ] +} +``` + +### sllm-cli replay +Replay requests based on workload and dataset. + +##### Usage +```bash +sllm-cli replay [OPTIONS] +``` + +##### Options +- `--workload ` + - Path to the JSON workload file. + +- `--dataset ` + - Path to the JSON dataset file. + +- `--output ` + - Path to the output JSON file for latency results. Default is `latency_results.json`. + +##### Example +```bash +sllm-cli replay --workload /path/to/workload.json --dataset /path/to/dataset.json --output /path/to/output.json +``` + +#### sllm-cli update +Update a deployed model using a configuration file or model name. + +##### Usage +```bash +sllm-cli update [OPTIONS] +``` + +##### Options +- `--model ` + - Model name to update with default configuration. + +- `--config ` + - Path to the JSON configuration file. + +##### Example +```bash +sllm-cli update --model facebook/opt-1.3b +sllm-cli update --config /path/to/config.json +``` + +### sllm-cli fine-tuning +Fine-tune the deployed model. + +##### Usage +```bash +sllm-cli fine-tuning [OPTIONS] +``` + +##### Options +- `--base-model ` + - Base model name to be fine-tuned +- `--config ` + - Path to the JSON configuration file. + +##### Example +```bash +sllm-cli fine-tuning --base-model +sllm-cli fine-tuning --base-model --config +``` + +##### Example Configuration File (`ft_config.json`) +```json +{ + "model": "facebook/opt-125m", + "ft_backend": "peft", + "dataset_config": { + "dataset_source": "hf_hub", + "hf_dataset_name": "fka/awesome-chatgpt-prompts", + "tokenization_field": "prompt", + "split": "train", + "data_files": "", + "extension_type": "" + }, + "lora_config": { + "r": 4, + "lora_alpha": 1, + "lora_dropout": 0.05, + "bias": "lora_only", + "task_type": "CAUSAL_LM" + }, + "training_config": { + "auto_find_batch_size": true, + "num_train_epochs": 2, + "learning_rate": 0.0001, + "use_cpu": false + } +} +``` + +Below is a description of all the fields in ft_config.json. + +| Field | Description | +|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| model | This should be a deployed model name, used to identify the backend instance. | +| ft_backend | fine-tuning engine, only support `peft` now. | +| dataset_config | Configuration for the fine-tuning dataset | +| dataset_config.dataset_source | dataset is from `hf_hub` (huggingface_hub) or `local` file | +| dataset_config.hf_dataset_name | dataset name on huggingface_hub | +| dataset_config.tokenization_field | the field to tokenize | +| dataset_config.split | Partitioning of the dataset (`train`, `validation` and `test`), You can also split the selected dataset, e.g. take only the top 10% of the training data: train[:10%] | +| dataset_config.data_files | data files will be loaded from local | +| dataset_config.extension_type | extension type of data files (`csv`, `json`, `parquet`, `arrow`) | +| lora_config | Configuration for LoRA fine-tuning | +| lora_config.r | `r` defines how many parameters will be trained. | +| lora_config.lora_alpha | A multiplier controlling the overall strength of connections within a neural network, typically set at 1 | +| lora_config.target_modules | a list of the target_modules available on the [Hugging Face Documentation][1] | +| lora_config.lora_dropout | used to avoid overfitting | +| lora_config.bias | use `none` or `lora_only` | +| lora_config.task_type | Indicates the task the model is begin trained for | +| training_config | Configuration for training parameters | +| training_config.auto_find_batch_size | Find a correct batch size that fits the size of Data. | +| training_config.num_train_epochs | Total number of training rounds | +| training_config.learning_rate | learning rate | +| training_config.optim | select an optimiser | +| training_config.use_cpu | whether to use CPU for training | + +[1]: https://github.com/huggingface/peft/blob/39ef2546d5d9b8f5f8a7016ec10657887a867041/src/peft/utils/other.py#L220 + +### sllm-cli status +Check the information of deployed models + +#### Usage +```bash +sllm-cli status +``` + +#### Example +```bash +sllm-cli status +``` \ No newline at end of file diff --git a/versioned_docs/version-0.7.0/api/intro.md b/versioned_docs/version-0.7.0/api/intro.md new file mode 100644 index 0000000..9753d2d --- /dev/null +++ b/versioned_docs/version-0.7.0/api/intro.md @@ -0,0 +1,9 @@ +--- +sidebar_position: 1 +--- + +# API Introduction + +Welcome to the ServerlessLLM API documentation. This section contains detailed information about the various APIs provided by ServerlessLLM: + +- [CLI API](./cli.md) - Documentation for the `sllm-cli` command-line interface \ No newline at end of file diff --git a/versioned_docs/version-0.7.0/community/_category_.json b/versioned_docs/version-0.7.0/community/_category_.json new file mode 100644 index 0000000..8a81fd9 --- /dev/null +++ b/versioned_docs/version-0.7.0/community/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Community", + "position": 8 +} diff --git a/versioned_docs/version-0.7.0/community/meetups.md b/versioned_docs/version-0.7.0/community/meetups.md new file mode 100644 index 0000000..8e3811a --- /dev/null +++ b/versioned_docs/version-0.7.0/community/meetups.md @@ -0,0 +1,11 @@ +# ServerlessLLM Meetups + +We host regular biweekly developer meetings online. We will share project updates from the ServerlessLLM developer team presented during these meetings. Please find the materials of our previous meetups below: + +Date |Topic |Slides +---------------|-------------|--------- +February 21st 2025 | Fine Tuning | [Slides](https://docs.google.com/presentation/d/1rnw3mieAAbMabDIoIGS-ciMGc3hJ7AICYSaNJp-Fk4s/edit?usp=sharing) +March 7th 2025 |Quantization |[Slides](https://docs.google.com/presentation/d/1uSbP-LzGbbvPsemIAE6jCFsggYm_ATxQguCHDmdwoXE/edit?usp=sharing) + +We are always looking for contributors to join us on the developer team. If you are interested in contributing, consult our [job board](https://github.com/orgs/ServerlessLLM/projects/2) and claim a feature. For any other questions, please contact us on [this email](mailto:Y.Fu@ed.ac.uk) or on [our Discord server](https://discord.gg/AEF8Gduvm8). + diff --git a/versioned_docs/version-0.7.0/community/talks.md b/versioned_docs/version-0.7.0/community/talks.md new file mode 100644 index 0000000..66a2531 --- /dev/null +++ b/versioned_docs/version-0.7.0/community/talks.md @@ -0,0 +1,8 @@ +# ServerlessLLM Talks + +Materials for ServerlessLLM talks will be listed here. + +Topic |Location |Date |Links +-------------|----------------|---------------|------------------------------------ +Efficient Sharing of AI Infrastructures with Specialized Serverless Computing | University of Pennsylvania |January 29th 2025 |[Slides](https://drive.google.com/file/d/17GwXsqaDDS7Xw8nX_-RaKiwpaPQgu9WD/view) \| [Event](https://asset.seas.upenn.edu/event/yao-fu-university-of-edinburgh/) +ServerlessLLM Tutorial | SESAME'25 | March 31st 2025 |[Slides](https://docs.google.com/presentation/d/1ioGCVpsg0x3oCxX19EiE820aMiY22X5MG6jgImZ1W18/edit?usp=sharing) \| [Event](https://sesame25.github.io/) diff --git a/versioned_docs/version-0.7.0/deployment/_category_.json b/versioned_docs/version-0.7.0/deployment/_category_.json new file mode 100644 index 0000000..534be1d --- /dev/null +++ b/versioned_docs/version-0.7.0/deployment/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Deployment", + "position": 3 +} diff --git a/versioned_docs/version-0.7.0/deployment/multi_machine.md b/versioned_docs/version-0.7.0/deployment/multi_machine.md new file mode 100644 index 0000000..4c75f7c --- /dev/null +++ b/versioned_docs/version-0.7.0/deployment/multi_machine.md @@ -0,0 +1,296 @@ +--- +sidebar_position: 2 +--- + +# Multi-machine + +This guide will help you get started with running ServerlessLLM on multiple machines using Docker containers. You'll learn how to set up a head node on one machine and connect worker nodes from different machines using Docker, ensuring proper network communication between the containers. You can extend this setup to use as many nodes as you need. + +## Prerequisites + +This guide requires **two machines**: +- One machine for the head node (no GPU required) +- One machine with an NVIDIA GPU to serve as the worker node + +You can add more worker machines with GPUs as needed to scale out your deployment. + +### For All Machines + +Ensure you have the following installed and configured on all machines (both head node and worker machines): + +1. **Docker**: Installed on your system. You can download it from [here](https://docs.docker.com/get-docker/). +2. **Network connectivity**: Ensure all machines can communicate with each other on the required ports (6379 for Ray, 8343 for ServerlessLLM API, and 8073 for storage service). + +:::tip +The **ServerlessLLM CLI** (`pip install serverless-llm`) can be installed on any machine that needs to manage model deployments. This could be your local computer or any machine within the cluster that can connect to the head node. +::: + +### For Worker Machines Only + +These requirements are only necessary for the worker machines that will run the models: + +1. **GPUs**: At least one NVIDIA GPU is required on each worker machine. If you have multiple GPUs, you can adjust the Docker configuration accordingly. +2. **NVIDIA Docker Toolkit**: This enables Docker to utilize NVIDIA GPUs. Follow the installation guide [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). + +## Multi-Machine Setup + +We'll start a head node on one machine using Docker, then add a worker node from another machine using Docker containers with host networking. + +### Step 1: Start the Head Node + +1. **Start the head node using Docker:** + +```bash +# Get the machine's IP address that will be accessible to other machines +export HEAD_IP=$(hostname -I | awk '{print $1}') +echo "Head node IP address: $HEAD_IP" + +docker run -d \ + --name sllm_head \ + --network host \ + -e MODE=HEAD \ + -e RAY_NODE_IP=$HEAD_IP \ + serverlessllm/sllm:latest +``` + +:::important +For multi-machine setups, setting the `RAY_NODE_IP` is critical. It should be set to an IP address that is accessible from all worker machines. The command above attempts to automatically determine your machine's primary IP, but in complex network environments, you may need to specify it manually. + +If your machine has multiple network interfaces, ensure you use the IP that other machines in your network can access. +::: + +:::tip +If you don't have the ServerlessLLM Docker image locally, Docker will automatically pull it from the registry. You can also adjust the CPU and resource allocations by setting additional environment variables like `RAY_NUM_CPUS` and `RAY_RESOURCES`. +::: + +2. **Verify the head node is running and note the external IP:** + +```bash +docker logs sllm_head +``` + +Expected output should include: + +```bash +> docker logs sllm_head +... +2025-05-29 14:29:46,211 INFO scripts.py:744 -- Local node IP: 129.215.164.107 +... +(SllmController pid=380) INFO 05-29 14:29:53 controller.py:59] Starting store manager +(SllmController pid=380) INFO 05-29 14:29:56 controller.py:68] Starting scheduler +(StoreManager pid=417) INFO 05-29 14:29:56 store_manager.py:226] Initializing store manager +(StoreManager pid=417) INFO 05-29 14:29:56 store_manager.py:237] Initializing cluster and collecting hardware info +(StoreManager pid=417) ERROR 05-29 14:29:56 store_manager.py:242] No worker nodes found +INFO: Started server process [1] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:8343 (Press CTRL+C to quit) +(FcfsScheduler pid=456) INFO 05-29 14:29:56 fcfs_scheduler.py:54] Starting FCFS scheduler +(FcfsScheduler pid=456) INFO 05-29 14:29:56 fcfs_scheduler.py:111] Starting control loop +``` + +Make note of the IP address shown in the logs. This is the address that worker nodes will use to connect to the head node. + +### Step 2: Start Worker Node on a Different Machine + +:::tip +You can adjust the memory pool size and other parameters based on the resources available on your worker machine. +::: + +1. **On the worker machine, create a directory for model storage:** + +```bash +mkdir -p /path/to/your/models +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +2. **Start the worker node:** + +```bash +# Replace with the actual IP address of the head node from the previous step +# DO NOT copy-paste this line directly - update with your actual head node IP +export HEAD_IP= +``` + +```bash +# Get the worker machine's IP address that will be accessible to the head node +export WORKER_IP=$(hostname -I | awk '{print $1}') +echo "Worker node IP address: $WORKER_IP" + +docker run -d \ + --name sllm_worker_0 \ + --network host \ + --gpus '"device=0"' \ + -e WORKER_ID=0 \ + -e STORAGE_PATH=/models \ + -e MODE=WORKER \ + -e RAY_HEAD_ADDRESS=${HEAD_IP}:6379 \ + -e RAY_NODE_IP=$WORKER_IP \ + -v ${MODEL_FOLDER}:/models \ + serverlessllm/sllm:latest \ + --mem-pool-size 4GB --registration-required true +``` + +:::important +For multi-machine setups, setting the `RAY_NODE_IP` on worker nodes is just as critical as on the head node. It should be set to an IP address that is accessible from the head node. Without this, workers might report internal Docker IPs that aren't accessible across machines. + +Make sure to replace `192.168.1.100` with the actual IP address of your head node that you noted earlier. +::: + +3. **Verify worker node is connected:** + +On the worker machine, check if the worker has properly connected to the Ray cluster: + +```bash +docker exec -it sllm_worker_0 bash -c "source /opt/conda/etc/profile.d/conda.sh && conda activate worker && ray status" +``` + +Expected output should include both the head node and worker node resources: + +```bash +> docker exec -it sllm_worker_0 bash -c "source /opt/conda/etc/profile.d/conda.sh && conda activate worker && ray status" +======== Autoscaler status: 2025-05-29 14:42:30.434645 ======== +Node status +--------------------------------------------------------------- +Active: + 1 node_f0a8e97ca64c64cebd551f469a38d0d66ce304f7cc1cc9696fe33cf3 + 1 node_3b7db178afb8bdb16460d0cb6463dc7b9b3afbcc00753c3be110c9b3 +Pending: + (no pending nodes) +Recent failures: + (no failures) + +Resources +--------------------------------------------------------------- +Usage: + 3.0/52.0 CPU + 0.0/1.0 GPU + 0.30000000000000004/1.0 control_node + 0B/526.36GiB memory + 0B/18.63GiB object_store_memory + 0.0/1.0 worker_id_0 + 0.0/1.0 worker_node + +Demands: + (no resource demands) +``` + +This output confirms that both the head node and worker node are properly connected and their resources are recognized by the Ray cluster. + +:::tip +**Adding more worker nodes:** You can add more worker nodes by repeating Step 2 on additional machines with GPUs. Just make sure to: +1. Use a unique `WORKER_ID` for each worker (1, 2, 3, etc.) +2. Point each worker to the same head node IP address +3. Ensure each worker has its own `RAY_NODE_IP` set correctly +::: + +### Step 3: Use `sllm-cli` to manage models + +#### Configure the Environment + +**On any machine with `sllm-cli` installed, set the `LLM_SERVER_URL` environment variable:** + +> Replace `` with the actual IP address of the head node. + +```bash +export LLM_SERVER_URL=http://:8343 +``` + +#### Deploy a Model Using `sllm-cli` + +```bash +sllm-cli deploy --model facebook/opt-1.3b +``` + +> Note: This command will spend some time downloading the model from the Hugging Face Model Hub. You can use any model from the [Hugging Face Model Hub](https://huggingface.co/models) by specifying the model name in the `--model` argument. + +Expected output: + +```bash +INFO 07-24 06:51:32 deploy.py:83] Model registered successfully. +``` + +### Step 4: Query the Model Using OpenAI API Client + +**You can query the model using any OpenAI API client. For example, use the following command:** + +**Make sure the model is successfully deployed before querying.** + +> Replace `` with the actual IP address of the head node. + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +Expected output: + +```json +{"id":"chatcmpl-23d3c0e5-70a0-4771-acaf-bcb2851c6ea6","object":"chat.completion","created":1721706121,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} +``` + +#### Delete a Deployed Model Using `sllm-cli` + +When you're done using a model, you can delete it: + +```bash +sllm-cli delete facebook/opt-1.3b +``` + +This will remove the specified model from the ServerlessLLM server. + +## Clean Up + +To stop and remove all ServerlessLLM containers: + +1. **Stop all containers:** + +```bash +# On head node machine +docker stop sllm_head +docker rm sllm_head + +# On each worker machine +docker stop sllm_worker_0 # Use appropriate container name (sllm_worker_1, sllm_worker_2, etc.) +docker rm sllm_worker_0 +``` + +2. **Optional: Remove the Docker image:** + +```bash +docker rmi serverlessllm/sllm:latest +``` + +:::tip +If you don't have the ServerlessLLM Docker image locally, Docker will automatically pull it from the registry. You can also adjust the CPU and resource allocations by setting additional environment variables like `RAY_NUM_CPUS` and `RAY_RESOURCES`. +::: + +## Troubleshooting + +### Network Issues + +1. **Connection refused errors**: Ensure that firewalls on all machines allow traffic on ports 6379, 8343, and 8073. + +2. **Ray cluster connection issues**: + - Verify that the head node IP address is correct and that the Ray port (6379) is accessible from worker machines + - Ensure both head and worker nodes have their `RAY_NODE_IP` set to an IP address that is accessible from other machines + - Check that you're not using private Docker network IPs (typically 172.x.x.x) which aren't accessible across machines + +3. **Workers can't connect to head node**: + - Make sure the `RAY_HEAD_ADDRESS` points to the external IP of the head node, not localhost or an internal Docker IP + - Verify network connectivity with `ping` or `telnet` from worker machines to the head node IP on port 6379 + +4. **GPU access issues**: Make sure the NVIDIA Docker toolkit is properly installed and that the `--gpus` flag is used for worker containers. + +### Container Management + +- **View running containers**: `docker ps` \ No newline at end of file diff --git a/versioned_docs/version-0.7.0/deployment/single_machine.md b/versioned_docs/version-0.7.0/deployment/single_machine.md new file mode 100644 index 0000000..ad8bb0b --- /dev/null +++ b/versioned_docs/version-0.7.0/deployment/single_machine.md @@ -0,0 +1,217 @@ +--- +sidebar_position: 1 +--- + +# Single machine (from scratch) + +This guide provides instructions for setting up ServerlessLLM from scratch on a single machine. This 'from scratch' approach means you will manually initialize and manage the Ray cluster components. It involves using multiple terminal sessions, each configured with a distinct Conda environment, to run the head and worker processes on the same physical machine, effectively simulating a multi-node deployment locally. + +:::note +We strongly recommend using Docker (Compose) as detailed in the [Docker Compose guide](../getting_started.md). Docker provides a smoother and generally easier setup process. Follow this guide only if Docker is not a suitable option for your environment. +::: + +## Installation + +### Requirements + +Ensure your system meets the following prerequisites: + +- **OS**: Ubuntu 20.04 +- **Python**: 3.10 +- **GPU**: NVIDIA GPU with compute capability 7.0 or higher + +### Installing with pip + +Follow these steps to install ServerlessLLM using pip: + +**Create the head environment:** + +```bash +# Create and activate a conda environment +conda create -n sllm python=3.10 -y +conda activate sllm + +# Install ServerlessLLM and its store component +pip install serverless-llm serverless-llm-store +``` + +**Create the worker environment:** + +```bash +# Create and activate a conda environment +conda create -n sllm-worker python=3.10 -y +conda activate sllm-worker + +# Install ServerlessLLM (worker version) and its store component +pip install "serverless-llm[worker]" serverless-llm-store +``` + +:::note +If you plan to integrate vLLM with ServerlessLLM, a patch needs to be applied to the vLLM repository. For detailed instructions, please refer to the [vLLM Patch](#vllm-patch) section. +::: + +### Installing from Source + +To install ServerlessLLM from source, follow these steps: + +1. Clone the repository: + ```bash + git clone https://github.com/ServerlessLLM/ServerlessLLM.git + cd ServerlessLLM + ``` + +2. Create the head environment: + ```bash + # Create and activate a conda environment + conda create -n sllm python=3.10 -y + conda activate sllm + + # Install sllm_store (pip install is recommended for speed) + cd sllm_store && rm -rf build + pip install . + cd .. + + # Install ServerlessLLM + pip install . + ``` + +3. Create the worker environment: + ```bash + # Create and activate a conda environment + conda create -n sllm-worker python=3.10 -y + conda activate sllm-worker + + # Install sllm_store (pip install is recommended for speed) + cd sllm_store && rm -rf build + pip install . + cd .. + + # Install ServerlessLLM (worker version) + pip install ".[worker]" + ``` + +### vLLM Patch + +To use vLLM with ServerlessLLM, you must apply a patch. The patch file is located at `sllm_store/vllm_patch/sllm_load.patch` within the ServerlessLLM repository. This patch has been tested with vLLM version `0.6.6`. + +Apply the patch using the following script: + +```bash +conda activate sllm-worker +./sllm_store/vllm_patch/patch.sh +``` + +## Running ServerlessLLM Locally + +These steps describe how to run ServerlessLLM on your local machine. + +### 1. Start a Local Ray Cluster + +First, initiate a local Ray cluster. This cluster will consist of one head node and one worker node (on the same machine). + +**Start the head node:** + +Open a new terminal and run: + +```bash +conda activate sllm +ray start --head --port=6379 --num-cpus=4 --num-gpus=0 \ + --resources='{"control_node": 1}' --block +``` + +**Start the worker node:** + +Open another new terminal and run: + +```bash +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 # Or your desired GPU ID +ray start --address=0.0.0.0:6379 --num-cpus=4 --num-gpus=1 \ + --resources='{"worker_node": 1, "worker_id_0": 1}' --block +``` + +### 2. Start the ServerlessLLM Store Server + +Next, start the ServerlessLLM Store server. By default, it uses `./models` as the storage path. + +Open a new terminal and run: + +```bash +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 # Or your desired GPU ID +sllm-store start +``` + +Expected output: + +```log +$ sllm-store start +INFO 12-31 17:13:23 cli.py:58] Starting gRPC server +INFO 12-31 17:13:23 server.py:34] StorageServicer: storage_path=./models, mem_pool_size=4294967296, num_thread=4, chunk_size=33554432, registration_required=False +WARNING: Logging before InitGoogleLogging() is written to STDERR +I20241231 17:13:23.947276 2165054 checkpoint_store.cpp:41] Number of GPUs: 1 +I20241231 17:13:23.947299 2165054 checkpoint_store.cpp:43] I/O threads: 4, chunk size: 32MB +I20241231 17:13:23.947309 2165054 checkpoint_store.cpp:45] Storage path: "./models" +I20241231 17:13:24.038651 2165054 checkpoint_store.cpp:71] GPU 0 UUID: c9938b31-33b0-e02f-24c5-88bd6fbe19ad +I20241231 17:13:24.038700 2165054 pinned_memory_pool.cpp:29] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes +I20241231 17:13:25.557906 2165054 checkpoint_store.cpp:83] Memory pool created with 4GB +INFO 12-31 17:13:25 server.py:243] Starting gRPC server on 0.0.0.0:8073 +``` + +### 3. Start ServerlessLLM Serve + +Now, start the ServerlessLLM Serve process ( `sllm-serve`). + +Open a new terminal and run: + +```bash +conda activate sllm +sllm-serve start +``` + +At this point, you should have four terminals open: one for the Ray head node, one for the Ray worker node, one for the ServerlessLLM Store server, and one for ServerlessLLM Serve. + +### 4. Deploy a Model + +With all services running, you can deploy a model. + +Open a new terminal and run: + +```bash +conda activate sllm +sllm-cli deploy --model facebook/opt-1.3b +``` + +This command downloads the specified model from Hugging Face Hub. To load a model from a local path, you can use a `config.json` file. Refer to the [CLI API documentation](../api/cli.md#example-configuration-file-configjson) for details. + +### 5. Query the Model + +Once the model is deployed, you can query it using any OpenAI API-compatible client. For example, use the following `curl` command: + +```bash +curl http://127.0.0.1:8343/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +Expected output: + +```json +{"id":"chatcmpl-9f812a40-6b96-4ef9-8584-0b8149892cb9","object":"chat.completion","created":1720021153,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} +``` + +## Clean Up + +To delete a deployed model, use the following command: + +```bash +sllm-cli delete facebook/opt-1.3b +``` + +This command removes the specified model from the ServerlessLLM server. \ No newline at end of file diff --git a/versioned_docs/version-0.7.0/deployment/slurm_cluster.md b/versioned_docs/version-0.7.0/deployment/slurm_cluster.md new file mode 100644 index 0000000..f6d9a8c --- /dev/null +++ b/versioned_docs/version-0.7.0/deployment/slurm_cluster.md @@ -0,0 +1,411 @@ +--- +sidebar_position: 3 +--- + +# SLURM cluster + +This guide will help you get started with running ServerlessLLM on SLURM cluster. It provides two deployment methods, based on `sbatch` and `srun`. If you are in development, we recommend using `srun`, as it is easier to debug than `sbatch`, and if you are in production mode, `sbatch` is recommended. Please make sure you have installed the ServerlessLLM following the [installation guide](./single_machine.md#installation) on all machines. + +## Pre-requisites +Before you begin, make sure you have checked the following: +### Some Tips about Installation +- If 'not enough disk space' is reported when `pip install` on the login node, you can submit it to a job node for execution + ```shell + #!/bin/bash + #SBATCH --partition=Teach-Standard + #SBATCH --job-name=ray-head + #SBATCH --output=sllm_pip.out + #SBATCH --error=sllm_pip.err + #SBATCH --nodes=1 + #SBATCH --ntasks=1 + #SBATCH --cpus-per-task=4 + #SBATCH --gpus-per-task=0 + + # Identify which conda you are using, here is an example that conda is in /opt/conda + source /opt/conda/bin/activate + + conda create -n sllm python=3.10 -y + conda activate sllm + pip install serverless-llm + pip install serverless-llm-store + + conda deactivate sllm + + conda create -n sllm-worker python=3.10 -y + conda activate sllm-worker + pip install serverless-llm[worker] + pip install serverless-llm-store + ``` + +### Command for Querying GPU Resource Information +Run the following commands in the cluster to check GPU resource information. +```shell +sinfo -O partition,nodelist,gres +``` +**Expected Output** +```shell +PARTITION NODELIST GRES +Partition1 JobNode[01,03] gpu:gtx_1060:8 +Partition2 JobNode[04-17] gpu:a6000:2,gpu:gtx_ +``` + +### Identify an idle node +Use `sinfo -p ` to identify some idle nodes + +**Expected Output** +```shell +$ sinfo -p compute +PARTITION AVAIL NODES STATE TIMELIMIT NODELIST +compute up 10 idle infinite JobNode[01-10] +compute up 5 alloc infinite JobNode[11-15] +compute up 2 down infinite JobNode[16-17] +``` + +### Job Nodes Setup +**`srun` Node Selection** + +Only one JobNode is enough. + +**`sbatch` Node Selection** + +Let's start a head on the main job node (`JobNode01`) and add the worker on other job node (`JobNode02`). The head and the worker should be on different job nodes to avoid resource contention. The `sllm-store` should be started on the job node that runs worker (`JobNode02`), for passing the model weights, and the `sllm-serve` should be started on the main job node (`JobNode01`), finally you can use `sllm-cli` to manage the models on the login node. + +Note: `JobNode02` requires GPU, but `JobNode01` does not. +- **Head**: JobNode01 +- **Worker**: JobNode02 +- **sllm-store**: JobNode02 +- **sllm-serve**: JobNode01 +- **sllm-cli**: Login Node + +--- +## SRUN +If you are in development, we recommend using `srun` to start ServerlessLLM, as it is easier to debug than `sbatch` +### Step 1: Use `srun` enter the JobNode +To start an interactive session on the specified compute node (JobNode), use: +``` +srun --partition --nodelist --gres :1 --pty bash +``` +This command requests a session on the specified node and provides an interactive shell. `--gres :1` specifies the GPU device you will use, for example: `--gres gpu:gtx_1060:1` + +### Step 2: Install ServerlessLLM +Firstly, please make sure CUDA driver available on the node. Here are some commands to check it. +```shell +nvidia-smi + +which nvcc +``` +If `nvidia-smi` has listed GPU information, but `which nvcc` has no output. Then use the following commands to load `nvcc`. Here is an example that cuda is located at `/opt/cuda-12.2.0` +```shell +export PATH=/opt/cuda-12.2.0/bin:$PATH +export LD_LIBRARY_PATH=/opt/cuda-12.2.0/lib64:$LD_LIBRARY_PATH +``` +Then, following the [installation guide](./single_machine.md#installation) to install ServerlessLLM. +### Step 3: Prepare multiple windows with `tmux` +Since srun provides a single interactive shell, you can use tmux to create multiple windows. Start a tmux session: +```shell +tmux +``` +This creates a new tmux session + +**Create multiple windows** +- Use `Ctrl+B` → `C` to start a new window +- Repeat the shortcut 4 more times to create a total of 5 windows. + +**What if `Ctrl+B` does not work?** + +If `Ctrl + B` is unresponsive, reset tmux key bindings: +```shell +tmux unbind C-b +tmux set-option -g prefix C-b +tmux bind C-b send-prefix +``` + +**Command to switch windows** + +Once multiple windows are created, you can switch between them using: + +`Ctrl + B` → `N` (Next window) +`Ctrl + B` → `P` (Previous window) +`Ctrl + B` → `W` (List all windows and select) +`Ctrl + B` → [Number] (Switch to a specific window, e.g., Ctrl + B → 1) + +### Step 4: Run ServerlessLLM on the JobNode +First find ports that are already occupied. Then pick your favourite number from the remaining ports to replace the following placeholder ``. For example: `6379` + +It should also be said that certain slurm system is a bit slow, **so please be patient and wait for the system to output**. + +In the first window, start a local ray cluster with 1 head node and 1 worker node: +```shell +source /opt/conda/bin/activate +conda activate sllm +ray start --head --port= --num-cpus=4 --num-gpus=0 --resources='{"control_node": 1}' --block +``` +In the second window, start the worker node: +```shell +source /opt/conda/bin/activate +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 +ray start --address=0.0.0.0: --num-cpus=4 --num-gpus=1 --resources='{"worker_node": 1, "worker_id_0": 1}' --block +``` +In the third window, start ServerlessLLM Store server: +```shell +source /opt/conda/bin/activate +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 +sllm-store start +``` +In the 4th window, start ServerlessLLM Serve: +```shell +source /opt/conda/bin/activate +conda activate sllm +sllm-serve start +``` +Everything is set! + + +In the 5th window, let's deploy a model to the ServerlessLLM server. You can deploy a model by running the following command: +```shell +source /opt/conda/bin/activate +conda activate sllm +sllm-cli deploy --model facebook/opt-1.3b --backend transformers +``` +This will download the model from HuggingFace transformers. After deploying, you can query the model by any OpenAI API client. For example, you can use the following Python code to query the model: +```shell +curl http://127.0.0.1:8343/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` +Expected output: +```shell +{"id":"chatcmpl-9f812a40-6b96-4ef9-8584-0b8149892cb9","object":"chat.completion","created":1720021153,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} +``` + +### Step 5: Clean up +To delete a deployed model, use the following command: +```shell +sllm-cli delete facebook/opt-1.3b +``` +This will remove the specified model from the ServerlessLLM server. + +In each window, use `Ctrl + c` to stop server and `exit` to exit current `tmux` session. + +--- +## SBATCH +### Step 1: Start the Head Node +Since the head node does not require a gpu, you can find a low-computing capacity node to deploy the head node. +1. **Activate the `sllm` environment and start the head node:** + + Here is the example script, named `start_head_node.sh`. + ```shell + #!/bin/bash + #SBATCH --partition=your-partition # Specify the partition + #SBATCH --nodelist=JobNode01 # Specify an idle node + #SBATCH --job-name=ray-head + #SBATCH --output=sllm_head.out + #SBATCH --error=sllm_head.err + #SBATCH --nodes=1 + #SBATCH --ntasks=1 + #SBATCH --cpus-per-task=12 + #SBATCH --gpus-per-task=0 + + cd /path/to/ServerlessLLM + + source /opt/conda/bin/activate # make sure conda will be loaded correctly + conda activate sllm + + ray start --head --port=6379 --num-cpus=12 --num-gpus=0 --resources='{"control_node": 1}' --block + ``` + - Replace `your-partition`, `JobNode01` and `/path/to/ServerlessLLM` + +2. **Submit the script** + + Use ```sbatch start_head_node.sh``` to submit the script to certain idle node. + +3. **Expected output** + + In `sllm_head.out`, you will see the following output: + + ```shell + Local node IP: + -------------------- + Ray runtime started. + -------------------- + ``` + **Remember the IP address**, denoted ``````, you will need it in following steps. + +4. **Find an available port for serve** + - Some HPCs have a firewall that blocks port 8343. You can use `nc -zv 8343` to check if the port is accessible. + - If it is not accessible, find an available port and replace `available_port` in the following script. + - Here is an example script, named `find_port.sh` + + ```shell + #!/bin/bash + #SBATCH --partition=your-partition + #SBATCH --nodelist=JobNode01 + #SBATCH --job-name=find_port + #SBATCH --output=find_port.log + #SBATCH --time=00:05:00 + #SBATCH --mem=1G + + echo "Finding available port on $(hostname)" + + python -c " + import socket + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) + print(f'Available port: {s.getsockname()[1]}') + " + ``` + Use `sbatch find_port.sh` to submit the script to JobNode01, and in `find_port.log`, you will see the following output: + ``` + Finding available port on JobNode01 + Available port: + ``` + Remember this ``, you will use it in Step 4 + +### Step 2: Start the Worker Node & Store +We will start the worker node and store in the same script. Because the server loads the model weights onto the GPU and uses shared GPU memory to pass the pointer to the client. If you submit another script with ```#SBATCH --gpres=gpu:1```, it will be possibly set to use a different GPU, as specified by different ```CUDA_VISIBLE_DEVICES``` settings. Thus, they cannot pass the model weights. +1. **Activate the ```sllm-worker``` environment and start the worker node.** + + Here is the example script, named```start_worker_node.sh```. + ```shell + #!/bin/sh + #SBATCH --partition=your_partition + #SBATCH --nodelist=JobNode02 + #SBATCH --gres=gpu:a6000:1 # Specify device on JobNode02 + #SBATCH --job-name=sllm-worker-store + #SBATCH --output=sllm_worker.out + #SBATCH --error=sllm_worker.err + #SBATCH --gres=gpu:1 # Request 1 GPU + #SBATCH --cpus-per-task=4 # Request 4 CPU cores + #SBATCH --mem=16G # Request 16GB of RAM + + cd /path/to/ServerlessLLM + + conda activate sllm-worker + + HEAD_NODE_IP= + + export CUDA_HOME=/opt/cuda-12.5.0 # replace with your CUDA path + export PATH=$CUDA_HOME/bin:$PATH + export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH + + ray start --address=$HEAD_NODE_IP:6379 --num-cpus=4 --num-gpus=1 \ + --resources='{"worker_node": 1, "worker_id_0": 1}' --block & + + sllm-store start & + + wait + ``` + - Read the HPC's documentation to find out which partition you can use. Replace ```your_partition``` in the script with that partition name. + - Replace ```/path/to/ServerlessLLM``` with the path to the ServerlessLLM installation directory. + - Replace `````` with the IP address of the head node. + - Replace ```/opt/cuda-12.5.0``` with the path to your CUDA path. + +2. **Find the CUDA path** + - Some slurm-based HPCs have a module system, you can use ```module avail cuda``` to find the CUDA module. + - If it does not work, read the HPC's documentation carefully to find the CUDA path. For example, my doc said CUDA is in ```\opt```. Then you can use ```srun``` command to start an interactive session on the node, such as ```srun --pty -t 00:30:00 -p your_partition --gres=gpu:1 /bin/bash```. A pseudo-terminal will be started for you to find the path. + - Find it and replace ```/opt/cuda-12.5.0``` with the path to your CUDA path. +3. **Submit the script on the other node** + + Use ```sbatch start_worker_node.sh``` to submit the script to certain idle node (here we assume it is ```JobNode02```). In addition, We recommend that you place the head and worker on different nodes so that the Serve can start smoothly later, rather than queuing up for resource allocation. +4. **Expected output** + + In ```sllm_worker.out```, you will see the following output: + + - The worker node expected output: + ```shell + Local node IP: xxx.xxx.xx.xx + -------------------- + Ray runtime started. + -------------------- + ``` + - The store expected output: + ```shell + I20241030 11:52:54.719007 1321560 checkpoint_store.cpp:41] Number of GPUs: 1 + I20241030 11:52:54.773468 1321560 checkpoint_store.cpp:43] I/O threads: 4, chunk size: 32MB + I20241030 11:52:54.773548 1321560 checkpoint_store.cpp:45] Storage path: "./models/" + I20241030 11:52:55.060559 1321560 checkpoint_store.cpp:71] GPU 0 UUID: 52b01995-4fa9-c8c3-a2f2-a1fda7e46cb2 + I20241030 11:52:55.060798 1321560 pinned_memory_pool.cpp:29] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes + I20241030 11:52:57.258795 1321560 checkpoint_store.cpp:83] Memory pool created with 4GB + I20241030 11:52:57.262835 1321560 server.cpp:306] Server listening on 0.0.0.0:8073 + ``` +### Step 3: Start the Serve on the Head Node +1. **Activate the ```sllm``` environment and start the serve.** + + Here is the example script, named```start_serve.sh```. + ```shell + #!/bin/sh + #SBATCH --partition=your_partition + #SBATCH --nodelist=JobNode01 # This node should be the same as head + #SBATCH --output=serve.log + + cd /path/to/ServerlessLLM + + conda activate sllm + + sllm-serve start --host + # sllm-serve start --host --port # if you have changed the port + ``` + - Replace `your_partition` in the script as before. + - Replace `/path/to/ServerlessLLM` as before. + - Replace `` you have found in Step 1 (if port 8343 is not available). +2. **Submit the script on the head node** + + Use ```sbatch start_serve.sh``` to submit the script to the head node (```JobNode01```). + +3. **Expected output** + ```shell + -- Connecting to existing Ray cluster at address: xxx.xxx.xx.xx:6379... + -- Connected to Ray cluster. + INFO: Started server process [1339357] + INFO: Waiting for application startup. + INFO: Application startup complete. + INFO: Uvicorn running on http://xxx.xxx.xx.xx:8343 (Press CTRL+C to quit) + ``` +### Step 4: Use sllm-cli to manage models +1. **You can do this step on login node, and set the ```LLM_SERVER_URL``` environment variable:** + ```shell + $ conda activate sllm + (sllm)$ export LLM_SERVER_URL=http://:8343 + ``` + - Replace `` with the actual IP address of the head node. + - Replace ```8343``` with the actual port number (`` in Step1) if you have changed it. +2. **Deploy a Model Using ```sllm-cli```** + ```shell + (sllm)$ sllm-cli deploy --model facebook/opt-1.3b + ``` +### Step 5: Query the Model Using OpenAI API Client + **You can use the following command to query the model:** + ```shell + curl $LLM_SERVER_URL/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' + ``` + - Replace `````` with the actual IP address of the head node. + - Replace ```8343``` with the actual port number (`` in Step 1) if you have changed it. +### Step 6: Stop Jobs +On the SLURM cluster, we usually use the ```scancel``` command to stop the job. Firstly, list all jobs you have submitted (replace ```your_username``` with your username): +```shell +$ squeue -u your_username +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 1234 compute sllm-head your_username R 0:01 1 JobNode01 + 1235 compute sllm-worker-store your_username R 0:01 1 JobNode02 + 1236 compute sllm-serve your_username R 0:01 1 JobNode01 +``` +Then, use ```scancel``` to stop the job (```1234```, ```1235``` and ```1236``` are JOBIDs): +```shell +$ scancel 1234 1235 1236 +``` diff --git a/versioned_docs/version-0.7.0/developer/_category_.json b/versioned_docs/version-0.7.0/developer/_category_.json new file mode 100644 index 0000000..89a7abc --- /dev/null +++ b/versioned_docs/version-0.7.0/developer/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Developer Guide", + "position": 6 +} diff --git a/versioned_docs/version-0.7.0/developer/supporting_a_new_hardware.md b/versioned_docs/version-0.7.0/developer/supporting_a_new_hardware.md new file mode 100644 index 0000000..2dc4f0a --- /dev/null +++ b/versioned_docs/version-0.7.0/developer/supporting_a_new_hardware.md @@ -0,0 +1,25 @@ +--- +sidebar_position: 0 +--- + +# Supporting a New Hardware + +ServerlessLLM actively expands support for new hardware configurations to meet diverse deployment needs. + +## Support Standards +Hardware is considered supported by ServerlessLLM if: +1. Any of the inference backends used (e.g., Transformers, vLLM) can run model inference on the hardware. +2. ServerlessLLM Store can successfully load model checkpoints on the hardware. + +## Steps to Support a New Hardware +1. **Check Inference Backend Compatibility**: Refer to the specific inference backend documentation (e.g., for vLLM, Transformers) for hardware support. +2. **ServerlessLLM Store Configuration**: + - If the hardware provides CUDA-compatible APIs (e.g., ROCm), adjust the build script (`CMakeLists.txt`) by adding necessary compiler flags. + - For non-CUDA-compatible APIs, implementing a custom checkpoint loading function might be required. + +## Verifying Hardware Support in ServerlessLLM Store +The hardware support is verified if it successfully completes the [Quick Start Guide](https://serverlessllm.github.io/docs/getting_started/) examples, showcasing checkpoint loading and inference functionality without errors. + +If the hardware is not publicly available (i.e., can't be tested by the ServerlessLLM team), a screenshot or output log of the successful execution of the Quick Start Guide examples is required to verify hardware support. + +If you encounter any issues or have questions, please reach out to the ServerlessLLM team by raising an issue on the [GitHub repository](https://github.com/ServerlessLLM/ServerlessLLM/issues). \ No newline at end of file diff --git a/versioned_docs/version-0.7.0/features/_category_.json b/versioned_docs/version-0.7.0/features/_category_.json new file mode 100644 index 0000000..56e0bf0 --- /dev/null +++ b/versioned_docs/version-0.7.0/features/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Features", + "position": 2 +} \ No newline at end of file diff --git a/versioned_docs/version-0.7.0/features/live_migration.md b/versioned_docs/version-0.7.0/features/live_migration.md new file mode 100644 index 0000000..504cf66 --- /dev/null +++ b/versioned_docs/version-0.7.0/features/live_migration.md @@ -0,0 +1,211 @@ +--- +sidebar_position: 1 +--- + +# Live Migration of Inference Instances + +This example illustrates the live migration of inference instances in a ServerlessLLM cluster by constructing a scenario where two models are deployed to the cluster. Model `Qwen2.5-3B` is stored on both nodes, while model `Qwen2.5-1.5B` is only stored on node 0 (e.g., due to being less popular). This example will show a locality-contention scenario where `Qwen2.5-3B` is being served on node 0 but `Qwen2.5-1.5B` is requested to be served on the same node for optimal locality. We will find that: + +- **Without migration**, `Qwen2.5-1.5B` would have to wait for the completion of the ongoing inference instance of `Qwen2.5-3B` on node 0. +- **With live migration**, the ongoing inference instance of `Qwen2.5-3B` is migrated to node 1, and `Qwen2.5-1.5B` is allocated to node 0, thus can be served immediately. + +## Prerequisites + +To run this example, we will use Docker Compose to set up a ServerlessLLM cluster. Before proceeding, please ensure you have read the [Quickstart Guide](../getting_started.md). + +**Requirements:** + +- **Two GPUs** are required to illustrate the live migration of inference instances. +- **At least 20 GB of host memory** (this can be adjusted by using smaller models). +- **ServerlessLLM version 0.6**: Ensure you have `sllm==0.6` and `sllm-store==0.6` installed. + +## + +Start a local Docker-based ray cluster using Docker Compose. + +### Clone the ServerlessLLM Repository + +If you haven't already, clone the ServerlessLLM repository: + +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/examples/live_migration +``` + +### Configure the Model Directory + +Create a directory on your host machine where models will be stored, and set the MODEL_FOLDER environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +The Docker Compose configuration is already located in the `examples/live_migration` directory. + +## Test ServerlessLLM Without Live Migration + +1. **Start the ServerlessLLM Services Using Docker Compose** + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +2. **Deploy Models with the Placement Spec Files** + +Activate the ServerlessLLM environment and set the server URL: +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` + +Deploy the models: +```bash +sllm-cli deploy --config config-qwen-1.5b.json +sllm-cli deploy --config config-qwen-3b.json +``` + +3. **Verify the Deployment** + +Start two inference requests in parallel. The first request is for `Qwen2.5-3B`, and the second request, sent shortly after, is for `Qwen2.5-1.5B`. The `sleep` command is used to introduce a short interval between the two requests: + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-3B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Could you share a story of the history of Computer Science?"} + ], + "max_tokens": 1024 + }' & + +sleep 3 + +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-1.5B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ], + "max_tokens": 64 + }' +``` + +Since `Qwen2.5-3B` is requested first, `Qwen2.5-1.5B` must wait for the ongoing inference instance of `Qwen2.5-3B` to complete on node 0 before it can start processing. + + +4. Clean up. + +```bash +docker compose down +``` + +## Test ServerlessLLM With Live Migration + +1. **Start the ServerlessLLM Services with Live Migration Enabled** + +Use the following command to start the ServerlessLLM services with live migration enabled. This configuration includes the `enable-migration.yml` file: + +```bash +docker compose -f docker-compose.yml -f enable-migration.yml up -d +``` + +This command will start the Ray head node and two worker nodes, enabling the live migration feature. + +2. **Deploy Models with the Placement Spec Files** + +Activate the ServerlessLLM environment and set the server URL: + +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` + +Deploy the models: + +```bash +sllm-cli deploy --config config-qwen-1.5b.json +sllm-cli deploy --config config-qwen-3b.json +``` + +3. **Verify the Deployment** + +Start two inference requests in parallel. The first request is for `Qwen2.5-3B`, and the second request, sent shortly after, is for `Qwen2.5-1.5B`. The `sleep` command is used to introduce a short interval between the two requests: + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-3B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Could you share a story of the history of Computer Science?"} + ], + "max_tokens": 1024 + }' & + +sleep 3 + +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-1.5B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ], + "max_tokens": 64 + }' +``` + +According to the response, you should observe that `Qwen2.5-1.5B` completes ahead of `Qwen2.5-3B`. This is because the ongoing inference instance of `Qwen2.5-3B` is live-migrated from node 0 to node 1, allowing `Qwen2.5-1.5B` to be served immediately on node 0. + +As shown in the log message, the ongoing inference instance of the model `Qwen/Qwen2.5-3B-Instruct` is live-migrated from node 0 to node 1. And model `Qwen/Qwen2.5-1.5B-Instruct` is allocated to node 0. + +```bash +(MigrationRouter pid=1724) INFO 12-10 22:05:02 migration_router.py:106] Executing migration plan: MigrationPlan(target_node_id='1', source_instance=InstanceStatus(instance_id='Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab', node_id='0', num_gpu=1, concurrency=0, model_name='Qwen/Qwen2.5-3B-Instruct', num_current_tokens=0)) +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:164] Initialized backend for instance Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 for model Qwen/Qwen2.5-3B-Instruct +# Start multi-round live migration +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:178] Migration iteration 0 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:183] Number of tokens: 353, delta: 353 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:198] Migration iteration 0 completed +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:178] Migration iteration 1 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:183] Number of tokens: 14, delta: 14 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:188] Migration completed: remained 14 tokens +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:201] Migrated instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab to Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 +# Finish multi-round live migration +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:215] Instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab removed +(MigrationRouter pid=1724) DEBUG 12-10 22:05:13 migration_router.py:77] Preempted request: ... +# Resume the instance on target node +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:83] Resuming request on target instance: Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 +# Qwen/Qwen2.5-1.5B is allocated to node 0 +(StoreManager pid=1459) INFO 12-10 22:05:14 store_manager.py:344] Loading Qwen/Qwen2.5-1.5B-Instruct to node 0 +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:92] Deallocating model Qwen/Qwen2.5-3B-Instruct instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:103] Node 0 deallocated 1 GPUs +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:108] Model Qwen/Qwen2.5-3B-Instruct instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab deallocated +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 storage_aware_scheduler.py:188] Migrated instance Qwen/Qwen2.5-3B-Instruct to node 1 instance Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 storage_aware_scheduler.py:195] Allocated node 0 for model Qwen/Qwen2.5-1.5B-Instruct +``` + +4. Clean up. + +```bash +docker compose down +``` + + diff --git a/versioned_docs/version-0.7.0/features/peft_lora_serving.md b/versioned_docs/version-0.7.0/features/peft_lora_serving.md new file mode 100644 index 0000000..9c42003 --- /dev/null +++ b/versioned_docs/version-0.7.0/features/peft_lora_serving.md @@ -0,0 +1,105 @@ +--- +sidebar_position: 2 +--- +# PEFT LoRA Serving + +## Pre-requisites + +To run this example, we will use Docker Compose to set up a ServerlessLLM cluster. Before proceeding, please ensure you have read the [Quickstart Guide](../getting_started.md). + +We will use the following example base model & LoRA adapter +- Base model: `facebook/opt-125m` +- LoRA adapter: `peft-internal-testing/opt-125m-dummy-lora` + +## Usage + +Start a local Docker-based ray cluster using Docker Compose. + +### Step 1: Clone the ServerlessLLM Repository + +If you haven't already, clone the ServerlessLLM repository: +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +``` + +### Step 2: Configuration + +Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +### Step 3: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +### Step 4: Deploy Models with LoRA Adapters +1. Configuration +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` +2. Deploy models with specified lora adapters. +```bash +sllm-cli deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters demo_lora1=peft-internal-testing/opt-125m-dummy-lora demo_lora2=monsterapi/opt125M_alpaca +``` +3. Verify the deployment. +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ], + "lora_adapter_name": "demo_lora1" + }' +``` +If no lora adapters specified, the system will use the base model to do inference +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` +### Step 5: Update LoRA Adapters +If you wish to switch to a different set of LoRA adapters, you can still use `sllm-cli deploy` command with updated adapter configurations. ServerlessLLM will automatically reload the new adapters without restarting the backend. + +For example, to update the adapter (located at `ft_facebook/opt-125m_adapter1`) used by facebook/opt-125m: +```bash +sllm-cli deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters demo_lora=ft_facebook/opt-125m_adapter1 +``` + +### Step 6: Clean Up + +Delete the lora adapters by running the following command (this command will only delete lora adapters, the base model won't be deleted): +```bash +sllm-cli delete facebook/opt-125m --lora-adapters demo-lora1 demo-lora2 +``` +If you need to stop and remove the containers, you can use the following commands: +```bash +docker compose down +``` \ No newline at end of file diff --git a/versioned_docs/version-0.7.0/features/storage_aware_scheduling.md b/versioned_docs/version-0.7.0/features/storage_aware_scheduling.md new file mode 100644 index 0000000..59a62ab --- /dev/null +++ b/versioned_docs/version-0.7.0/features/storage_aware_scheduling.md @@ -0,0 +1,123 @@ +--- +sidebar_position: 0 +--- + +# Storage Aware Scheduling with Docker Compose + +## Pre-requisites + +We will use Docker Compose to run a ServerlessLLM cluster in this example. Therefore, please make sure you have read the [Quickstart Guide](../getting_started.md) before proceeding. + +## Usage + +Start a local Docker-based ray cluster using Docker Compose. + +### Step 1: Clone the ServerlessLLM Repository + +If you haven't already, clone the ServerlessLLM repository: + +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/examples/storage_aware_scheduling +``` + +### Step 2: Configuration + +Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +### Step 3: Enable Storage Aware Scheduling in Docker Compose + +The Docker Compose configuration is already located in the `examples/storage_aware_scheduling` directory. To activate storage-aware scheduling, ensure the `docker-compose.yml` file includes the necessary configurations(`sllm_head` service should include the `--enable-storage-aware` command). + +:::tip +Recommend to adjust the number of GPUs and `mem_pool_size` based on the resources available on your machine. +::: + + +### Step 4: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +### Step 5: Deploy Models with Placement Spec + +In the `examples/storage_aware_scheduling` directory, the example configuration files (`config-opt-2.7b.json` and `config-opt-1.3b.json`) are already given. + +> Note: Storage aware scheduling currently only supports the "transformers" backend. Support for other backends will come soon. + +2. Deploy models with the placement spec files. + +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 + +sllm-cli deploy --config config-opt-2.7b.json +sllm-cli deploy --config config-opt-1.3b.json +``` + +3. Verify the deployment. + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-2.7b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' + +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +As shown in the log message, the model "facebook/opt-2.7b" is scheduled on server 0, while the model "facebook/opt-1.3b" is scheduled on server 1. + +```log +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:27 storage_aware_scheduler.py:137] Sorted scheduling options: [('0', 4.583079601378258)] +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:27 storage_aware_scheduler.py:144] Allocated node 0 for model facebook/opt-2.7b +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:38 storage_aware_scheduler.py:137] Sorted scheduling options: [('1', 2.266678696047572)] +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:38 storage_aware_scheduler.py:144] Allocated node 1 for model facebook/opt-1.3b +``` + +### Step 6: Clean Up + +Delete the model deployment by running the following command: + +```bash +sllm-cli delete facebook/opt-1.3b facebook/opt-2.7b +``` + +If you need to stop and remove the containers, you can use the following commands: + +```bash +docker compose down +``` + diff --git a/versioned_docs/version-0.7.0/getting_started.md b/versioned_docs/version-0.7.0/getting_started.md new file mode 100644 index 0000000..9ecee0c --- /dev/null +++ b/versioned_docs/version-0.7.0/getting_started.md @@ -0,0 +1,136 @@ +--- +sidebar_position: 1 +--- + +# Getting Started + +This guide demonstrates how to quickly set up a local ServerlessLLM cluster using Docker Compose on a single machine. We will initialize a minimal cluster, consisting of a head node and a single worker node. Then, we'll deploy a model using the `sllm-cli` and query the deployment through an OpenAI-compatible API. + +:::note +We strongly recommend using Docker (Compose) to manage your ServerlessLLM cluster, whether you are using ServerlessLLM for testing or development. However, if Docker is not a viable option for you, please refer to the [deploy from scratch guide](./deployment/single_machine.md). +::: + +## Prerequisites + +Before you begin, ensure you have the following installed and configured: + +1. **Docker**: Installed on your system. You can download it from [here](https://docs.docker.com/get-docker/). +2. **ServerlessLLM CLI**: Installed on your system. Install it using `pip install serverless-llm`. +3. **GPUs**: At least one NVIDIA GPU is required. If you have multiple GPUs, you can adjust the `docker-compose.yml` file accordingly. +4. **NVIDIA Docker Toolkit**: This enables Docker to utilize NVIDIA GPUs. Follow the installation guide [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). + +## Start the ServerlessLLM Cluster + +We will use Docker Compose to simplify the ServerlessLLM setup process. + +### Step 1: Download the Docker Compose File + +Download the `docker-compose.yml` file from the ServerlessLLM repository: + +```bash +# Create a directory for the ServerlessLLM Docker setup +mkdir serverless-llm-docker && cd serverless-llm-docker + +# Download the docker-compose.yml file +curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml + +# Alternatively, you can use wget: +# wget https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml +``` + +### Step 2: Configuration + +Create a directory on your host machine to store models. Then, set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you intend to store the models. This directory will be mounted into the Docker containers. + +### Step 3: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and a worker node as defined in the `docker-compose.yml` file. + +Verify that the services are ready: + +```bash +docker logs sllm_head +``` + +Ensure the services are ready before proceeding. You should see output similar to the following: + +```bash +... +(SllmController pid=1435) INFO 05-26 15:40:49 controller.py:68] Starting scheduler +INFO: Started server process [1] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:8343 (Press CTRL+C to quit) +(FcfsScheduler pid=1604) INFO 05-26 15:40:49 fcfs_scheduler.py:54] Starting FCFS scheduler +(FcfsScheduler pid=1604) INFO 05-26 15:40:49 fcfs_scheduler.py:111] Starting control loop +``` + +## Deploy a Model Using sllm-cli + +Set the `LLM_SERVER_URL` environment variable: + +```bash +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` + +Deploy a model to the ServerlessLLM cluster using the `sllm-cli`: + +```bash +sllm-cli deploy --model facebook/opt-1.3b +``` +> Note: This command will take some time to download the model from the Hugging Face Model Hub. +> You can use any model from the [Hugging Face Model Hub](https://huggingface.co/models) by specifying its name in the `--model` argument. + +Expected output: + +```plaintext +INFO 08-01 07:38:12 deploy.py:36] Deploying model facebook/opt-1.3b with default configuration. +INFO 08-01 07:39:00 deploy.py:49] Model registered successfully. +``` + +## Query the Model + +You can now query the model using any OpenAI API client. For example, use the following `curl` command: +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +Expected output: + +```plaintext +{"id":"chatcmpl-8b4773e9-a98b-41db-8163-018ed3dc65e2","object":"chat.completion","created":1720183759,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}}% +``` + +## Clean Up +To delete a deployed model, execute the following command: + +```bash +sllm-cli delete facebook/opt-1.3b +``` + +This command removes the specified model from the ServerlessLLM server. + +To stop the ServerlessLLM services, use the following command: +```bash +docker compose down +``` \ No newline at end of file diff --git a/versioned_docs/version-0.7.0/images/favicon.ico b/versioned_docs/version-0.7.0/images/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..4ef51a7c4d8cb55f59100813fb8ffa9c88795d74 GIT binary patch literal 46169 zcmc$Gg;SQz_cz@g(j_I`_0S+CjocvJ4bt5u(j_S&As{J@G)Q+N-QC^rTfX!D53i2n zJ!7!fUc2Xf>Hq}=4Sv48!qY$HjkLYr;@6}%a^z|1K0{BhW zsmL4(Dga9Ey|}u^{9&eR2I>n*Txc763L#Hs`ZFEUStE_||2sIw98$7B62 z^C+HnWkmn~f9yRAcBw)fi>+W)_<4a7CE98&(wSKxla`2*R5(+(yJ5${8lHWHrHb0%y)EK$4p>Up9s=F7pZCe+g| z@kVPn?RXdtFL7K5ykh(0IJ_s6(r#_@Q+^p z#uYAx$F&!IK;RCHM)BZ{!|Z^O;=rQ{m#D!PVqEQy-#{lDWe5@b+V_GT5He0_5@&`7 zi6);x@j=5{ydCP#`to9R$rMn*J~VkEO-O_EPP!c?KNr~pBQER*mN?aze!HC|Q$r{} z48~dLBBeY{sv?aJZlA3DTQ;uf+(onx=Ufc^qnpFZeCKnGdW5D#hPb~sOh=VP@A2C; zaSA3pq0-)#E=!~uS$1Z|nl~N;K9F{uDp>&m}n=O1nUNHuOi$ z-X2^hfw!A-Z@Bh;$!(hEtEDEEmuIq6LA-T1sSUIEo=jfH|k7}BNtwzC2S8{5sUJIRbFaXet!m`=F+gWLN) z#7#3!cE(MCJ@6$GvW&aaQv)gysZLOgq}GDBbk5)<#+^%{ud@%W_QSsXZ;fLa`@sQR zP*DhJi$mN_oNDj-lx8)~gzR|O2IT1JxuP%enB>BdQ=`7L8YqiV5_ayLjoZ3Y4|4vM zcMA8q!G!SsHoovP)q+t83>Et!3cEEW)&_HHz0PbHkXw|&<4WWI($50c5@)N}ZYtC$ zLyi{e==oT~FEqs?SX6AEQ(BjHo=(ST{r#6!)#ShRb?^F&(b!hE_~)kXgBumdN<=9` zh|{Wbz*x`oX7)$uCzMsDkgpeP&<${L(FkcJ+ut~i4fQMhC{P$0eEqaD$oIczs!!Le z=r1T}DdHx%vP^N;Ojx>axICQR1&83#R+Wt&j-+#{iDvf_7_Ob$h2Zm8KeG|w zn5ng$oO}}p%ilMGW0%AaN_bS*Rv;m}Fnd?JdIoRl>Sbl=sIp{OSXvN%J`dKcQ)E>$ zIkw(~jeq_-h*4ckB0(&HCckfmT1gnHGh*PfQj(A>U**Mir-PCASR_cC>m|5dLhi?9 z;d9o1;8qworONM6K^JeI%EPh3o~uLHn_S=|BdV@UWi$ z1=7(Tga#h_r~izp`^8x?nU@FlCm7`EbgHq~kMLms@j{K}GvEExVK2+Jg+5m$f{&7s z-54Kwl&C2caqeAVEFL&b5HeRJEcVyXn(f>nxiMxZ&Sg4x3})oEF;Ox4UR&w6W8_9w zE!6v)q;K%U zQGugg-1O2^!9#l0{tZCo8 z1!GGIJXVh4lPeTG30OW&C->E78YyVG4_N8Q{^?^99v{b_?e-1?Lb3j-mFoKRcsx~$ zu@41pEg*$-I^ue%Q(+Y+qra2KK?qK!`Ic@>JDL2!Q6IqsVXMkhnJ9AhB>c-|KvAtK zo{63M6mPZ5U;TN={ZE)S1V1QJxJu$r#*CTus$$Xa+e}VPa}1z;I@ptzV?v;56eJ*s z$a4STHV`WAUaS)GZdg8?$x(ku<*s`FVr!tqVpB&J<+WjA$V=2;MVpwC))VlkP*q*k zH!$#4R4@}whnS&XLzPh*nI`o%+7cN!2yhnwl)wNVP3_<Agt>AEH@)F`)O(C;% z5!k{6K^+nb=9T%Mwc{A(h8~-v!|md$A6I=%_L|YvWZlByLQlUudj;Z#`@uT|Vatth z1QJqXY37FRN#pi<&f>yRss}Q{;TUof$yU-OB*qT}_^@=l%Q|ArW{N^yqI(HeCu^Hj z)C0%I^gXD$t=9$w5{cBl!HQKgg)9)nyi4rk=$FPB#-S*i7cZH_#eL9(bZn~9Di8~Z zz1WV%6Ki~)Q>O3~8HyJmLSJO)kXG1VNMQ7}%;KydRSAAd3#Nwy+XF}GI3Xm#DH~+E z>at3=>O)yyv7<%}!zLg1Emh1fIXVZ{DUH23)$wm6p>C?As?vXZR&t+)dX!2Y>zCGOFZa~-U7@nAu!AM>)V=)$Jdw4r@gn~73zf2Igd>0L+F^1>`so^8%rb?VywEtwj z2@^EmLJ(`;OZ*aVCJ)_Cp;qbL$-!OuCQ(5;BCXrzx@t#OkvqRIS``VU+7482(;=(T ztp5@)vu~ldY87o44EpDf!u~asL`g!>`Iki0so48)T7JNAK}iNa1TF+fwfif4+%6Me zCHMl@d60b{ly#^{0CtqWUFAPaSW5MYP-a&6?VQjPFS5GH5PkVoBJGU7QD>6;HUqK+ zF+}fGKmUi^g5@!+W{F#RZ%$897NOjnXvxuwRPmRjheFNejsV71PC6>Va4q8KNYX3* z48>fDaA0gbtCT4z51C!E4U^tH|kD)@@X|?CIQvR^pCa zrHGF+F3Q&3+#R!rCZz?3MeHkF+=#RT1$&r&&k+P71tqZeYbb*~glSu0`;Z~;U0aclnfq!uP0Z zfd_#XVqPR36WpH57-Hyl$Mp#JL4NRKGO2WsfyE zP!Lr)c9Jyis*vxjSsjaWxP);jL^oS*4b8U8IwR|@O(Xn-1b;$3M0m*8n{E9{dE#uC1JnMbp zhSTNVqEh?tGZrK&7k^1j7{(AWNUei$ZufgfeLAil%(X`%lT;o@La9G{j%8%^v!nZV zh<7JpHg_IZO2j%DEmycj?Ha4th9av+v4odVeyBePz@3r|kPC7S%+5Qy@K#t>#lx2m z3MWeZa#1^|w6Q+fo0ya}vDW~aY`NsLPmMJqMa6P|OmGtaq6n5{l2<~A?bjAv^>%ft zzG2y|POXj06Y%CDHI6NnyK(?3n^1)%ic3Kbp#QOcUd#0YpHD+9S4sG($`5lIy(}|C zlu0-gV0B_`J$_w%n2xyQ@048kk~AljglhaM`lGE?+WHTcWUv?2{LTYNtb)#O z!GqFLv&f11S50@`arHXQe2cf8(-}cU)!_2qGv7}qTVf>T{<6{E&eg8F8Vd-6Ak=sT zQloSm%q~K~gy8rxlPXQe&A5i2j&j>a2!7&tUzDR{!@)k+pUa`}{K8LyWV8SF*Ds>I zJs+y0nOB;#znIfYAZ^N0M_+$MX@+7pr%tQGL%=uZFqfq&R3?6dJG$cBkUZ=?uwL*? zjLvDHOcpwqFMO||{55VudYoD=F3q@;0O@>f;3)IkS1gAch{bv4_mRBv{*kp}>DRi~ z+hKi8`YWfZ9KLB#{*x+qH~ybk%l^vTBGRHF3jA z1;zz>@|&lutTa{EUHc34*qgIxo6&7h%;L=Ck6ni1i=~%!8q@ehm-=F0 zlWt*h8Y<5$zqL~izFs3IL-LP3XvX&qRG?$NH;ReIEBQHjLhk2A)#kOInVKZ%)tV8d z{zmu?wg;1qt`x~lqp{L^o*AlI)qLWIx*Pesor#BMiNvL!*#=1B(%PJ;&)0ed(PX5~ zpQTi-T)p^pm%3(rw@T;SzJeWnS7Oz9gbvk(uPdb*Zx#;NA*+xbDW4v9Fld^t=aC_u zCPfRxMr28(%zjK{BN-LY6cIzv(hyDp0fhSj; zs+(HL;O5GUOt^ijra1LiZXwaRD=gA=>G4>;oowM45nVI!B5o zOidddl3ngPODSV%Z5-!<=h7C$i{Y;3~4Y3YEexVZ8pLm>Xaz9|j>*Z;Z)Z&GG2 z9v|_W@AeP|Y|^3sYpEG;^-QzsS0Lm18mUAv3Z9kIR!pg0POO;G!P9q&d;V9EozdQx z7NNU)ZVjHEYs6Vzz8r{5Bwpv30n#7Ng-*SHXy7n*43B09)b9?4 z`oUv~cMaOZm5b}Jd~tL4smU8@Fp^sIUkKF|uy87zF3zn>BM|*GERU+;SX7Mt72ijE z%D~XjLyjCQ&=8Ok1ycPe9u7a8oMM#pBUTJ`-rr{S0y;O4-w!J2_^!kMGW8WtH;{s` zf@<)XPF}8>hSJ%;8GbU82_;D}7<|SJn6@=MUsczzo&O6F^?<488|ycXEuWIk3(W6= z12BY0J+=FG4fY+GUS~s&6k6ZluG6Cg60we6>7rABdy1_w*xo4~0VwQ5Bs>V5Re!iH zyAAV=zCQ+6FJ}0j9%X+ z`)sNOFDvPM>6=ZyeqUUi@Xd1VOJu>t#sb?!$zJa7rk3WXJ?Yo3BZ#4H#qmH`Lg|EI z0|Uy*_7~Iw=Z;cGyzPJMDE91l1%N!e3Sv|{4f^pOGUhecAr~&h<}%90C+jbj1w`a> z+N`U*c|7t4T#WmV#NjU|+{$Nc7!)pC!eomrnb@`t6E?polfjB2oTz(|A#>`vJ%%V^ z_4Zc~X>#t+gUR%kN%&`?s6-Q6=u%jTr^OlFvR+lVubFFRoz(cBfe*@SjTZ&y!v{O; z)+*vr#^>-s%}pL${*R+!hyQ&d3zu*ayrgbt6K{rAyY<&XepF_HGTFj-nk%ld*)B|E3%zP zz!lV^$J5}~o$dp4_*H_kU4%&6C1x7F)7G0U`R6gz;P2E+`}o{i0{zcm<7YvwU1*9~ zxMO!e+#@;b{fr3!-g6}MI`l>HmZ3COR4r6e0sxGTfKJ8fIz(Az%a9rJ5lP(47xg0L z^I984xzRx@iiFY6zl&B8=CY2gE6vxopPO9Xq*+yB%)sJb2|XSnHlOt)JO-*Lw?flv zkF;+DJorP9A4eGj+`@$u1KNHhWxF9mkQ5MLQ?dl&4g5FG37@?K2JYWv`Pgy1c>2>u zvy>~96y_G?(`fO#_YfX8I`IP7)8(4aQ`h*Z)r#Umbft5E<>6PcJuT_kw?SX5+iR@& zw|RUNe^M?iV!A2R=Jby{Nz{2R48)43`|`yZh720hbo`#}gK``eFW#rZi2t}j-aT)$ zM8VAY2^CBFZ8Wqn+T{2UIf_Dqln=J5b*|rAp6_DOo9SO;g+wgQ?kLqdWv1SLL9eC5 zLWzh7x8U+VxVhPZ$KT^K)fUgNuoef^s?=(V7@5S|nmd1{;8TA!XrR9ssX#ibd{|F4 zx!*>x<`gl4a^>?fgW7P@D8Rww_OA4Fe4k9{@)FT@-WTpGK8h4Xj{SS8uqXdY<1^_< znn+{fs*K6T3G-anBklJ&*9L?NQ4a-)6~{;1FPMhoiK3KzTuELsalX@8soRUN)wm1; z+|D;s$#!BKOj?Z&)->cnSQ=G67aENva|+{sLhr`gG`UCRA! zxPG+C;PQ1TRACU)s}7zO)gR?LVKzC5*M@Uhe>+t5(_uEL`p-#^v&p-DxZj^2sQm7_ z1eq6VQY=3vvQm!CC;3x^!%D;>+25_G!n`=6P$o>%%WGr2n24#m_sS?fulPO`6K462 z=}+_;+=p1K=qZ&9PWGF^6Y`aQwO{dI)f{H%o}nUoho|wcsy-uypd4t43q_?iF;jyS zdi2ZXSQ_fbEuS%z9}klolKkm{yKWQkB*aBApPfa@l-P*sXgJksMeC+$CO($t|D;K< zt&L00kd6HNx2)}!?_ps7UV)3;^^}$_HI?5Oicgn66Obqc)z8xO;=*c6eUm55JKgXl zMRqmr5g%j|x~y=+&ixJAGO9nw^hEtE7$1>TW?Kz~FF8e&6lR#@X|Ip__cny`Q8a<= zRP-lQ6yQUK-?(#Q!p8T9HY9$u6zwRG3}?6cli_};|M`SU^{M&@>*Z#uDIUMzZ*fJ@ zrfGrG&X$G4D368e1Qk6cc;w#)NTaPy8JL=%3v=dv@$K?svlF2=-;N3E`YJJG1$16J z@pZ_a#bRl1ucWTi_#R3J=k!9J`ezB8U#KhgE{$YRr63#woYaT}4uiB@f1ZP^ywaT> zOn8qOH!4#^%-169Zecr8zXs+OZJPf)YxzF9b1<8MZ<2EdBcA2wP1NRp4V$j5C!9oj z0(A^uiDUXqU^y5Y9(X+%g0$1BnsAG7DpT{JssT&FkiBowsbAswe&ahK?~myIK?=09 z$Hyd}lhKObM`C6)O^S)hyZgS!Q5-_gc_e?HswU->?FOBJMgoM3$KyTt$L;J3t=cQ7 z*1o|-f-rrTs1G3nO@fi)5cpu!>f19i0u%YliCwhuUE6~ZYB`k*D=Hq>r<_sXbmT|7CGO*16I;3L5Ui_;m%%E#(YU0>|DGaVb zga{kIOApenzDqqmQSW^Tf%$ql$_0G`-In*AJ)4nx^72`SZL@x*G2+mzTM(UvmHc+c z`|<2gD7T=oqMwv%|DeJKy}Rzv$w|`})nHU(jmwC6b8_!H6Tj>F`cI8UqpHZQ#nM%9 z(VU)IR)9HwI~d89{-pY=)F#}URSI9%<=*4-cR|#Eb2`!10AiD8i^6nd2ACAKs7aE7 zY21STp9~S;&TD(e{pTH^?H%AH^VR*0{q|98to`@4B9$2)!B5SfF$eihdaXIQir953 zblAT~DUfmmfS%E_gIQJ49Ao&dlz;*`hlfV5uKb8s>Io&56S`}wFc`tKZ(^sbw6ND=xe%><}p#ZSa0 zP8Zv9$pN`y@zp-VhoFw*n-@CwA0j{Q+7O&qH*GtCoBhhx4=?eeq+uz_7kjEtSm~FQ zbn%0NY)M30E^w+QJ{DCYTDpY^3i#E8Wc95q_?Cy>z4aeEzu`Ol*Q*nmY;)K_e>&Hn z@SeZ4YR3URQsb!o2pzzbz*ZJ^Vsr&U{UM@68I^_IRYNo3Ts^s}!HpfjD1oJ;KqRb! zDPTUAAW-cB2XQSs6wyUhONsgIV)!QLNGvVKfzx_eQyJOE3_swP3wc5u4b5kng^NO0 z*V1AXC$5j4m(L^(X2;A+HJYKTEkAW`Xa2HP0wJ)pzoHm#$Ncovy?hH?C>r6~;lC%> z2p-ohTRtzKq&OV}Dw$^LQE8ulH{@0O&kO6dCz3yI;}11C@#0FOwI>@-vtasDb_EU~ zko(`^MpxFOUc8htQ2%)-o(oXW%ERoMgz} zg=+8L6?mR*qW@HDnsYpgy|mKQ><1Lw(fiCWAf*-3h{|V&H4kx*WDC3r?q~-!>=ssh z=@2s%f9A93d)MI1Jrl%49INDn^o+j432g2f{(p`?ikqlRp}6Ay!Ko9W(&hPdGY3}W z!vIM>oRK76r9xW%y#H|j69m#fSap^{vD9@%pfG80_&fK!GYIl{IzC(U+46|yak2+( z`&ahLz?Ow_=Ra0aU(ksCE^iMWpQcAG>umgjvffJ}9cBEI3(pWVL2GN=;q>#I{h7gM zf+j%p>%LaF6>8x|^L$_C;PwJF(&Mr__Fr8^mArU@X?TT!H*cXLskAJ12;OhnG+AZd z1mk%2L8BlMJ_Kw-1vv!xJ!HIm`Zyb|4dfz^#}7>k;%Q_L7#I6Nv>}~qB7!Ve-taiX zbO>+E*UoQ|q%B`>wjcocn_z386}Q(x~4OwRkrG%h-H3kPvtoomS0`sHqNI#k_B zg{i$3*<=qo9rgM*#SaRrQT>0dtX(VZ{LS1`MkT0;wa~eJronDyz_2U+!HtUx;U~F8 z)IKO{P=ScMNi)X2oJyAq`=>wWhIXR7_&77P-99YT)Hq*<%U&=1tfoJoV*6|Y>cbPC zk<`|cnaQVDVj>8!gG& zyuF}b@81dA!oc&n*J7*K?9Q8@s41FOzhNza30kRn4u#c6=2@DwYJ8~e{hGg|ykXu6 zy&L>ZN7%4iQ<)mYDIFZrTcI+Jcff?igu*uZ5~YVYaNoJS;5mYy^a|F0uB`k`r3&R< zSOU(Z3rzemr96(le_WCd1}I7R)QI?T6mn3UeVis?_#9?Klt5(3snR#$Oc=xKdp`f~ zb^iuZJ2p)%1;tCchk(F7IktZ&hrF3b8|jO|9|fP-%9-4*$%y$w@AHet>d(HH4T)%^ zKI)s5_Og_rDR==wl#8yC$h)JqosDz&%M>-Fz;VOX7^acc4tsrUTrvmG$H%AIQIE%; zu~vIyA|*`kia!_!r2!_pxqPtW0Q4#jqM*HCClHEC6J>7Ey(E9;$;JyOTG7*n3VtdP4(pS>|5r;#$*Kd0CJ z+n#>8TUqR$-bscTo7!JrqWyk%BRB}os>BKY*v2CRBd~xyPnVhgI4M~vB&~wJ4K4u* z^QSs|u{KO82oPA2UT(X~6bP4RMoRmSw-NpQk~BrN7baU;0(;=+e1vQeaSE5j&9k!a z)8E5Nd=N9?`5*^(do3dMj3-Z0e(u__K4C6FM%x$!>@Bylxk82{Vj+C}Fp zt$S1g6sE_cBxTQ2gOiP`UX(W4j-i{+BC!-JjvNd#yw@iKjc@#}HU*oX&+hfps#X2l z^W$k3-8cT|y3ORLN@uYsVP=;u_sdh1nqSog^S?c8kZIb{fM7eixGIG~)RX%2KHQ#$ zQ|zJ)Z%xWd1g-shdJ%no5V2>`c2g*6%Po5K1Dax2W*L~O015?K;n`u+pn%%azY=45 zDM%%)&fW3w&RL7jNUW9W%-H&7igN4UNK}oScg23JV0$>duwI_0j}!>UlNTYgo2xpQ zZC#H)bs9g4F>3GMjzas}!*Lo-=phBaMK;9^de6P4B#C=yTeBFfTQf2y5I@RE-J1`? z${q9fh+YEP$BZAEx-&aSV{oPH^t+<8pUZ<3D8qHnzyC%Ih_2`k*uRi50e_E%p6)cV ziERsX>*!z43w0Gi4;0>+qkM-_`=N1+#UK34iQV;F>D&!`=|TnU-b=crq=XLg#6CA> zhEY%3Iw_O6d1s{@YkY*ZP$t4R8J-dhg6p&+_tQ##S5j=T)k7xJaxP=UN!+m3xz~6K zdZ26#WJfO_z(gg0oqM)muP3Ei&RYoTqtN-& zft-BZ`P4z=l@D|1j&aJUhKd z$^u24R*WaW6ai;^S(GFl$^!Ne<6BHBnNT6z;mTALnw+b6_K7EnW{M*Ig76ar17r%Z zCWn=qam(h$W;VHm+wEG53$Qhlp<4JsJ3AGd1pfXcDYF#Rj|_5eKvzq8JvlTrsOWyJ zj%+KpOQ@v_o((cG*~Qjrp#(#ug9iRiT=lw~i8ZE*NI*}E)Uey>1#(4V1>2emBzT0v z0jBftzmNb^8yL~=?#RK^J_uO>5?Xl%g-DS@gcqVZ-TR77F>=uFiU)4jx`j@MnPlav zC_`2;M_D$vLQ5D%h5?nt*P5{$V@>5r2B?)^ZCX|_QP2Ogp75w;pKEU*BV8ysH(J1zU@Vpfp72&eV9(C*%IsA)`8sL_u{wmh*?PQ1@yK1&Y3O^7`GsR>oU;4byz z1%HazXw~(rh*2{`I8D_5wbq0^V_)QuBpR9F6Wa@jL?9$%6s!o<;ppDep%9oqm8X;k_7oKBwi&}{re z+0`vDJwT|25FJ4popm)x%S2T@NjS{Ln|V8)Cj8a;S@8reZp6ERY0ef!X;@$ah$oxP z9PtAHw)+PL7@j~Z-x*?{mW^)41Sd0!G_@D#H&!fz>jqpzsqjNkJutyz3LYfs;r6+x z9W&=W??Xr=exQ`moAUjfetGkI@BJCR&mMjNl|N29A$ROcKzYnZ^(25wE;dI$Pa9wa z&>q}4tWf9|)Qyj`HYvxMa$FqP>yEU{X6K={F~Wg1mUZ&HhtGJR(}@3U$uT~3Ch56u zQSUG*#pNn)W9@p@Rg!&{*K1(0H3Qp1TPIYGCE;+esHN+*{T`|rI+Pl{HFeq`nN~!* zHL%-;mm=}Bq}cG2BWv5!HN=bX*udU*1ZXHG?0x(#ej0Yd?N!kavPxC#HT;N)<6_zo zNdlA7uMXEzn2n{St6K@2sa$SjPSW%O2B)=(NH9vRHd|gt`=w(rba<^G26oIX_zUZX z5ZKyijFI&JKKP5kqE_{NP&V4T7lv#6@#(-zzqXKqR(c#x*SV3{l&GXz!OI#4*h*1^ z-$rXHg9ZGz2}|E_0kA?=u_Z>$@=HWD?W=(`g;>|6GAw!i_jEr9{wo>{JJBB;a|;y~ z#MHR>+p7;0#6NmP$`{y=x7U$(8iH#C5~l;eo*Lh%unD19!e4zq;&{=`bvG@X!a>O< zNzn^bsk@~~)K7I*2D%!nY%mgH)@pBeh6(fhXI9bHF>7lP6KTTVG(~oo@Z~XDBesfI zDxG zBnBSHc?tc>tBX)nJpWUL7>(0=(E86#J=5GwUxbq zzQ2lSEF&B^AiDm~+)wwfisCYLZRWRZ@#M2R$>z%r>sJM3$9;Hf-?<>pke3s9XUZ@4 zO5~nK*vc)HaIt)2Th8o#oHE5Ee{OEr&NfHww{pMY$SSa3R15B`U*F#KewQ~{j`weC ziCS+)YxSPI>#G9U!wcY~nE=9E!Kxi5o(!n@ry)AE1m^rgy7_&Ht_=Lv?s zy7=#D(r#Xyb$Cqu8mP%|A7ekXcCQ)r1{QmWWM_(9g!>S*enX*jT+a-tA-&=nC-q-i z4C^9G$EAh^ zud!~6C$v-c9}1}bZNY5m>(P_BJv$;uOi7Y9*T%v0yFmGZw{zMqx;%+*B}8bN22yax z-nVAYcY_Q3e8xvW+qu2sEC12Gw^kcFvuE}9+3>5V&<#~EO7+#71K>S8ZZ9(t3Fc$z zSgXBn|Jbh}4?>F(JRH#SL3^qAo}PBNzz{hbsm4~x-Q=W)G57$F#VaeTJ#?vQxt7$R zqMkHKkFn8PZ;sL4)_ zHTm-h&$k%juXCoR%lH;5sU{oGG6og~T6XmxR%I2bZ=TNDgf0#Q`R#JwV1r=l`r3)E zykunJ=*-bfwtV7JH(Qp+ck$4<|DcC}*5b!&f6OuL`APK@v9qCBM5vV#NFp?;3-9e` zbBGT|lrUs4wW~jNepeuO+KS?H4bKt+}6-xQdIh=KrHmGk%9I}+9>zIsi ze^2ke==rod+S9{~URXo}s1XPSTz&m#L-atyBAEe4%Zms${rc~(LjUt=rcDEI$NQ>= zqza?ysd>U5o=**Uj~EU#kPq2PHU#}2*oF%NrFX6BDDB*yI4`@Ob49;NFAUC z#n-(hmsyK-Iz!E=gqE|!_f3Yw1y{hduuvuAdj}w~7#cI~YA^|KJ4zJMEZra2OVv)0 zT(qAi6Qcf=N_Q4h;=O^BV0?}XVk=rlmA-}UWHgfc$PYx+6wy!ki4D{tKwjAb4io!# zZkhcFbnj11Pb%5f$mmr=fR464HxU$BTW*yzjnZ_K@?iQNcL?4ASqIo|&*e!0;^x30 zYJKS|5qfCP{7-5DAkN3LrdiIDz?~xd@waFQTH71bf_H=dqHBTn`r+f0xJX~l?qUDw z>8BrkCf3%S;+^AxwzvGr`gU>H4O6KeGgcHtgQZo~)v;YmKOjNOZUcmD8SJPmW7*nc zo|4Xtzj()4@_%MiBDhs!2!0e$T*6+s9N zJXvuPNg2s1@;7()yz9ob2MZ6(3iZDy%(}lTLt_{j_kUOw-MOf!7oo?)pm6nI+c%q$ zkWC*p;UxU6jVr5Mn~;(w4Pu(0^niTe-}X*s5NHE2WE>4h`JWfLYt1R6-7&Yp=XvEyAWUwFl@S=o(h{m6%cOpNtJQt)_yc+87j!VNAZf>2Xk;;+^MlN*LK#4x#Kr?n#k` zCZu1pa%4V$TEV+BE)!4I#Ea+<3d>NSNyKu{XvIfoBcs_5~cJ`TN5C@r#8g#t0I#aNzAir)Z ztBMgEQ3X;ta(-^az(E5kuP#M=ec%H&VM5%``HG5X7VH;)zwM0pu}nUdr~)Nix^v0M zk92Sai=fqM`;CK$1|~j9G(zlMSMKQS+NH;WhJTIIg3eA>f#63?d%=HJkv~QQd9maF zjERKThsM()XN&exyj)OW#Kt1FI>Li{6-AiZE9&P+V(@Z=3bfn``JZ5&_WpJ8APWTG zXarFB8TZrY2*PFIR4@0%n+6BlmhXk$t|lnOlRuH~R=?#+LyI~ZQ#Q8O%G}nCX_r^Wz4Q!Y@bqQMN9LVjS+^&10pOerf3sZO8xMApcRY!Ayw>@Y z(!o6hmK?RdZ!`?ulXNtps+g-SR*Yus7D$@znHYinZa%_o_ke#DR>NPTAfF>$n%*Zw zlm7pPBj`k#qN8ePG;EiZCFOUDB5CA8{-V5i_#y`yrmo0j)cut|_t+<>cN`5Kt@c{z zU=95Ufmd#A4e@Wi639{cWWmcxxQUS*7t-PINLnX>G?7q-MnP!P@OeoykPYkbwsb$R zrT1ukX;nT4N`W}DKBQ~OQp?{FyX>-YC%yWTfD=#S6)yhX5Q7XobAv3eh5NAC)GBXd zK-mMM$7%M-nu~YSVj;+r_XXZP9?|o0{dc~gQJ9kbyfw_bcEaC}0DGrpwP~>cqH!Io zxwZ%dTPg|Jaz;x3Y3(uO(`gwi{4K{q`r_#6!#%$QqK-~Q#evK^D4IiQQOt5R|FyjO z2r;Ba=@rykm|;X{oUqj_lYokSL8)>l;nek9U zKV9B=ksklU*@df)*V!JKI3TM#WU-hl@C9ubglg&dz{E+j3&N(Enp&KkW4ZaX>71IZ zqEeSJ@A+aA(=Fg78)xwAw;VKhkW-)+#RY`5d832Z8%6wW^wp4Yf4tYKquBj#f-+bz5%hAb|{oC#FT_xGm7i1QB}m8s1|y`{ZmL zvBae_E~brvb!fE>(?1iN*7Fm~Ho2k<1nDu5(ycJ|{tbQ7)$lh69|*a`Q$wQFc`Uuv z!WP0tDb`0t8cSAnEAP^oRllAjtjU?Ej9H$vv~*|0KvbR`@AFQ3!leWr!+J9( zAzB8P9e4kc2j*cG*0CF&QI12Z)ZmQC$ryIOW!#^=T%t|W8Z8WPb1#wPgg`oWmLh+8 zPdtG(!if{OsUWrl4|=!f^%B0LKfT<%{mb*d1>WXAZi5&aNAo=-^2QVE)~i$be|pP_ zS>Dj;5Pox6BQ(eUwPFNoVjV$kej`Kb1j?GTE_eF>e&Poh4UoyEd%6%im}&M(4KLWQ zI4Ged-S~@6$@0t#7BVB7gSQ;YF)X5CFW84$;akA7c35zDNiv%?Y?qQ%U~qAF$K?6= zQ`yD2s$@lAQKasug0jOjzgbCV7*mtaA-_02qVWyLUV^*o@f1kdL#}0wtE80>Czx7d zB=w3d(ISyFBPVfZAMO2xBI=4=%tDc_#j4PQ+Ig0dN=#+ z0QfcgF3Ux63rqUMI=cVMpV<_E0N44vc`=FhFT0w=_~fpSZ4J>NV=7mt?Q!~DQ)OWj z%m|e^vLMJMoFW80Uay>dOwAm@cdlV>wHe8j08>Vo&lkcH5nc*IeB)mgxG(1kv~S)G z#k*`@|MUBSl0rY`xMqpqM)udiJLK0TtKD`8e=WXP$R;&(Sx@7jl1MRQ!+D09QmpAJ z^dVEp)`~q#5 z6OV}og$9UBJ_q*#-~p#|MOzk8y7+Bqk%7wOWc>COMO4|Lu?`f>?QJAN7Lb`foMa`9 z-7BK8^+(^?4rV#wK{_nIG7JdAsbBARVW$=@Vo1Q4=8chVB3r`I%31y#w6gp;z)K1H zo(_qMX0QLFHuQUT)aXxCufS)tj^kut&d#i-9H`V9z~(mvO~f2O!Hubz6&+9_C)hk?Mja;(RV_i`o5N!@aC5VKgU*wpx9N)VOa_Gr7;$Rsj&JdS~_}W!xc1_M`akZ_Nx&90gXv22%uX_tN zzu)QEPgja9Ob#+7bXmAc(#BiKu#cN;_OIR;?O^gNe;|KYi(Y*8A9eVFN)audETB^E zh}L}D)Wk3{NI7S{HoMXg1Oi;8%U-`Vjhb>ZT!7QZQAGLO2-b;6E#q%{Q0do~w`tU_ z7|&q-w(u zk_}fzpU9EkNy5Sbh8L^HWvh4shiYift807$#HHrYSclBT{ODlMISH})t!*4VJ~jG{ zat!~0N+?CnSQ;r@MSsWLG%qXBxyBxuTv0f^7Q=!q?XZ=h=38MfX}MD?Wur=MTI?Ih zEZuL6p3ZE6~a9}+PBuY5M#*~Z4!qaK>k5Yq5jjOm*5o{1&yxfb8=n$S{zh(K-y zq|E_20)g926*I}^2g2~TPg0gHn|n2mumwLDpizNnEi<^YCQdz!o>7HD8N|>orRVz# zMO-{NdVx()kDv!Q%$I_LMW0jrD~f}_fKkIiG1pZJZ|S?ojc8iX3Ql^?5c%}^Gi}&X zi3MT@a4T50K?EztI_an(J@}2kvI2YjJS&Bvp?GpKrb?U?U_L;w{->7kTYE_FGhf|t zh2dztw&@<>(2Fe3Mu;23xMDe0pzN-4ve`9Rx#3mMC$Ha@h`9hUl-uKqd}>X~3@eVE zC&q?(O@YG;@cMZyND-oQf-qoUi>&YYEYX{x) zvXA^3$sti1+z8=m|G+Pm8L&Me2u+2W|9y0HPLN&={8RLcjqg&fBAvTd(_5 z2!i9S&vOIAh#pUgS4-tcCSUN*|5oVbJ0tovy74JD*ERGqxBWX5iH84Y%AH`UNC`xS z=2C8e%{Yrx*)F>6Spk#!D8*{6LsaC(=zW0kD6O1!c^5SsbFPazG8gr)z7Ca7$_qzG zkRn~*W4v-<_YaFFB!}esI?a(0UWf?6i(8(JK{Qt3#)@w4rp@%pCzXgHftpbCMTG@0 zUmYee5Lu7T33IN_o3;|P?)|h?Yd?!z9bIo1GZG3n1c29?^D=XQc^=D`Jhr$W^ALoR z6ECy4!m154lARojH^APV&=10p5`aRO3!v?g1&TPp(0Jwx@5VJRY!NnX=6u1>G7W9I zw=P*I7MO$~Wu-u8Q7R_ZW+<^*2dQ?mX+`y!dT^_{j?dSx5Mu=v%{WK|y!YSf{i*fp z^mv;TG{0{!G7v3j@+})HNI{2_&p_y7_Mr02vGK-nRf980q9}!G6ez6#H26G2k!Kr- zw$aG(FX_Jq88^)f96<^m$m+*!R}$7FM?yEtDC?>x{Z;S5fNYzu7j%Wtqbr8`lsnOu zDj2o`Yyo%ymr9aHM(z)uCVQVTH25e!Dh2A(J7nbOBz1A(5j7rDRN~$uQP%9;eN=|F z#P<*AIsZ5@?Fi&BuxK!fR^(r|A%>4CvO?^yxR{2MS$OP8ZKeC8bf*#&3mb!*yHU(MJL?4~@Am_Z?qmscffw(A>YgenJg( zA+mGmm?>^Y0B!*20HDgMstL?P!v*DhC0Y;>efnp>3TzI1(_OmXxz;&tmVUvx?|hnV zVTau;hW0Cpf}9IuXK?d$^+nlOK1H)(?QibZq$<8nO^DY;KO#jc8LKfe^#98LN#~f~ zoDr7o$4BSJHpn?0UG{idIoqey_8pko+y&d-75U};9i8|KO_zwwdTn8Ct>+A2u1rmJ zVM2}ySe~el&&U ziN59?ZDSg$>+zb%?NGL-V{;x4)48~lMJKQxYbqIlqqWcNajz`F#PY=)z&sUj2|EnqEB(%gKceL z(06IzTLnOqmfwu8W55RO<)Ntlg(ko4^@fwQlaoQvGQU|P_B-7z5qeqGLt`9z{ayny zeh$K}!%DnItdS*foy7xrAgsQn@gAR1GxH;{q~;8H(;)F&OHy(gBwu*$K@qS^y z%?DPoEC%?G5D2iNGq~t%?g)S#07fOqH2d9mUrTjX|;&*})5(l}YnA#qEDHJPy*fVbZCP&yv3|)B} z6O1+98qgA7d2JNmsyUdoCm7NJem1rFo>Q&N)FK~@5r5D(XZ!=ig8bKUBr5P)y;LhEiZP?en?E9*F`g1p7C~?433Jg&i;RXF4O=lUFMcZ{@ zK#)>OKm?>qy1S*jZc>o$lrHJ+ly2#gZt3psRJuC^zKiEQzW@AS=AOA`@3q%DH_e3p z4?E%^jF&n0a82E}HBlO8+e&S9Q7qriLx*y!_dl-Tg0|9!k!bIM-*P-wHOahn?x1KtQG zB7rzyp!>$cCNu^@<{2p>Qn3LbS3+z2PRwtk;Q;~gAZquek5(*TbVOA}A27C*rJzL& z4XOAPJh2Ry8cu6|-J8%yWrCau3BEPUl|MD7*LG4;>k6=o+Zsz5LJodo z?}RV}?wuIsLCp&Ak#g2nPksUB#bD49W|XR5#zqV~)>@Abv_qhmBLv3tuQeAJe{s6Tv^L+t zIW{lf@>;#{kOLv!-;aNcDWYgY%m|-FvuFMqJ?&^M##PY2=4N}(X@n_Ab_TDDqKW(y zSl>E6#09bBXiL~UmUme$JBoa*Wm|Pep34w`eARB%)ka#@Y`QiQ-)jQy(!QqJ=P$vxUVN!(w1qHyK+yJq@yA zpn*5FgLm;7c>s+62?IcFr0S*zR%mmiVS15W)B$2SOnNEUI8yS z{D94=(iZ&hZhhoTj-=XexU2(AL!BK>f5UPO@;G0bNmECZQ=AF4OQvTdrgATMI+tu=Z) z;#L{yD`~d2{UnJ|M$aJN@`nwk1C3Zz6d{|Xy&L;)noERrw2}5i?N0g_IQ^y)ML=0H z5P~@WNIk}|*a@F$L-Hio;Eq5H#%Yr`XCue#u?i5#ox@)-+!UVDrx<=OkiZ%rJKN8x zU72*SxS4MBF?Ux~Qi-r+RayjAuvahF-*DiwQ%l?CWWbpsCkj(WwewJ~gBHhw@hAro zEp?m^NVC9JpPK{H3DZop_$y-A>`GstVQnF=+dcqo$IR4mV*D_#yfMPGrf5dCWGJlb zZ_m-&^)m}c?4yhxO&0g zk3F?DydlD3D1&pKy@b6xi0tyJ636DO4r*X<4U_R0(&WA;zV-I}>)6ulVBgYmjniL+ zUqDqL8>>Q0q7}ss=e`PJ01j&qj~n?4)3V8^BGKf*pViWhe`Ax=+wAW;sX~eP8b{Wq zYAb#}u99P8tY5(={dQnsp?ougT+2P`%81{4fV_Er|EsyBMg${ZKlS4@J2xnt0c`~U zVx~4vfy3MOfwgKvfwQzAQhZIZwGHNyz8{^+G(;fMg_NHh)7x%PTQPw<)!OAL0PV%` z2~j$tE2taj*ulI0R{7Z{nAWWE50LPGS$!qo`J}!1M=i2GyZOw`56J?hZgL%oB>3Cj zM>w)1ZSJ(`JdqS_;x92NUJxvgrf;NIvQFGeo~VRLeUP0Rh)Ubl!0SPGqP1XE1}DmP z;j9OO#jQYOlNT?wk5^kk{oOH5js8!|?`}IidFlqPAB01*n#cvkzLi$i$(M#awiFB` zx>!F+;ad~F#G!zHc01U1j)SKb($XUtw-T{Z1Q8Pp5&ImPbHTscNZ^K{m`ZNG`-p!C ziw*6+b&8yUfh0IdF&$p_e1cIWWc)Yr@r%oeK@I1MizhUU5PA8H%3k*I<4`z={dE!Z zB#g8XXvH}P6;}KiR*Gn@SVb36gnjKNq5=6RJ!c(Of?mPp9J=aK4Rk9G{*^4NA8sz6 z+>3MPzL9g&KmjAZ7Dw7=S%aNZT^sOqyImdGVKFn@S+0654pqP*V+;=`$RXE$tl~{( zhFxBeiqPtvMY`CY6=j4kS)eG_V!PV zm2P$#EPz@VLj278PDK%Ji8>9x)VVKKoj}w)lv*VarQhh^DarD8xMp$WM?}c!!l3Ll zTaW5kkuW0>gWmyzsdSpF&7Se~$D$)Fc%d)ISzVQWdgya!#v>#Ke1)W6Fv&bTFT5U3d$lQx{5etX} z0NPS%cr@E!@3^25cR6kjhVp9IWu(hPTr3j5umua!Yw|QW%-_@4u<~*?JGfM2#uXz; zAu-=Ar2mLsjC}mCDDxaW*&`+I!DaLH*)G(;Cv<*oMP0rL2H}&^w5tO*Wjjeo?=TjG z93Sf&INRvHfX{3<7+2&+;SED}WHx-KC^Di|miVKzp5fMD-B0x<;oc# zSYKQgIXzLWoLyh0VZI%<6_2#78WD%U_-F+)8&*riM136Y}QI_0$%e-orWiq05e2NkI_}EKe+S8Wpe5U{tO7+Os*! z))w`cf@PrwQ+AqLp1_mr?b9A@z$fVrY?gTwukN;P%5#a90>St_bVk^B0rMun%q~dX zt06!&Dv{?qbp!S$o!}Y2GNg8_B$oPxfk8pT7r{59!?*XhdIlh?Opj~s3koRgfa}if zsVDZzUA=w_?HgfQX)Bj1x^EVV#Qp{+9k0vVwZ|zhyCzRNykD3*MyJ)T_6ALSpG)@$ zA0g8P(^Fr>cXI2iSNJWI-=9i|3|cY+wF;nN$fgo8(QWa7{DM?9^=bH5Olpy%kCe0| z*+8uKf+0Xn)g70tTl{tl1QFh?)-nF|-$Zc!zaC;TsF8GeaSImgz;H0SvlQ{OyJf@1 zX`p&zMKXn2G(+BOHRbMDwX=aDX6;xPd+M9vWOgJQgfCBwuO)Me%VWq(>VXXb3t~Gv z9Wxv}UgUbk@72LS!;E9pC|*#8hW6&l+RJMzU<%0PeQ!|#o#1B0g?_59$9M}dYm}gU zX7ut08Df2(w3S~}X*~P=dAYKWJ3DmF(Wp|~J4ty-lBxWJ$Q`46vwz-=n)+%WIsg=4 z%YhK}ckcjrr1jiJ79^pNfqCS`M`gBs;9~@};kcq4mZ<_aPXFkPLD^1-9+Kna9==3q zeD#FA|Ir6wqVH1{6v^(90;cF*99$NXkfIYe01D)f5(q+s67>dBMpWiF9MvC*7gE~G z>`h{lve1o77u~##Qk$&17qDu|glQEG`V^6&eQX)hLtiI@K?Vw&tP{w(A}VI9U?w0N zN9>H!jP73IffMvI{M1(%evQVEkk8dP9Ap&_VPZLCD|~nn3ccMeG$!`IwFk!Sq-9Ys zt9UMtb~GPzLj;_J#`1nrt{?{YYH}YZjbXs2Xv|&RtY&qlyedPibvl6#?$X#XG zCH?zkKyGz?J;;w<3;}a@u{}j@aY|PyBS(7XNGe{JE*YC&)_r>XnppMM`W%&LL@j#) zd>K3b>QMQUOVN_zJ>-6dM2U->0qo`mc#BWZu%EokDS6P8G@L6FswixN>huZ%PY?Xj z?b>#4&G1|$PIOuO)3}Vtj=d=`0e@sKg-&IL&B-kr149M?dt~_h>&)c203B&zXNEB6 zEvNf!dLAF1D0=Jh(wr*au3^viD=a)J#aF+^JFq_j}y z6Vy~*a|e!?l97g~$_fP;24Gz@^zpC~5GnVhLo12>Su2V$P#RbLC&K#EmUgU? zs+El%WyYPmHPW0ApO+F-oAa6{QVvEq+qQ!FOwirco>?iG*NqXx-l}L7m*$5}N;sUQ z+JX7Fi1f9cP##uFPI1fm1}<N}je+Gz$I3QsYrJ==lj*K2gzfe8 zboF9$1dfAnxv`mzw^QrkY>&X;19UX?7Gn#U=Gx=lIXtRR&_@0G(N}!ym*QQ|%?B

V|+jeLV99KIhW7UgfC*;HeiH;j=Z=S3yByMyS1jyz03(uU1Av=+J zsMcTfSP({Qb@aN+H@Mn*hLT#3H0C@NPhx&P09+&F$SB-koVnIGSlw2N&`Lkj|v+fOLc}zIqA!#=m=Qr;oX1%TS&ko|Kk#1oAoE zFLJ*lley@1Q1_yMIcTy0BaR_2I&|S!64knh*wsES`%zQ5=WhSr6$#;Ufc< z$hN4@SgmVb8S5YRNRW@_N2#STSSa35X5mDW327@MXO=IPUZ9A8sQ6YOmi-;58OC&; zO|_%25>|w&q4Zgxn{l@i`To9S?0L_ov^J}*di}^G)BV39fcRf+Z$48d@-~A`*XXOi z04BmcTDyDB%{O8P{{NeQ*8<`A@lgwgc?-wAKu0|T_IhFePAM5}!T6hBd}AVn7BHGV zY5`=^3h&K+pXsuiv3;cRA%qENeKI0M%WT3ilmskFKyksY4l%Tdr=$xhMS#|SQ9hE* zyl3ytCkBsTyIQ3uLlVV%303|@%_#T7>>e??J)|HISN_!ez@cBMbMf2ZNR`^0+qp1? zJo#WOv-qZ3Kz*ToDU5?)>JjG+pBvProm6bM1!0-NeM84f3`g8kwwYsk-~h;s%_#9_ zs|6N|R_RSU)VnlflAT9a)d2R_nx$aPu-^BZUiVXFc7*^l30Zh?%O^{P47jHX27$s3c)N8$$ZRHpsN9FLKV!2*ilYh=QD~|L zvR)NyV>BlW+1WoM+Q~GX@OaA3?4jK)G_HSH-l&84|G%@FtDxTTZIdjA45$pzt<6ld z6F~?R`#i z7SdJ1j*PKsUaFPjXPOGH78J1HijO5e?h393x(DGu077Q_Ri`ZaZSq^i?q0i zCVN3FFh7IW`9qOp-cNPOItpE;)GL*qcW_^c{1@eNorA0gD4TA1HK`n^iONYrZ(@ou zq8f6x#!5pF!#{Fil7$53tMY53n_Ruu#DB~WGu}1Pse+QQ*u;f;J*UE=!^-|7RQ!^y%}18V_CT7bFQNOn%#K=pSs3cWsj z6^w-&vWvZrIS!?5m>SCYtm-BNPo|Qv`49i~`}aitx_3=h_(jDBtWm-Uqe-l3pz8zN z5+L6U1M~uADOGFn-3dx7mJTGu^L26Y%V>oWusG6#DtTZ(Q_qb2$`=q+hqzqi{>3jY zJo@)^nT@?}3K3jNY_wW>lcLXz`v%xmG*P`_mHl52uch92pq@fIXkV;-1Y9FoZ2b1G z-XBcu+}~7nYpadqJ6yKsHtnqfEzCy&CsQDJYO8DGeDd&ed_Jc~l;%M5)?-Si?x^JO zieH6W3VGcy6>37Uk5Ej!Sum@ZufPLIcR5%H%mk*!4R^M%@Dw4naSC7nhn0F`x4}W! z13Uv)a8Y5W;$thR?rPIil!Eamaybhd+Z|pbCYS5JNn#A?$1M8y;_kS0zpwL;9yDr7 zEyRs?ZbZ8|O5Hv^PcQ?P9TTo01}#PGyLXj8)dysz)YFmer(Qz3M3m`3!LFK8maL+v zd@=3b?Rc;sa!*b@Jk^mYy4J8_gHFk6wiF665y1sH2_Cn@xX-+}jii+7_LUwlzd*Uu z+4#M&I9``o7$BUc8aMaa?t^a`yf3VbXw zebR^wF5eqJOWmG8@H6hG^b|Brpww{nMvf)Q?K^HLs88&UFA4DY(OP78?C`JK2sCO` zNbe#vQ;HY_`4iIBw~q{+FmxQBcY6l0a+ecqH=zcz=W}_@;sm1h=Q66%@BvL~BGp%Y zI3LI9;2K9IIskV0cFJq>`gPMHiVCz1BJ{?HCumYXQ|KDfpov_-rNO^J2*3g`nq*|- zNb)j2l5q+At{4?jfk*@fSLKqhreIwT5eYOrt+X0WKEPINX@go73?hpwM=+CTo_m8ImZuI)r_GZ8 zv#Go9_vfl+k-pxJOHK?Q+B(_CCq(K#&gFK6O;_@P^JN?@qnONqBVg>Wx?G(cf&Hxa zLQw-6VPLR^&fH<%!0}{N5je~D-RUygv-UPf57MraMgtoc$QiSejXGq}eDiftfnGHfcT{ zwbNk4PO6Q0xjQI9ZOD#^e)Jv74S=Jh2_1Z)$aH**AmuN;sJjx>_V8GceF!gPnk}@0TPKd0)_V#hY2lNq_dFz+O2ATd z$fq>)d5m_q{C($+spE5>|8b}f^+X-aGwkqdjoH>8ebsX-A>9-H(DIJmc+OO%4nw-B z3DyRel^BUO^P}14DGWNdA?i@AE!OfeIDSIgacs>ln^!&m_K&3zKXw0CFjkE$X;eAU z#7^TK1zXSAOnAX2|ip-Tt!{3?lBFXZGB%Tzt$(k2*ngU@jMy{0S9C`3ERyy57*y#3Lg@<0u_^4eh*h@hp2!AHd86Uo)j9#P^T zi6b>F((V(iUPZKHjMH8l2+d)8vtv`9s_9~DI<7P2dRBcdxYU#EB$7^&5yN;V0~*ar za!Qbo$wBI9tK!y7%h2H|ilo45cpS-;agdb!xT9Q@ipPv&ND=jMH1OyF5Cc79{lDVn zopI`D#ge5QS_7Pj$M=iYr2!fFFhzf1gl(^;l$9WAH_x&3y0F&a?r;HqKqX28Hr8mP zPrb!^^L)7Xe~`l{o@(%;To09?MY?=VKLy%A-CR0UHDGz>l!?|2n6?`-#~L!l0X9yC zIT-cD@~6c}IRz$)mEI`G|B8)`mjH%xn2x5SpoZs(xp~LJ{2w8Lo?hSGrh=69f~Mez zC3aAoD9ZaQe<~!U4qVuH%5ZvAsXmvdd!Nw|xSQO8v`(4(kc-)qew*iz5e|vNF_^PG zv~>-CZK<2%MLaWy_TMc-ij2Cp_Y5DkhK8N^_UH_aoOJ>U&2c(n32=r9OQW_yQSF&K z2qP$!V!t%=7HSheUL3Ea5iO-EvX!K@(74hzFd;FE-m3_gjV!&VM|fj@+k(Cx$0Q;) zjlmucc3?yF3;h2%U$|PDS^R@85oRpSi$33X10ED!*Ag$-RxIDP<5mInrVOUf(wxuX zO8!uQIO(zCDf8;-oBTN?ymvQNI z?PHNDAk?L_X zsnkC16F{p`9l)k_b4h0wUdLC@fM)zO=!su!cBJ3zDy$Pb=d%Va*ZvAtI73ifijk!5 zaYy6}>TUpx=?HMp8H~X}QV~dmN2;&n+@?AR?=&j}495s$>0^q{b-=ar>O!xleI6y=% zMwvY>Hy+ro>gOUqYQqc=r1BplY~Q(1)=>5$$!Dg8y`<1|<(;5`LtDO-nMQI!ZNN^# zdz6C$G!u(hOEBNTKLv{0S(zbMKg#s4W@QYznVS3*Yl9xrXrm? ztHkV~%2&zniwuv}KbOb?TR^G4IeYh0Vq>3<;pAJKOYyeDNH`l%XJP69lJi zRM+31J$p0BatEZM0ugkY_my2oJsa0Or+GaTKHamgFKZe9ntrQv-}>^e6KxKDJR4gf zm_E#hw{7)ZclUK+n12ab#LOqA2^rv@1IBjXbOVq*V1^f!Y@7Yc#4se5NS6$T;Mk?h zBTXO0xvi!)t)QFq`H*P%)Hs|1axuOWp}o22M<&F06L5C-2Q$#r-s+)mzizVihr5Cy0JA!;e-H*DAFKjNt$0{j|jV zYXl-uLXzJBasaF?$d64js~f!fDiRMa#*r6y?art|OPf-5rD(S8cle z&Gi#gkbz%0K&nuiP1l6DK$~H^Vy5XSc(rLtUAJ#I5oOqe&%nKI^3Oz7gtYlAa%R1oHmzjk4ViFZ&i zw*=R(1W$24>nf4x7hA|pD<|KsoK58^96R3c!QvicG6E<6K4 z$R90FbHw8z&|X5CRVfXGc2sczes(gJj!biyQsfgdF+v~&APnSepqhuq5??v8F1BjR zrHTcVMUdmDezLio+C=H$lAFAJE4Rs_5Ws0{K}0?v44QOM=$}_vmaaNQH&M8&oftxu zwcH^0ON179%Xbf*Z#P9HAgBsUrYr zZ>!sX**dLwo};hO(x^~vGAHna(fnwc4Aj|4$M$)QoNTfRVZ0c_``&kNAdVkiPX0Na z_%oi?jN%LxB@irtGuw15^4Z3&_hSSu70e*+BPZ_$dx5Uus$Vvm+>apDf`hzxLquXA zAR$#T1DEo(y79*wd>}gv8{Dk*H?oKU8&l^2s_jN=l+x7vB9?5!4~8%}D73|Mk7=Z& zC~WMOmL*AanCb-yJtuCBIzzy^cB{BdbGmze(Xl^;mU^m+CrebEU#Rvt@!T zL801K{6ir$?AOv)#Y>{)VT`&D@?uzIEVL){=M4Vd(K+kL{=1;^s{%f31?|d0{&UqgALSozN*h6o zSYt|l%Y`6kpcwWF$fVB0x4FW-AI6m~p7HI`=P%~p11^IXD&O<)S1|3I**f`Y9SfD? zsPS<^E6S}RKiEl+O*@MeD@u&iI1c(N(X``)Rpx~=?FgTOs0;6yR_ zcWo8)6qWsnKw+ShBSeWJtmg<8C=@YWBzbRhGX9f8=j_&^KBo}mDa8G;d{*{)_dsWA z@w4)1>6>({KUwCut=4kmhc9VEW52)Cu~3G$>qa*ocTUaM9H8-gRm$_ALlBs!Vh1EF zl!qhe-8C z<+nJFFxRio9Fjoq-Dc zmIoE1QOfeup(1MtmrgLfg23+`O}SK73{gcz%mj^h;((=vFvM}xi6%J)!uf&BA8LAN zFpD;9Y{(IyK_Esc4&!PJQ0Lv1$gWJ6yczJtZJ?n-Y4sFDaa@oZ&~PUAU|uKx`WzNH zqjS8B@{E`Zj4f(l%l`TkY0ul+Ny6BM*F1~z^N9sI{{Dn9A`H+Ec z%|O!BLdgxJxpX;~S02@%_|)ptp#TR&frHmi?>t?X@lXu%{FYZH=u5t>Iv68Sm-E?} z*YAWH*WA4#=52$Li;7A_=&}Wk&i-p2n)*W%FpzJJM0J!#T9hU5E-vl|%$hhw3r`TA zPhc|vXa%cWPLA}=7Mb&^$^M92dT{?GWf-rxYi4zCDl2k|4$dZAKfVpb8Z5@8s;J#M zy|r_znc>YcQK;)NynL|tpoawe3+wrIpy^~1R*VHLZS`z?1BJwyizncg z2@{V9jAt+(4?zI~B41uoqlv89gA9U>Mnf9qQ!`o=L)P4@G$td~n-vel+#t+8PKJu# zr>v{b94=lgiq|+7nD=MI9tC zUOYY^lfY%Ojf1vs>uzS}0uI3mkTfkH5VDHXK6pF6KfBw-rx{wwfJTC^TH~po$IyO; z_^MYJN=$OIWiOZ0%7OnsLSO#DAF?__8es9K$USQiDY#Z;o!CFW64aW#p7h5`kmn#U zo@nxe{VbJgd!xrT7r=W?NX$x%5Yz<=BlOP)ilBnxk9E}hf+fGXgTD6&7;nVfXG^%B z1;sHU_-m@Ujcj}4nLsrls+l)Xy405G_4^Kyz)Mig@cmfEaMW>2?6M3VfN9^-D#ey9 z^WMmS%ywpm=X;IW-IiS22N?X)!V~LfLKqb2Th)Iz$iJTrRH2}Rm16wA1Yu*RLG9uZ zNyZjheLGDTXp;f+Y>f5O+q1h@Kyyle)y7At89zu9$p4S(I%I`TKx`EdgP6h=UtgAkY;>e+>5_AND<-1ECmf^h8W?*pchp->%d=caab@mNa?D_VJz^E{IpLmSf8W#*_%WV4v zQ_v-s0NaC59rnx2le@-?p9{a#6pEWWP3{ZwUxHj%Qy6{r{&*BA0|&i9rql|}*zj4z z0j!YB`X-MW&RT&oYXOw{OydIi_mk!F`HwaWv){1G`2-nD_dDlwh}fpIL=j71g@s^V zemTPWrQDMdJ{ne@6gd--@82rEx9Gtks#06AJyYuxo>WYW zobTbZL(^=v=fM4FTUeF2RKXdf5hu3)@Bn?c04s({*0Y%Sz8;v!0f4+YE#A+RRTo^1 zEVMs6f0)@|I#dbTBu%@s+^nqVgf@->maZ%#v%L$qpM5O1#zrVLhnq{?n}fj#|^ofq-5rN zIvQm~2uQ)s{X^D%-+2V%OR+V8_zGVD84AJ514Vg{7w#6`7M<~sY&kCS7{f=2R!r;< zWlW}d{=!Dj9-^-EgINSpip)Tl%k4`PNi^( zWRxwL!(WdF7-?1afI@jnZnX*X?(4fVFtf(_5ucom3 zi~1y|;W&l(ivSFZY1of8(vsg@qNS5b0vX7m1eSGU&AlOX?eXF*`w2gO$eVt{{DSc| zy+|}_gGDwN06+x52}kF|L>S9g2vn4mB5q%>1bGhOp5Xq4Fr;l}z19BYO%W8-g~fI_ ze=x_q?m-+;PlREmV9;sSo!?9VM;wwt`tGcWMk+~xN+xUteM|8;_?quojyz7g`N#Y} z1VqH#3Gk^6sre%UByte)(pAu72P?o{qA8pVai9kQ{?BMLd3%lcR5^fP0W2FXbX>Va zvc%G*(X_oSPUl(JX$cJVR2z|!^`)u@M%2fHw@!qUNeMFuEBCG}l)L;%w(7)~Zp~mde|SEQcYN>y=4*JWrS^x|1LzH*OC5-lA4>*S zoOuBi2>3ruWbwpf*@Wy(5N+LBXmzU7z1p6buxj7-$IS3|82-7Y*=Dr+qpJd$X-1Qz znqPDr*ZFbsUTUnY4`c+9$zuYln z`(Y$};`VU6{=3J96`e1NW6gbGYlP!B`{vzM!S6MB7SOV)*+jd3z1AS#7_@gSshXbwq_)-Cg-o@nsT{98*#pYQp`7e=D z0Jg(G2$b99eF>^~vi$zO36bh6H96lUo>z(kl4_M+A9URjO&8&ty-pix+3iT0v5iFO zv2NXHQawTSYDb25fyM_0Zv#6)+z(0=3#VB}k$p&SAv;A*2W}rx5$dQ3$rFlJqyz`k z8v)c{#n|-e+J5BW$1|2)$hq)DWql!7fq*4kqB$<5Ou{N`3|L#2zt8nTjUd#JPE*@)9)H3eaZdW z>&(7Ng;u}bK>WQ^ZVQ9u{e6qv8=AFX2)Ku5A0YpQ=w%N}2?C#zndI zn(95l=>s|e03qv1Oz~&yST-4J$|-bt6w(BX6;tq4X4&i;nqs;j5%Dvu%2V@F$|ZOO z;?<$?rsr=&q&#Z5cr`?Y6vc|Bq{ZzIewrw98&g59WDj{2gp^CFwJbg-lB1|jpKpZ9 zF1yZf0YQ5I-k8AkiEq|wt*^ekS<8z76Y>|VcSH2=Tv5zEfgO||6shS#RDrs`qZIaU zqvndS>(ZxrRZmz*R$AD)9ezS*2y8@wkEJURSy=-bP)lZhl@>yQC8crg9wyaC8H^2| zAR?8--HX94;cvvUR20qA_$XYkU#B^eS3nk%njmIp{Q`)=`F}vn)WE0&G1~x8p7t+N zaQ|CW37{?@CjP0yY*PW`ii%SU3#W#!RfFJx%va8HL3XFTYTe&1RZm*1SuFXYTh?6Ql@tVSh-H>eD&rNmm7W(k0n#JS_qOW zoe5N?WEjJPml$DLrOv!qP3C+52O$dFnM;i;pew zfW8HII3rm(_#g!to0$%{Ud-UM_EgO(M!S8LQqN~zM_6vXDFGsVZ2v|2(1@$R=1lYq z*omUrTH`vxwWlabm?~O(aHSp(+tLz@sWp>dB}qO_bk6I&Qm=KDuNVbJ^bEqMI{{$2 z+p=|ydBK|J6ZPd2)`OiNgXT>m+O|+b(4DCn0!0_!z-1yTz@`jv1Ee6(PacN?Ro@%U zL{Q2!a2szq`lIzBHWXGTe74lvXF$6|0$m>9KWs4&aPpOCBWiWu6O zn|HlTyD{~RnYL=KV8A%~9P7Ct1>B*aB4s*Bxu{H%jd^Sq@r+A%7SQUWEUDWaQ?;n2 za8S)LZLThq7gU5e5tdGurGXsT3$ExZ4W_DkgFc>*Zq)X3c^qG+?Iu<`4$<41A7+#N zu{iL`8jf+QRCXBG(=c>54@(?hJFniSqc~8&%1TRaE+Wers7vyr@?UX3ps3wQA$IqU zhUB&2N=Qw+++9>nf;~4yu1f$0uO2N5mL$Nj4&Npa#GeAGsskea$C%xBn)smnq>39Fc;9< z8vjmKgER#IIe|CJOXDr-RB_>!xTf}f31$C0qY+LSfjcaJ(v*_Pdk+&q7g>%MiQv9#Al8W#7-5~N)wU^imX8#_0`L>J5xV7zYmtKZJ-8{ zzygKhI_|(fESY8z3rZC8C)h9?Vu-N~xQS^6R<42AJIsYtsQ@zu@=g+xQr#QpFIdf; z-^g~)T-fg{9k;2YY(yHDsC6Q?0Gb&dD5u)20${`$Fyubr-;HNEkNpiuI&Tg@tB=t zvJOs?xFoRdGB|cQY0HVAR7^4I_p3@Z4r@l+>l!;i?cJ3DJ&&r@YTxnagyAH{G^WpN z0Y9mz0kC_k30Z!tNbP;kZk6WU|BU+oJ>+8{DI;>1xK;WTKG zdKIlH`tp@V(ZoM?`ei?Y_?~U)>iwk(;}WU;6qBZ&ETf+E3~ng)j(M^__%<^)P^J2& zIeeQ`1{|A1X0qhCm`=Fr-t&9$gQk-mD&666Xz)ZhE}mR1-BepYfMvj9ijZ$CmdhS{ z$zR+%XeX89cw)Z)F)6rM(b@sd7nkd!u}iDYU?0~n4vQ|24aPyKcbPUqSUJrYH?l-^ znTITKtj*j_)Uxn|}= zPjFwqyj5?rBismK?IHj6{)ip(U*MF!zMqNj#ytpjQ40=(#Yd zhG{%kWiLp&S}9lfv)z9quF?6Xh)0TBSFK40qS!Cv@VP&Ejql?(iqX<>1w;tbXa}ZQ z>rUsrR!wyEP)J{60#_~e7HOUaH{sfnO?fAgJc>QtU_ZlKl6yfvU|8v_H>d&fnWj- z>)5hHel2oP1x{_{7R`JNQDOa7Smh(zJfvRzs4&L=ff!W!O&Z0;)F$zf+Z;D5!2^bd zv)wUZ15B->zwpxQI!dUJdxt8PNlvSCs=5<~Tq+K~gHaoc(7&jGE=OCAE*>4n$GhGcc4En=GVhB=}ZKxD8BcvvLa$l8Qthwl^Ii1H!Ud= z4fN}kf0lRw3+$m6%rEd{kog3Q9~kJ>M*t(#II5_pLpRtj!Erc)`}H`cu(dI6bwOly zX%m4Tt+L`=f2UC@4qF4xKo+8r-FMYxTo{c+mxHOdn1%~Uyv6>D6wStUYHUi1b230g z#D1v;vSwb3d>UU&J}L-(hq4*xkQDy9RCOdq189e20%62G3vQl*dXcKI@eY{2FDtO! z{;`Uc63a@m+cq#Wd(ow^{kaK3C^4@feJ9-hbdJmSNgP3 zLP6_7nURQmwo(;C{S?jxARy+vPeh0mx!W5Nr#mKK@Rgb%CFH~nbmenE$OFFo?zAG+ z*mrd57&6*zx?t^Y0@(KhLfxV2ZP)$kIYrIIXr=QtsTQM|qE$0FJg4PJED`dm*4b?f zk+7w!O+VySZGIM)hqQ_pFEriQF7LBS#+ueOy8_A-;DRFrk&+SHrP)@|X13t&(Pt## zek{pDbW_SnWd9PKKHC0I1^`AG^EiWq;1&Sct6xT%WFTeSBuJkKQ}V-XdQt?C^?*W@ zo8b;tpq{~ZqT7GLizA4QciUMSXW=pmk!?n)deDW76GNRn{{Wkw0}mLT$LEY*XWTlP zmEpzr6pN&4%TQ1(Pm&@+Dk_lZ9|@y;a~V;S<%&|)#?fwf=T zT2TbZ6?@=0cmWj8sWD1Uc1i#$oVw{7KS3Yv1(=`y7BxLBZTF9YAp%JL1h=yhfZ;c(>N!z2q6A5ZXhu07*2=6hAJXXz^%Y8F`alj=mn_oq zL(g1|fL)P3pK{^%=a!m&gH793MgPw>w+GJn#=YI>V11#X6w)_P=z^%V!+rcnF*i-BU`^RCS)MpNlZU)0JR1o&?~axO!# z{p8Dx#AoR_*JH8V;i!{_q?2VA`T@yA{!cn1F&{dQ&lTtL$Ul56wp>>*^3#Exp56y! z#din++~gl>t=uCs-vPYcmiU`ZADb0HNg;N#0acwlT`r5dOQh7BcxTO?05roq zAe5yIDe4FE|E_iIf5^hUn06#NWCSNk&w8Q9*s-_s9DjG=lp@Kos@iVeh!EE9f(U4nt z`>-VG?LbROf6dRVV$Fqj)<#n`U+39Ar+)Em{)(M?&t*0rUH`6@&!Zbwxw&QYXx z6*o$BxAhQjT7M$8Pcew3p$33kjRuSh_kI8{3)BW&&9rHDu9q4(9~i_aUba2~Rw=$U zgU;7zL8|5QcuY?l(1Jg^7<@93{UGU9JR5%&$`$8&m0Nqrso*p9mwcf3tmQ5Bpuucj z3@}9z_zCs?J^QtO@m402@ypykSh)CMOZ@cM#KL`)^)n0PZIwso0F9gP3!(bM%R0#T zKNhW;CUQ7~D{7eo_1~YB3bx24LH+r%?Osf@;n3rWhz^Y{LU2a zJOBKyMFp$Khza|!TIX)ck)-78_%Ja z)xa1RHkQ^ovok_MNm@(i+W%MYXqL}oC$RdmOdG7MyIdM7G&^*zh;C2B9w3wq`o!cC1ozmQHR`$%#zuC>6F8?|t=$jTL+n4OmdQwd_ z*ekycLBCj5c`7(xK>^I($;vnA8~#g@W>ht1&t+T8>{wlPyCox&YR|7+;>G*W9iTb| zYj?qc;fQ;*OpAwIjR-bYT9HJ-esS(F3&&L&R<5826!|*<<3*JHkSzNP#+3cTm8rBi zfXaM-x~T^=E)I&^{|XY^ntP~v20_L=B9R@eN0k8qDB0VBNW9*Plh%xm9@nPRC2bu7 zFFpPfI-U3S9xa@nH9G5RYb}w!X>SBWZxLCkujjDKb^o*<;+R|)m%+Q(4CH6j$u|4c z3mJ^^&)AaGK{d83jMrFb$E7cMUA^&92o_|VbPTLRQ?EHQbOz}ySYhI)vtEqRRHneq z;I6afz1&9KS@qRB2fBCSD4m9kpm)g)#01EG?Xh>ykI|nK9AyXv(g4>YDRG|P%eyki z32fY;=6tRRS$3c50q>Y{+K-;hO!pt@@PiLBF^;@MLl`qxWxO0-Z~`qyB)~fjU~6Mx zCl!pa51X{f2)Qe|rOXVxI09DA0V4VI5U1;S|1Y5R1tdhkI#sRZ$N>CXrL`+x4> zd6lEQ$86eScxnWhe4sN4DkoNvp}z@ob?smVEV3=iug%H=vjA7!Z4qX+zk0#o{hnMT zz)h;hbJ6xoW@)xAbGypAbvtZ|1}>I|=ig|fqvs=hp%p>6f2N0>+M_x7mT51m9;+Qh z{`^gNL5+EBGxVh-^#==yHbuYRF|wPT{*}gK&%2Y!{)s|TGIsrWiRXloX0ZC%D}&3* zox##aTX)a%N3l1FM0cPzYn?Y!NH1B+Jq1(BpT`~?ww|Ps5C?_kG^q0^4bNsE1W0IoZQEP1l?HW5a0+$vcgi4~;`N57&u&hr={Dr>iZ+QnTF? zyr=1S65(N!ZbLymO!nudWghi)c0?0v3|Pg~3+@f!Weay6r+dUt=Yzc+g+(4Gq7!RF zXsZKe(2xYMorxE9wbi#K9@g`(&C%{Q>R8#|BRcn*tvoVr-uQ}i0l(f^l*7??GhuM! z2M%`Mw_%mu8K5KVr!2Yb&%7k(`|UV!|?H*kO0-gl&@94LOmNI-2tCYVyC|2|>tn zm4fy@du>)>7|~SX z7H5W%Ph`ofL6Ur4Lv^xjwr^6~Q%wMU921@EB<|D2(j2$-K~oSD*Q=PYz(C&-qd5vX z>ZXe;K|9u-HY}eB27Ol28H6xvLKAttd0ro%E0fZRN=Qwl)#lx*zFM!|p)@Xqe#apv z9*WmC`ZJ@rYsST>M3k(cBZngxq3Ghl_st2tjJoO}eu+7+`^GFCHhkI!JQ-?? zs0a(c)f@X_rc69Dr&=d5>(sm zKD-um<X!ZN3@J<6{O$<%x5Z#>T&z;B5xd@Lt+ah|TPf^2 ztJse*x=dOshOpdH!}onULe4cXf>aK!w3c10PnXOt0*0D%jUPX1Nt$}IVzM58_7g7* zpx!u$-n&@I;8hXqFw}-7&#f+valhVw%fAr1A^e{3gG}{+qe59zK~Daw<m zCC0u^oQ<_pO2!JSp?O@|Ua5UJquP89$0xbILONu!*RC6Xk*6yX5jo{&ElZu{5r9$n!sNR=0}6AYP?k@8w|w=FJO9@MDk};|^mEatTR^bHJX! zPmjlsbu5y&dc~%H2jd7OKI#~aS2f9H)eJiDae6o~`C=uj>CS}=ZV$`caleK4Z!81m znae*NWz4-Lpdjg_&@@`=eLoJ9chjc_CrN17Be!C|1OC&OT4z!5Vc>Z8`nIw5J^yQ% zK^uhEa$V|EFeX+qb|L4Ys%k)?Abt8VRh}t@M(^&q=tTp*nP!2ZZ89WT(32E zMs9j*D!Ofg@3@rYV|b8fA@07C{ki#cjn46x3#2U1ev{Kf`MRbbSD6yncf;Mp<{;2t zTy)OT-$r#!-IkeBpOiop@k4ouB)dw)(sub;q&URyB>K*z_)Y~SWN+8YtF?g{Uf@{d zQ=>e@%!e3`SZB7c04&D+(43xJ0c_jc7s!vqK7Q4P5<{4h8-dC*Wu;^1z8L2P88JoK zupjA{bs|mZTZdXMCf|c4)~22F3{hjn>i9q5NGE-z3{Sex31Ar_f&Hd3i@Qz(o~{(BnGzC@M=NRayA@& zcA0_Z>e< z>n_bJxLT5PgJWogxZ2`zzlF^^@HBdTsw+gRWJ0acyW71cyd1S>pKWl1{T?hRnh8VB zXu3>CocCLjDH-mme03(b=d;4rTAo3CLYSumo!Xu}Ji1yDgX$`R7zH)n$Oe1gwO}EK z&>-y8k2V=lux0WZRo>?C2!$J^WEegwPM{gW=baOUT|m4H`$K+3!eMY$kQ@HxugAk9 zE^aRUpkT&Q3y#mrtIq;dXvaoSc}qjQc9l8QP;G^#m&D8Fjl_E;a{fegYh|#gV78Ml z`>10U3_n?be7L`ZM~-UVZsS6__U_(r?!Rau|9$t@C=_$wv4o@>JrwGnwl;Z%Ey2Wi zGPne|WzNXJdZyvYOfIY_dON;fYja{`uG9^I<~*S_%MQ)MyddSjvsL{xn*vq>VYQ)a zl}42v@k#4|2{>q#+wwYsdjbSXO|1Daf%x~$@WfO+yrcrwJMAPKa&drg6Z)ey+v^V4 z;|YG$Bslw2^j9m=*SR@!CwdQI!gtA+T%^H=e7tf+%FF=&y~f*Xn@>s>M78ws8z0pW zfs`MPNzgw}L0-WO<94mu;FJ#?6U*#VAIiJYw4wPci=%cn5`CdBTxVk>a*XmSUpKm1 zwK*13d4cU_pW(uSah@F#^&ve(F?*q|tq9S`Lan;2h z8Jp=SGJy4?gq&=m@*S%;{^9@4aj|terHH|%CcJ~2FuN9T<bsnE5`qqWW7bFc5#~ldizL1;;>CyZ{VT5IlZLXTuQ0l9Xq`1w%`3L zC^z|gB2`AR>pmk(x@dR|t~|g$OwCWZY7xUNl-wQM-Tqy0q^F(l8yO^qyIQgxL_HGx zD5vPVmn)N--w-+U0ImVYfoWKD0{g`Im~9zRA1t)+!`qRgfdRPK_b;Q-X%O>Y0a=L$ z;FJ564gmw9f3ObBChS3{ydd+n=-1=^$21oi5)jHf8kt17h={FY5KhQgwhEU>Mzv&ZL*5ovOEG>10F9lh^_}_m-s9wd?o&tV)J}&Zo}Q)=iIX-8u;9Q z4Z>+$e6a;Z(5Kzfxl(5m9Re-=NwenHIOfP$tU2ToV=hBn66&HEJFBAs{9R>ARefvh zKgqA6sQc__hEe+Y=V(aCrnPz>p#+U{q!Sq6qQbV~V8*Q}4>mDIY9X2!s`7C1$BXj^ z0XWs0-^2DKrKQf)_UcF*dE#*6y1qFfE67oIhr_3z#xV&$&EzXRKh4HboCF?0#TGq2 zR}EIa36vNr!Ij9C|J^}p{RAJ2CnZZBtT}XIoeFFjL&eCJ4y(2c`ZT8YAMTi_AtGAx`c?T;(j{Nh zdl1sO^w3@E&RSncgRhk|&>5?^SKGOB>?w0%n!h`}D?= zHx?iI7mI{}K;om;$_GPpX5Az%zqWYTUImKSDD!$e`>(CB@?E#b5#Y6Y=!@#yb z(Wgg5L$v{vcf=A`nUP79V=nNyv&TK(zoUi-=%}{9n z({iftuom(f0JsG1Gol&i9|hd%LVQW$f_G?)C_l0`4XeAkN&oGqLL zE?bwMO^Fn(Zk;A;=ih96UoFw&Dqg_S8(9^;-j@Zc*NzG|d1jzpCGrAhB3YTRnp+dx z4q4?x(yB_N|F2KabiFYjOiJN0AR-C^r6K|45v@Ci=H|QFrdWxbmJ4#Vfvt` zx`Mtne^P(pz0IGx#q&tw_X9);QsTOp733?SLxHyCfh>@==b@mPj^>J_e&f#>i^W=5 zlwF5VPR0<$U2DLy=$j4a*R012MwZH%aAt5E#n|{s+ZSoK;;O4BVgtWb9umDp#> z(U8BG+)(sN$Z%vlGZ$Gm94AIedHbUp9rSvXo{8sv$|ns^)(mhY_;IosB^38vv)a{~ zqR|kT1-t4KX|%;&w7o`ov`ZiOf_=Oa+MwDRv<5fcmDYTE8eQ8B+??t)AKqgP!dY?4 z`3HUQJ#Ur;0`Zkhe$n@xC>C+5=EebtP)tWM#){&)vTW1>b$F&xnPM=P>lKRUG5ZAa z$%k*9Q2|=70OWZ;KOgt&Y4|h#u|((A6}U_#ck^xgwN#gZmV=yI@g>24D-a3b!~kGY zvJmi=9j?>b4M6!ITTGkCNrOsV7xQIL047;kcHqA0ANX(hzJwuF8SF1Tr|rQQ8HKU6 zJN)kLI})<>@H$WT>i7IPglmi4m-hVCruuA?xOp*3gpbuZ)$UEen(z? zN9enub_3v{jJD|_uloy)uJ`fVfVK;}O>>%s1Yi94(^I>{f15A*DHoxY>@)O` z&WwytO{eE`Hg?A(XwUa{>-<5NN;ywwJHw%YmM+$iXu3m{4?psG&7Y#FEs#WvlIe|? z0I3G@NlRiC4l9_$>%b=a@tWzsNYr6XdtkOb!p7lvou0YU>PC~*aQnJ!I|Vwuy3jMI z;Hk+dul#iOobF#EGN#RAxNxfV~&^vM1bj zd2VFD7O$cM^xCBI1<-QB%0#38jLctc2R#ys*)_~KI{|_yo87jhVB2$O$^|7;GJ6N-r#n;`mMM{6O|| zn-Rs+%_kZO-Z_uO`Hf!zHgu4OE&H~|!GJh??ej~>YcxRC3Z@teL<4G}`$NUU{Syxi zl4PR>jwH3xNI2#K86DiddcyIo&C6mSM~hk7vb0cmgKJ6+=Z~D>d0YZ(R(_=j?pr^+ zxtioB`|zNj3@A0MZ61*MnBOOE67h;8WE;`9V-dbZ0XTuJWNmKOE0>C1U7IqYXEp2K z;xoMKvDqM@FGeN6=gT$$H{!EM#vcg<2@I!3ZR`eA>Yu-zi}Q-Intq}fdeHL-{6ml| zV?etvnrVAF(K_kyZx)ju1&3!;>J0pbN38R@c6q#rZn8Tk%kP%-Pz8(7ejR4dl$W<; z#a{M6oMTNZ$^p)JZ0y4F>%2V>oneOexMI6NK0;EDmeay&hd`juwc78wwDWvsuE9wc z-0S0aJZMNEIB(+nhu2=(QGF|;*8*-yHvJrP3nJ*x;*D3kBwuxgqn%H48>(a1L~Fma zJ#pQTu@o1zUlxY;=iGK&G|j9ThcHcO^;Y3jy!|M=N5owN1et|RQLlbg+~0t%a5A>W zA7VWn^n~;{nhI1r1I3ED-hex0Mcu2PeY8_sq3TojcGAd=*QWTo$sEjRIEu{K0U@M4 zBFe^QpGr_nPkT)e;zk4TThO)m^0K}%d~%9nQbESN<_bJUsBym+)vV*<=+g*lCIoPx zr%p}886H1v!OEEF-saTX%buH?IzIPVxZ5}NJcSGy2i|x9;nK@Iks8#<+nEsZd>IIw zNwT4B$QwUG2K>&_?dCh@tzU&d&$q=Q%+w42-px}Y_#gq>Em%8Qr11m~_8O5=>f|sD zQH+UQpM2Cl>umgiM~*0Z_F(YOeU}#aeAj+ez;ru}*Am06B=!<$0ZY1jc5DW+ z%lkM&vALGlWMwE6IFnc<(N3+QH)9m(`oeG8vDen1hHDZ=0i`EHDY2oPN&*^?4;2C!}G_DckT8mzCTQPFj+i-jc)8A)O-rBYt># zpB6>n1}mR`de5yhjKSohb(sQ~82;|RfOKt}^6xP_@=5YQ`xF`&JyRi znGWP5yMH6Bc3q}!1kQj;ElT()%Fwejix z!~lJkF)II+8aQV)uE$f}4+oVfj)^ai&*VT%blgJok&du#&_KtiAF{S85pS5p<|SY# zkjdtgaw_iA)Ulo&IKoWS0!(FbvV2h>~`)wP0TC8nJeVn z(_jG5Ntim`))&dgzgL=~ljm4aKljO9VU3ImUw+;{j!zSGMRV5i-2U*KKThqw*&f z9B-9H@FY-Lj(p5FdI1D&lCmH65kXK^0f?UISOtxSnScq_Mgkd%gci62 z6lJ7j$G)8>F4f$XK}HdWg&}1|(4Cp<91gEVfp$^-Y4(ple>z%RXOIqx8jN^3>y2b- zv(h$U&-nT)B4dlz#}Tfb6T`ZubL%^AtMt~OD^4QT8`}}J?&uSQFq;azJNPqSp6)(z zQ+0fR@N0Sww)VDtE@|0h|1cZIWAt>KwH$aj_Z0;bD^(!ir#cr6V_f;W@ySM1fSz9z z`s}t<((>r%CnB#PFAo(3l8(hF(N3lv)tzghOnL`61R9g;+vseDD2V`F%tJpj;)nVP z1|Z+%J<|gxjitwQ5{9_)9`D(BUDcxp*CvZp;Hu4^#{6}KvPA-uf4Z9LA zi}(9)kpsim5BT}byafC{IZ&*wwq=|?cUb@YZNI*ce6rrk&xV!y6rpI>beXs3?Q~2p zR-Hnv5)f|?JstkYlJ(^B!xt2ACjy6jhN4H^&n|%4-b4wO&I9Iw`3Ane4JWu>_8L|< z^yQ!ihper-VFHB1W?+tY27EX)cqRxIyfR;g*c;-Yg##3|;ede7zKid)+9cF&7y5Lt9%EKkv@v0@+Q~Mfi~Zi#|;n` zSad4hg;Z!c@Noli7l3^oij&9ZHLQ;)DaIV-^TI+8Kd#1I>m?LK{ptDMlAPEWe$QR4m6ISqj{!m;3htjo`$B<0gDkA7#?$zeu(!4)Cy&|D zyhkF+1~_UpS-HLgBz=36gsXe?n63LY_RLB7C}>0W&8HiCsr>$A+U#k=U$;NcIxUCy zZ%xwYepfd>ue-{~ub#x9o-Qm)1fEFeyUruGJjoCT1K)FzZb=^OKZOiOtHuMe!j}(O zI_g6pYE7WUFe!UP13LgYiH0LhDOFhy_nS7f`Sf;NZEbnA(xN3F1%Lt|D(FqEqct>Y zo!j5spMz`zupC2@JunXQX0;tm&yO!tFnix~e?C+m8H={}r#_?lRP=|sP{TT-iWZpZVJHB536XQ>t^)6?_!3*iMhXXz#-r$a>ZL1$6S-q6mf${$L z1bX+saopW+&sOOL9mYk02!o|)CO8dVIY#rKR74sV4*uQ(AS!$-x59j{Q+BfX{oAsm z4r^{NE?V`Nes>H~6^Z0=O_=fFE=7nYWdW_Rq7I=KoT6Vxw3aZA!T4z;ezYwaC4er>ln!j~7wO8R!Ta;K;r+U}yV!Di|_ z?41r%v1=dBe4q_`WL+l$yLt1!-mzthhErt_NUIHqX^ybJUc7xoA9{o44y5}0#tq+$ zSNz{c!eppE5WX0Qc+s#!o>(c`U;BavPe1wU)N*E!20VEg`3DkGR0?SZ{80^B3d3L0 zq#pmbn>`-oHUxjxll4gU;}^7y=8qn909e8A_|2bn($a`l+^`A!zk^v@Nus&$Q;CORL=%`(V48~(pIW4VP znZ!i7v+&^~e%HpPfPezoHFk6n7rq98buc) zH)T<#6xF?4T978>DM0D(liJY%i#&de0+LfWTQ+GmhO(bq$k>o!OTrP0{d;aX<(NNK zwSZnC2w-peIk7tlKez2}dP{|U`KKFptzQcicU-6BH`dzYb-HI_ zTBIe2Qb^Z-eyzAyb5VvEqlCI#Etu)Fy4n&CX*&~pL}$9*gCNXLUNKN2>5bo*tbGu$ zZ-Ikg|6TM?)!<5)pyA;)V?I-c+R>2c?{S=RRSs2J#_h)VAhpN5uc5{vo~5ss#nm28KE z{)(YYi<+?$SK|q2lhv*)s&DuGc8L0JG`sswjj?5KVsT+K9UD?qTf(1vS=sRv_Be#1V>GkofP&bz)*UxMz-h5AwUB zAmd1q4j3DNvA5%0~Ag)T2Abyv+*@8GYD$Z>wFh zB~A-@V$YWj;L)Zu-hM^H{$o{dQmcp3po?-*kKlDPzmVcF(di20*-rbDTkzjZVJcqm$NS@l-ofjt0@+u3oUU*mOQ1oos5 zt7h{rYFK66W1X6|D2gNNTJR-reZ(~NDavG2Z-)>~UR=XR7d6cPhyC5{@cy9da+?EN zqoJ!9aSxt~iz+!A|08rrT$;x&~{5E(XA;J$YEli5Li)2ND2Cu84H?kHgy4jPHb4D^okujF%rAG!cpxw z=M=t(a&1}3f!ehrRiIsYkL4S^Xau_fUVI{L9uGfq(dR?JGD2{2Lc%wlNOA=~1c3;Z!u{{a06R;_*1Q^ZP z8#XdF{wSI%UDl&Tz|KiyVfcu2TfO?gQIWjR3V`b^S`WEtdhn-3=9%^ zu%avgqkx7|_&$WEEWZdvCfma!@nohAYH=k}{t#yt9LQVNJeZ2jV__exGxzDAmy zAE7Q+x$|9G$6$VSx(x>uKR_SQN`%6~4}H0@$xlpn`u`60XL>pPvSrp@K6iZkVr*eC za0{mMs?_O=QY2h=Z*lQ%YZ3*z;rUfk&?BZ}dv@~YHo8w-Hr!hPSB}k!(N@=%Wuf4& z2jzBk3oNvKVcf2oXnV@zCNEdjNs1ba#rhD+jLxj2t&Q0+pJ zGLkcNcU1_g8G@aaCH}Z!Lt`cx5Ysa#GLk`ekJge`D$nI;%`bQLAx*YsKWD33ntjb*wN(^9yz~_55t0 z7F0pr-wk=`GD@daFhcbcTB-BNhl{ijUw^eAt-seyU$H9B@~t3&Q8Jwis68M*nY%VP zJPyLx_f9UZQ*c!0$&F~@W)8G&B{60TPo*paWQsnYmgRll0D3fEZFazt8ef|am>8(4 zCM~*C&-$0QaeCFVr2nLM=VtxfgZzOw;1PBMgU@yT-EmrT$d25gH-fOlA{7>nLrE@U zXNCcgQ5YahY8JlsG32|85fIXppoM z$3E|$I+eal3;(2&d>MdCgtYF~$>p8gK>aN&ABw#iZG|J|r8jd*pfdta+$u=rwWW== zQZ`wwn*^Q@m5BRC3Or4ynODode>KxntIFo~^syCyUe9A{c~u!eNJCMqSd71==cqpI zWhV_8N@&0}4OIKqshELtD)j{skB~sdz+C{9^FPfA@FP^*rBF<%#Y1lpf@UcUmi~?b zM@npr&Eyd#5yeCT6e5vmxyWf+f^UO5Xcodd&6^%`Fu%avy{Q_zLz=R9!?+c6804J) z2O0L!^N+E(H6LTIbU08Vj%?PD<@UW`09zM+Bcqg-t2%aT7l9@V-RAy=1GW50#gVXo zy)JqZ!v;v?(c=|h5x%(kQAXtD;9R@q6IR$9p4zCr7@2PdiUH3G zzcaHLt(wXVWHj$)*1p`oI ztu!E=*twZl*vbVekefqDSjm4~a;u`AYJ>ot!~TKmIEyDTb>brnU zaWA)-4$I)VTCZ#b9sz}fO;s{g26TAqaXJk zA<1l6KMLe7KwFE1q#`na>SRuhD#V_CLl@||copC+L=(J;aQr5>*t*sKFb0M&Z}q!- z2d*}f$b?a^apMirB>kPp#GOrK9(nwIQ>U zAiwr^>UVk;PGP8%IK8AqyZNvC@1P8}VA5;hTX5*bcfDPITX!VRvo|Juo}YYpBI9=1 zNX01ICCT1|P0IYD9qeXVmGz@)%pjZuqg#cD1FH$76J~{xDDb#gPyT_fif|s|2w^A; zN2RhqntyzLDR71Zqto^c-65Mq5qU(_!bAhAZl{RJI}Q@Za+hKc=avZ+t&7RO9$s&d zzx_oI-ZL;=70ogw3Yx~>{gn&@wW7#!qi6&XcUn-vSBMH`PDd;(z`N&`9^k`y!*UAb z>_UoK=crklVltm?5qm(H)PTBlDoheTq;MCr@QwLIj5tzyf}B+J`>^~OicGusuF%~+ zFSrUD+ZHq)@mL9-sB1ggV=#WB_D`h4F<@NyFg3oshlP3j}v_fUE_3 zy&FvqQUBO`ptp60bs{jRqe-JMhJ6qZk@^{gHg(Hx1WhjEK;7s3bcTdM#9`7I1PA|r edcooT6j(dJdZqKP4N!c6K=RTmQokikLjDJx#3W<@ literal 0 HcmV?d00001 diff --git a/versioned_docs/version-0.7.0/images/logo.svg b/versioned_docs/version-0.7.0/images/logo.svg new file mode 100644 index 0000000..0e459ae --- /dev/null +++ b/versioned_docs/version-0.7.0/images/logo.svg @@ -0,0 +1,302 @@ + + + +image/svg+xml diff --git a/versioned_docs/version-0.7.0/images/serverlessllm.jpg b/versioned_docs/version-0.7.0/images/serverlessllm.jpg new file mode 100644 index 0000000000000000000000000000000000000000..81d88516352589e7742c7a1bdce0639194d4b967 GIT binary patch literal 102673 zcmdRVcR*9kwr`{=2na}#7Nn{)LFpwQAVm?QB1CCXibw}RiL|XEy@?1Y0#Q*>TBLU( zT`AI}OGyF(LLi}pKnidBo%8N_@2_|6eeeBo2X?YDnarLwvu3R|zqR)M_&yHOZHDr^ z4S`r&Lry^;kb@A80|F3MkU9YVK@NyP*#D*>5W557|3Nz+IPuRmED#8b;J@0(dP3O# z*&gik?_&AS&;NMmdg`*@bW&YPMdFJFb5pTB4ZIRK8I<+SlR6El`05Qwi|@ZGBxCeqh$ z*h_N`fP8X81R-Z2`nOzz0*o(TzWApHKYu^>Kkxs0p^W~S2?%6ZDbQ_uusAKlI&eS53kCc97=v_y_&t5AE^~ zdj1c67i0o#v-PLF$KN!|!9TRiKWK?Rbg(<>&wU}L{s9l}dV1Ummex3Z`n0sMo4<>@ zw9SK{V0Ygj>GOWB{&xfX@7@Y_cLUr0V|;&p1>yfk!#^{2LS0u^=j7kD{|{gOtiwOL z+r0Ak{2mbQ{a4+;ZsdCJ?)`s$I>2%U0{K6E{#PG4q7VrDJt%7Qf7P8$hCp5`Lm+$^ z|Eg0^hd_>~K_JaJ`%{o7pxFE=EZ{#Y3kwS?8!K4Y*xCLRb`JKx3&%eT=ilYvpK|Cw z%HO|cVPOOR4{@+_{73!2P3%vDQhsXx3xxk5%X60JtOsNuEc^#p`48;3A9xEcJNw@Q z{97ddObj?qaA2GV4{>oHfv_B4Wo2PwWe3L#eiQL$W3lnG3mjKB<`BGci&G}>sK%qT zHwR@+%36f34wB@~xZHhwh)Y;R^q82ug5rsjr_O3>Y3u0fnVvHHHlB{eEr8V|1bRT zgZ!|tv9Ypo{^92UOUNH)`PtZyt8)k#U*WtJC@7=x=-^S4v^Qlfhh)!OB?-CQ9pn<0 zJByPi|6%EGj{ZG{9{-m(`VWTwgP;9L$YIt4;DWL8Ltqdl^TgW($bW}xf8f8-SO@D%z31YC?0JaxOcQ~X8NQtK7$7N?YHo;yvpkG zNbt1G2xi;i1@R`lNv`-~)bHyLPrloeaeVf3xbGO|rGXReHnVFV!q=mi(tHYKYVQ8K zrhYoQj{feYb$APYovK_z7ubiih-0fG&nyZ0tWY>J_aPfJ2^$YmX}#`!1%O?9M-(n@ zuKx{Mpr5r-fFN|WA*reWT`-@Z)(Phg=cel7lP%gqcjDJ^!^O|0ESqE0v~G7Ude`2Z zj62}r6HxaBkG*C5^;5@O;eeN>>usmQW_k%*(_L50gnE6BUNM9GS1(}y$N#&TznGG7 zb@TG);o^lKnunepR2ldt_v}bvT)zAZ(ASWJ3wb2BEhFQwqyYaeQxcqcD3QR;t=pJv z+F7IJEM6^5smuCaC#gSatVikeu25-dv_{<`GCbz{tgsqljzD`6ZO-fAZF&;X4L9@njuVB$Y)}_CW z{nig+0=Wu*5jOTAD}l($tE4PmY`iV~**@fPxa1{`BH%vfo-WuCuQ6D2gX$kb(T?9v zJ>pt0e%8CPf0!8W%TQXtiU2Jmurlk1cj(9FV8?zZ>50Fnq*?Y4Bz}9fX^d=1hWmO4 zh6PII=J+a?=YEXSJ_G%W(|&6ih0HJ|H0q@pr>`pL^BbAl7 z4+$XxclrrPwLeqsu@7OtME)FDOm`c3^HU?Tg(o9cYSy(TR%p+kw8P}FW0Y^_KBJ!} zFFpwton2RX*@S&*L9?QSxlJ=yUtsE)Qs9;;KptGZuib%85^F9#QM;LR%=v z+}Tv3+VSb&vc*r(^MQdO9f_d=J3ui{h0(AN@x@5SrgkDl4Lm<}GF)oMFf+1SN&TNA zS(^RF49?@#jygxwwAbwDB@yW%8jbz?kO*D5u{|le!9FC)8<=|$$$e-aa)>t#&nHY= zVH6eh2(M!(cPAnxLEUIkYh$xWGoY z-sQK<`IHv{&8P0)Q%Gap$iAIk=q(aoaEG$t^wVm`C7Qc}ag@b66q?NBpnFl>aWpmI za0^mo4UVB?llHc}@kb>s(dxnUQGI2B?zj}&8&>R!kfgD_1*$1-v27i41MonxnhbC0 zU)hH!7pu5b!)d?duk0a6$Mzwd%FGdNas_2MYR)ezKz^`U*mK3D#o=!5C8|mvfyGep z>!Vm(R9wH;mQF}Y!3%=0RL0TFb!T+ox9oZK)&WX-ZtpBrD{Afr6%8)oQFwevnxF@1 z$E5j3+Yv5CAIR$kTzhY`4jsZ4CT%9r+u99)(KEUZcis!GP(AooNoDtc+Mjsad}60H z;lyi}|Ki2BKk|RCt4TG#-Qk8&CF zTi3R0t|_<9Z(g1j5xO7%a0zBREQLCUg!c?fAjt0EDmc^c(h!M}N11*0;qMqJqnqtn zgZJ2e!p2vppsNqGVUH+_JBxT2U3iRn$d^=?XBbA{NQ;mmCrjysV!@%YJ4sOP?24Y{s;r@7k;k{(I?M76XORKmE z=fGGyn&j2LTEs}4gGw>Sna69UjOz5L-sX2tw~h|`?0j#kw$hE}&=VC!tO0Z(HOdaC zpB@KAc4q;SuEV&|dqqVwUp33&QaN>tz#v^!+Dqe`Wm8?sIlLF5b*9s%JLne4McEN> zwO0-*`Z*KAh}eQB6l&Z)q%Bzb&)C=^!S39`)yJE~s4BRIWOVTFYGM&TxR-@%F|~9d zQaTRN#toE(Ym&Za;Fij2ZohJ8e@onIKaqnVIgq)TypO>SVo>i4qtAwkQ`Iy}P;6hxWzK}BXCU6wFitj^s(|JB*RfckuKNg{fc+Ax8 zL-5dF+xw82Q}QbceeUh!Cjeg9BPJTHppWRs^25B|BBE~>#kks?ZYxI5z z+Pb{Ez^E15?T?2N6=~<_j)#quP&@6BiUJYqM5n83J20e?MgOw%$wyO~ZQYzT=<(hZ7IiQ0N-yZjTe^ib1Xk*V<^wT&3w8Go z)L&yPx?wS<6~}waP>mK?wR@q^GGryR6Uog41(&tafjrh}A%=WX`Ksiq`}bk#>bS2m zNXq<8e$M;ST>B6?bUgYg6vpd4I-H% zC#@(Y)DV)2tLJgDIrf%oU`mmv+&4-rq1;508I>^B=E%}I=jB?sO@FI;vi-sb867Ww zSB5e;iO&HREG`!|mpgl?S=5Rz?SA%Qv1B~|f!t@Uvgef5q-JU#S+UJPjQpfE8cJOVjJHpK-_k}y+;kvQmYR7OGj4FE8qa=@s6ylSItm<#U%0pSxFJB-PgYRP!-NMbo7b$G zA-~?;wI;RD8KHwYOU)-2;5b{LyxNIOb!z?vQfcPPzR{u(o*Z3P9lkRs6bCI7KV#;Q zluw|x$N>D-lZioZ!v(BixZ8=Uwy=r;=id%|9!+&luT$4e{rPt=Eva#{N1GMGZ;}mW zG^rJMNvrm~M!Mlz(LI&vHr`r|UZ-lzUuUR@WnDz{^}rA$ADPRasJ50QOirAQ~3`Xul5K9)g4pvco^NuQMMX7CBvzHD^ z&-LA6xS)&T`(60JJ|qcCvV&7kEOvd!h?J%y$y`%?xucU#T@>pwyn6UI!S6%X)j{0&k8wid(G`x zD1;N^MdQ+u(1OlAM#`m%y$?;F{^=1owY_ zru^iTOsbs=M5s|lRsLhwW&SHJN1LOK+7_UKknR~?mc_*a zrv^-}Q8uqf0QE?5RDT=5OBXLf?Q|lpEbo~|yBjjLd$Gy^Qx4`sJ+e~Waqm+S#J zZzDBPS(Fs&I;eV9zmbF|prY*}#L&N)K8oeA{ ziFXRg1@3+YZsn@yH?#nBY%=JB??Y<*+u%`R7~&x^6t}Pw+X#KzbvP?DCSSAU;}wI% z!?$EI-LuU-B`?d}=}S*Sv{W%lfR-U>Wd`=KkpNwv5#J~uShgJxkJGe_3ZHc;=a~7f z#ad{6<^scLi!00pn_$0DmIV zz{YA9&nS(QtV*3bc6v&FpQ8I({8#xk}RupP)FZ$5M1W5fOkal?U``mLo?LDIY;13Dv!Gp>}gija`H4VAay? zGn_lR(5WM1Rg)XV%jL7GvUw&n~ zR){8o!w8{VJMHDM@!`imI(TK@8;X7W`}?)LLr1qx%DENs1q31qNi6rEDeCl&ewge@}2Mh z)Ef9aGb8IKP&%{yPW3JJ339@Q$(zE+r*n+XKpDl2O1B7ohFV3xL;a@D*swM1Wjjam(=Dyk5eILde^M6aU$Zb${$X zYmHwv8Jaz()ggfpG0QgZ#}`+7?lKN93UT(^Yw@}S(Y zt-}j%pS$kJ&KF#Lf5RUBVu3^da)2HgIpnb#0iLV+nd5-?v#AbPWy%|GQibVrPTJXS zO&({nPK&*>SCua>GPC=Bsgcil?*WYGz+5zzbZK_ts=m|J{+1PjZ{%UYLR9|k*R)s^ zF`?w$b-!!H;$yV-O{8QFDmDw(%-OG|(!BYzC16wm+gY5txyi_Z-9%0)QK~c5X$EwI zQ3KOEeHFyzgEgNI!=f>MhwTa@lHOv&0FP9vravW-9DMorKBN+pTA9l6Wy*14S>34Q zLCWt9L0>js=8!=vwoYEmIsU_wBr^|RU(qjLQgjGO_a)eEp${>=e3SZ2Zfo#(HqCgM zG#Qc@DLdg^U#!b{iS+B9=MLYTNAFDljWE?#InZy{c>2~dJbVdqw+u05!noruLbe9hTK zFcllPxB8MZfvsW$p{6sHs>8?RiY3R~aIGTu8u687!IttQ%N~!|z^3!u^Ojth4Wof` zbecGGQ%Ox1$mmigsI~1wxRmCB_eU!xCtHUvu82wZ2>b;JrEx`B|JdsD>ol}5uS-~) zyc}Nl`X;%WcR1#%aSH+CgV@zW`FFsoV8jV>Lo4WKggX$d!#XCr5;eV@G=8jzC68?6 zPha@d82>Jytm|2q=;cJZ4rbeydY;tPI?K)!L51K%9!t}Vl5xpeQvtV;f#Ty0VT z>bZp)VcEMWia21k-{aVitLYnt7tz|N_xE_OrM97|Iy6d64Fiq6*^~N2jAt_hak8KdAj#DM-gQhHQn{9H*+vb%r_Gj()qo1D(j4 z%d4VUG6$LuQMq4pIHZbr+lAiVI;JkSQFvaPR%UK*x+IJpT{IsufRT1uDA~NpToayp zD%r+tmPXf0h=hZ2`Gfp_Nws1ht5LO}lmO^az|YWu@colN7EblfKd-Fe`skxqL&;4_ z*93DEwPdo@0o+AG8_zS6=VFFs^l$%oVWK|4*}D&UAr6SqciJ7TI&NPhU7plpHWQX= zZ$Ha_k`eNQnvWd$HMZSQ;(&gWi)S8 z0zndnDvNjC+$4Vg-S55&ZH=W14O8CjLt;Re^YO!Q6KZ~eV?=>zMYEnlYUNrg4R&!K zVv#&N=y5MBDKP2`;#UW2DZp!Z8n#D$-_jWds-}#{kw-0sM`ca2C**ga-0>wu#Z#%D zm~UoD7VwD@z&(|!h-gcVts!;sVm^)UL;StoU*q&{vTYuzzG}4f^c=YbZTWDCJz|B7 zoEvM!q|ET7&O+y_;1dOSj&%I^5n*Ay zUikZI*K)ZyKk1;gPJU-HNrcJURC7Pdno-a!bk;Rh|1zm#W}Sxb+>ta~TiMa41cB=S zI{q!%FwZECTpTQpqR0D56*Amtma7mjo#wpg+gL(8-0y7D87S`M8mMZK$PvziB9`gLx-Lq1=@8BeW06 zvtU`Bq~e%&PKC=~~&bJ%?W|6-mdKGe!G=S2a>U z@bJHY{bg^z5raz$bW`WYGfVprcg^6NTzNdI+sn(yU#>J(pv4!& zL1)KpSLyQpAO_ItKj#qfG2F1^!iKUsAFixk|3l>$+%>-^_te#YVQb;zB1|qE?KmhA z#%ISX`(x_s1{!act<{k_i|xc0#x;l_M5w%$@7(a##Wui6)&PP3Y1$&%+caHzCj>bf zJG1bTaTs1(ndQ2z@${+QkiR`|7^3s%j-aI3Q5(@KW=A3a=5^pd6BEDq=zOi~*~7di zKbT3}ep;%!jY)@j15*x^07fiR{2N)H5=%a8Z&rHDS4D8AW^Cvqw7#lF^a1keA>5H8 z-@6;f7S!lMrcCY@l0OCN0Ll!SwHb`CC0L3bJI>c&Jp^T<^E3{pt9f4^Ki#q^rK$}o zzu8JPA{2~46j5J2AOnP*IKM^|f{k1Id28G_L|9M#%8 zwW;@^=r^0XSOJ?hTIIIK3YLB}mB~FuyU3i}K+-j-)^Z@^8Yy92uTWO`BH8qE9GRJW zPl_*-pkIPJDaG66`H%z45tITtOWW;K6TjNH ze&r{}K^>x#|04rdLTZAPaybdw79q9iJX&?k;FSZ95oc}fOLFcfiJN7FJkCp@2;wYV zFpGImoOYQx`M3=a@5p$C>uUQ{ZCO)8LKC@{EIAn*eIyf|<-n8ICa3ZR9#vy&vEv|q zNu()8wlD?3z=%I3gw(60oo+7EG^$!=RPf+C00g()A9}#A@bwy_-JZH8LuNhnxJi#Y zFCw$&SJ+E9j~!;+YC4|&j@gPN`5-z{<6v&&MPgrZP1M;;7GoSyp782g23)7Mc9&RE z@l*QcS6pK?W(7@gX2iqXvE%)qKS4@hem6S&nKY^Qq_#fZpl9l}F7!@AdhkZlhEPu7 z)@Am)uxdW~QALUsBii5$qlspUD(BJLxZax?YFM*pUJx>V&{t5cGA&iT`W)X(AhJuF z*aGTtW#$j41kd7QtUI?5)fHE#ZLapsD);W4{vNb8hq!gm_v3bHeA6$+evt}tJ%)bK zpuzA0f7UGX9yeY?uYOyJ%rTYKm%TMR(!m?bZ}=`418JJ=K z*7{b4r6Ya`vRDg6x3PXt z$TJ_>n(mqQErIsar547fN+TrvMHX~}4KoxlldxZR2fhW4Cx<=&qsDahJK4N-cqcna z&bDfs*_oe5KL5?(aYGgvmg}?w$O0G}FqeQN*&tJv*%zakLSFV}NsWvGq`D_p6m7<7 z#X=XlWM@^ z1S`I-ovex1i8Bw8^*x;Ruf{H5mV}p1w+zepv=vAvvVI_J!Zul%tVr{4Es|n1?Djt7;iO2LKhKA0*_Ke1(pNrwY#pvF ztuMRCr;X7;NDk*2_{W*Pk51Lg9zn7-Me03~RrHl-efUkT+GXDe7{+w#XK+M}*wHHu zb67s@gm7EeH`3kxr`EgNoUb#vly*ThH|i7M#WoWDfV2;xXp&Y5tJypisC2t{cn{@O z?&#==zm`a?=|->yy=>-3n8&mle5hm_85jC9-rxI{`0-2=7V|=bZ#ifjSN^k5@L;Hc znW~{YZ)RYy-u+PKAmZXLJ^`ku1Zq+Gr{sGnz0N<_1s#m%Z*;e}y z1&`F1*C;xdY#V3i!n2S=oOA(k=0VT7eF*hL>?#2d1@SiFMvEeL_-`H1+LlD zKf!c#xZjY_nh^)xc2IM8^JuJn8QMyAs<+0Q)HUm}IEg*ctOfcY+}ODSEbFL4i4`ul zKinf(0+g+KI zqJ0QZw5^T9b|Rku$0~y`bZPRXH2f=G*wEQm<7zI)ma`t^nnl{zE*i8nxV+gZ0@AiI zu^W4{tr@8sQe;F_qtl%AUH`+V*$WMWRZ6b$zG|<-9uSO@B@&9Uai#tP5YCHy1-LyI z?ARtA^JO*)Cb0zt(T)Djb`EPm^*fsH^3NjtHD(1|zC_^rzDFb7tUL%`}w(70Sdnp2v^ z7**j1~z#tc+8PIp0i3Vwby-oBlJ111uyBJW3zph=)1B5sqo^pnGI=Y)|*w=6@n zh(~wD9(U^b5C1w|!R%{Huc?aUrPj1cDUcxJL9PJnl%q)-)h)pm zVY>R4=9cE&-gNEQvkx3L2J_qkQLEyW7*~cf%)NgKO}WUZIIBj)PIPtdg~Vm+w5N~f zzwjqM0=-&FkWb$6JdLkM(e;)k@W~*G8si1b2M+Fp*5fowls25KS%31nVm3^5W6IFF z>4$^IlYPjT_T|tcWbVKjUqI}gj)MPgr}n#R`a-ea{i!Be-0#J@y?>a$F$TN(c!}W zxN612j8Z3kCf^<{HNQ2`jH@SIxMWvqnsyIdj?aLg!Qq-b14i_A8-7}o6-JlMZs_88 zFZO81cMVJ#V()r=p6zhaKph^N3BuCzch`OXAsrSL_i8Zz zsS>ZA#Lbi(#NOM;E*R?aF?I zSBwqZ42=Wiu-xfiy@X)3Ai*3=_&&DURahM2eWEVU)v zfW?jXN@PM$Bdq2JuQ!XmcQzX1R*JOPH4I*{*E7cyp#Zz;u}Y#N35sPk9sqC=!_3ASJZm5JpG7gvmD*9W?C-Y-YnS${v^DMc>hx_ z`R${ZbKP@(X)?}AyUZ>xMxsjJ>G8=6A0$6n>`g=Y;#+OssCn4HSLzrJKnv)F%+1fn zA-U&=nf%O|qCh-`Pxb39vUUC1h%aI-wYnWf*w7U1O^m0??Y2KcPGVYtBq(>YE|m{X zeM1U!DkqD?ZdRzal$usrTOUU-xzuUaK#P%) zG@oer;gN4cYts(b2K=?pfB4PS^ez@A_X>3*UrA=``%rfzYeKm8d#5lTz6RB4zIWZI zyx`gLk}_s7ZFn>{Eop24OqT-l0IF!(K#O^!m^_Jnj3Sb>dW)|R<+5i+b80>RWEEV8+J67B7B8X^UPJDVW@9+x8>4$hMg=-K%%LUaVs;sng$auFTr zUOVd-Z?(a{8`Py_|c2}(ZjSPdtX^qq2D9ECw*oExjX|`)-(y%7 zz+-nU97cLJY5ie>Hjg>(90j7&YiE*D#wFP!aV>e! z6DRP2-RtkdLeU3u$+AFwTltOKnRt)dT9*x7`P_KrX+6e+{_T&o%fkgLLn*naeTWWb z7)w={r3+vvAP(_3g-V*B$_T!uDp|h(En)VWjBg2b_st_CWThifI_}!TIv&~A(qq3- zz4s*K74`Zvl}JWL(T7IR@|yRC-FttSGN_(>|5OCzDLkicoZrwD^%(Qm>;AHy^mO?6=7lVR5Z?O5x4N-u!sK+g}tV zR?S#W?2Z?uBh`VH3&7lGScL~&u*f*wWK0n$6mDD>8<85L?2bI^hs<=C&=vlir~cF} zkXa(?r$qG&>Ac=JRP$qF;m$6o#$?wwR#Relv#Q!BL3eTQ3K@>aw!t17v0Xb|`E~tx(tY+9Nfi=o#+&M`4==F;5?C;zN7c=0XCBX_;lL5B4J_*ceMj?D ze!TAck^jNRXI}iK&E)wJG<2g|3o^EeqxpjvE7psTdV(`GHmTXjyF?|qsKT>eKJ(V_ zS<+9Jg9oJRph`W*Jrm~69AQa~n|qA^jUuXCd7ph%NY*&hr3-RGA}5bS={MEC&FicH z=0(^LaZt+KvqP|$k-;O&@ClyLB6lS*+qY|Ds@9^lLJioN@3N$z==S~;qiy>zdJ} zrkD2MQ{(FUC{3UGndO^TYBPV`@86lvn3y7a&3~j|22UR|7w40@sCl+Yz{o=|vFHkI zvoqzd8^O!L* zy<1ydacs5qL8hBjni%~!W*x!g31H;#we?SkfZ;W}690u6y6LDMP+cYI2=>`oUYnXCw=9QotMU-1D74$#f-GGNx9 z$ z+*kb4Gs+Erzn>3=p4JGjJUV*iR8Y(83Nm{}3>1}%%~JLRrXV!gK6wW-zGdkPQ`rd- z#W+Nv@>N=HAxBTy-wg?cZRiFSqlKKSUN`+HPQ|B62X;vCHb!PJ)~NnWo*RsqRY-b= zloq*(rgq0Kc*Tw0+;$wyIWKz5_>=pW2=12i*l|V5RWRE@oZuvGKZg9POr~}A@~DDw z$j6K4>;hbdSLb72rbimBhE2VlkB|#L-{W;H`jltQ(gOk9MU%vw)n~96UL8qn7L|E# z5QXSqDw9;}_~Pk8g-%O*8D$(wMTh2VULIr;I&HH^n%}P=0lk3i%8~xutAU}=a-gDy zE`nilxzY?peXuxP5P+My7zSMcSW=ZoY-&8<0f>7VDOa2EoVxx~3Cq8vwyMoNQ?R*SR8lp^u?_xP3%taDD+A?Mo6vxrkSi0COp68^RfiLY4>hj8nZ^xL&WR;%;%OUo>BRh~JTVT9`Laul`HrJH^hEoL^>SKQGC7>#^e(D6SWV&LcOULc_Okks{++!ni^b=^4lKNyd$#C@2x7J z)NxtId_wDVp}e2>uIm^bhC$ibH;~2>d_0CV#Se*bzq0z6eGHezdrU65wOk{Q>*+!1 zsDZJi(qqmMCVTqf*+xzb{{x1>$Vbd3^dY0ubQS96oFoh%+tL@=)!q#KF>u~D;?}7? z!IYJ!kqBh{(zyFgThM{0hJY-9Cxpib!dV#Q=~t*FB#1rTh?=gtn~P8NOaFA?0nEE5 zDe@P>(C@h8Gb_ty>0RWLG-06SFTitlTo%toU7IyB0uAGIl zARcHye<&X4AuC+D6GuvFI!xA&~#BbJ*JCx((lsgJG=#tCb7vd zs;mfIWb(ZH5a14->c9T8_iDc}IhsIFT(j^lv<%&oABK8j7vBSJ-_H8I8hpg&<~oFZ zb*3?;nWd(zxCv^WI-)`E%u|p6Bc8a#vscVei>QAi9(7><-*}GjVuHOAnyi=*Za6o~ zJ4z|h~rlv@Gdu^ObM_Q!6eO=AdWF)0Nf0mF&J zt^tF$cJm`#@J8&!ZV7(T#Z}VhT@J}FK~=DWN;}OQb9ZnAiQ6h+>+a+9#W+;dB-Y(* z>8O+*cw;a}V*BbGL5Cn22B4WWRls?GWTr-y(ZbMf{Cqb<{yL5>;ntB04wug z76HtO?Y~h)9U*^GZPX=Y4*73?5iPB%Ny5FZ=%0QfG^O(W^g;Emx;D{$$bbXRzZDyU zWsQ{bL~JkZYK2@8-t-L7tlrWof&UFjKILy@p#u2v=vEUy=tVBC>BkyL3a1ZpFnjXW2lmZe4Vc-`zD8erCWTw3*wscGwlBQc&Jnr4NTo)15NMa z<8&7Wwu)OV5hgfEXaN&;WYT#)39)Qm+nD0x&3VRG*J4)};A*bu_4Ri*%&EnldN}b= zzls5v@3mp1+APce!WM+oAEtf>#>XhajCWXnXnZ`WpAtSW!@y5a7$$@r`^a4GoI|Rm zT2FIWZ29fapEgmtIkqzb-XBZ?cq|*)3;isgi%zb3VbcW@IHL3ogKRZuIIrHlJi2&y z>e0Z3kc$s?M=rJOdD4u)Cv(^VcVvtfjDvZA8amdN2OaL}@%L%`rU3EuNGspGKox(N z_VPQ&B@RwPDqV<`$=y#nLxK95g8A-`{0X!112OSL*So>fs_?zY>@!&zz~ZDVHtN?| zM~{u(vG}sxK986$9Um$C5Uq$9k&|sVZxPFQR~ZS6w=hrO5wtwEttT}BT`3Y@Ojm#F zK!58{u=^|h7{4)BOy&d2WBl#K<7@;7feJoH^f(G==>d*0RnLc!W+}%jv%cg>>c8=g zT#{YUKDiUbMJL}lDS2>HVab+YfY>C`E-^>Zl~7_TS;jvBs6@p3HJW~Ewi9~3T3z*I zL%OQQ(Sq&f#udonyGZ3Aq|VqBX*Omn;G1n{*d}pccXc@7lwJ7qt@s)|OaK+zskbm2 zPj>(U3x9`fyj=5Jmw&LBXHM&vEA7brwy5}N<@)nL#wp7;nGft{*i_d{3O{}W6%1!d zWF|qh%SmJ|ylVIT8%fSb>0Jf;5PNJO%mqBZO7js3v>WdWz!1gQ%W0sKz8s9P4lqvpjfEP@PZvBRCP?;s=-stvw z?+I|7IH_+#brNHa8!6K7GP3p|Zcx^#pW!xS)kkwYT%$6XPaXT%lfne4DkYsmROY-T zyed+NIfx{Ii7u;Fq#^HQ#X0lnl5orB*eAJyG{s9U`VYn~_yHR>LS#+Uuy(KEPj;#6 zjD1K-+z#RCT)@n>=Wjq6O|t7BkDXJa3r-OtqiMR~C&>y0_>#M`mev;S+F3K_0yN_R#W{o`Z#ym;HYsiN^OF-kXryr2n}zl74eT*R^1qqf zBYq-1e`b)>fBc(4&W`Rkj6L-hS^N?;SanURrF2d8{)x6HT#0$QS05NKS%+%j$Ckx7 z+eEFqI`{=-7_}%gHBClMQZArC7c=SP(53r@OGBGU1R~JJxvEQsa3)Wr8jPPPF8iI z3wr%<(cQ}L@5fBc`uBA_Kpp0Fu)lQHwdT`A944Va=wwgOh$3~DECXzZoNh~9JBOy7 zHp-wU0xj*}V;np2Oa&^o4OuQHR^&8E4JfTEHgB*sBeuIM)yhqeag6dTViYr08Pjr z_a-V8>j_LuIzVG*tM2xb^;^iqMB?|BdsElD$jWOb(*#-;XBg0b4-6$#67$;Vsmv;& zmRD|->VuT)FOYj{!MD`6Ph8l_P2K(kW)u5DCn_it)Nrf-%mcKEE7ggWP7bUK&NsgP z+zqkokC*X#Euht`Evqp%)(em>r-FFH>U+?~;w8~*kVnpL4;XkJz2(zr^T9mvfmF+e zN-HVra;f@4Pj}L;?=fA>CY&lsUME0T$H)Rb_^vM)ek95j`59KYsH^nE?Sv(nTm$Cl!YzLRHZuB;~7c9w)i7L%MqO=&rq1f#URmMD*6lsYg}g zt*g9aCg=VKVP75(9&V zlf5h{`#xjenX!zQSvt@6cm6r&pL724mk;mzJkNVS_kG>hb=~)%Vy`CkiQ>GW63)@U zRa$Su4+kjVVPNb9*sp3DFAIY$oQedv@)g<>`g^j!G1Vt*^Mc}Xx_JK}*0^qiCuREx zRcDgpi(>oTI?i3vu~6J{T=+2}*GG0#BwaDK0Vz9;U`2-En9_xI?QF9h> zNpX9vZee&XhJvq{3FJlA*DoL5k+p$c+-N_zkjiEj5_bQH`_(7fI!w{roS5SvI$|KT z*>*y?)|I?p0l$5i(*8>zT{w!B)nsx0jQ8BCQ<9(p=Lw zul2HjpK>~TrCo{4wQFfIZ~nQB0ULuFYAtbPAZNPr0)D=K#Mao8ahwkvUq!Opnw>VN z@c5`oEuOdlX&+_l(QTE5kcO=?(Fayqr1ybK+f@&V287@0XLok!+J*Fpmo9D~Pdljv zbHhF3zn{7gQZyEEHvO1UlvSEz|Gd3)orahfoEWm?1pOxgo&bBzkp~^~LXK@yH2ZDb zjwoyY{yq)ymOpDvuvr+?mwoa}?o%2%OJPnofT!cR%Oj65Ll<=84TlDeE97BH-F8uzNtTaa z^aEbycAEz=^1&Mg;rFIxa~Ao)DnY4%y55zqy>gB?|EE-S{u#_S3D?z3-wMR6dvE1ZEt~-S>F} z7-?WRn+c@*8t}sqe35dNj`cbPi>B#6M@*j|kR@>CI+n-EW<8-+t2qwL2@o{|9L1h6 zc((lZ+SPZMfdV54RqpH4L>ua`waXht;W9`MZ>Kx$`pLVr{-81aV9pUi^zQFujRYbN z2zfW@ufYbwF-@{G(E2?p0?l0@pG4--6iKb6Ml&M1aJ`gIT7H3rFO_ex{03euo$}cH zXtmY-bUPz9J&VX*UAhI-eNpgUZhrp*>>W%KH-Z>53?ykRQRnC&9e5_K;Q~LkSzK1-@f@?H@Oo0_^X7Mbg9RtZCXs7Y5uqt$YAXN?BGwkVzXIs0tRqSf^(#&bAdBX7Q zbW)E1Gl#^{w*41= zYw6ip9sJqqg~Zfo+;Hmb4L{_%8P#<;&VjhcGx;oC;FG|@;Hhsu{DniBszNufw-_rJ zpZ}3eb*lePa%e%a82SWWND7)>Ki2PAF92%Qb(jA9-y<`Tmj?W#zOEg$3owK7NUKCl zOOMq|M_Z8%L5H*u>N>2JTrFFso@JxSQLcE-g!lB*b9YVyR2|gSO%DMifH{RBez6g& zf$otiivtu%3C$*Rj?}7$&wI`sl~duwH?wl`)m-3yZRNRcPm2r_Qkc;~X)*Zr-H6@RMQDWD_3UqGghsv-VeDY zIU>Tl?f~TOlIxbi`sHZri3CFlZ9G;Z*M4txd|X7+BlZXTL)cAv|HWyV+~LTKYd{v= z>#-lb+MAj;Aratbx~?5wy@Rua&XFW$7T3>_#gNc-lJeDhvcy|2<+n0x{hp89f3La6 zw_-cQqrZkEoi*P(0|N3NkWRd8JdXbTcljR^KgKz+oV*t^%XayS4odk(e;PGc4Yr&h zzn3FwfbdqjYvAk2kZgWZvQR+ZAPjXFTYtFUrm5@yy>}rQ$n?RVy9roWZhB?p52`3q z2y{3=**EHb=!b_lkXFatlB|lasG5pSx%%SPyD0qN%hW=*VM<(za%+j&hXSt8vR)<3 zSUcXIf#4ddT@|{|HMuS@yG(zs8*X5X5UL4l`P4%Yo0d7~yakJ>8j496|1CP)lW(&| zl&vgycQYhZN%iMDqGqu!h6htkpb5(YL>>fi7m`F1KPI%1Jxy`{fphE1ut_>=P2-(h zmB&Mx9s(=h`Zxq>)ng0$v?An076jN1rDi|o3;KriDO^-%Ow(h;W5S4yP!a!(e#Ed9 z`7Tq?bkFf8^ANf@Kp||rnig>7Bz&d%CIh$wIQ>VM(WeYT%NGcMsqe7lg64zJ4>P~V z^QV?)%q}*3sT}*4T`AL|0lI=rm!0?4w zrnAE6!%30U0(X=95!-#PC~jc=^vvKui1LbF2m+Sd=ymhK03E6(l)2cP8dmwNN-g=m zBiE-tES6f8@qg1*rV23LW}13_?4wW+)df95;Pc6A#Y8sB4Q0Ng{h3@luwcLH8s#qrWIrRvh>A?OJ(%laxpOV^@jcJx$r3>`of`Y9BvxzJMd4 z<292A{U`=JI;bis&#US22;0qYO!+^i52ZFs4{k`9j7FF^Ez!Kwfr)3Zo#Vbbk{@8YefZCgeGm6!L)#jL*a1p)fZH0eKUVi>FcEm_)UZ>~!Q@SOs-V|a?EqC+R*W`D#t2QF2%J~|kouPg$X{4?=$LdRFP_OZTxz~Zk zd}D?#->$@(HC53!i_$^=N;^YUhfpsJXRNQO7rXL#QmihO%MP|*K`$J6#iVV};Vz9m z6%ixo`fbMu6<;LfYI;ZiKN85X-{>}P1Pf@E`)`i*w3O zpu~D5Dbgo~`~9l&NLdtPI(A_o&!PfxG(h`Lmy6Dus{WlWG#^V~eD0u4Hqi@5EF^By zggvP)GmwTM4ji*H8u0ce{-dMX#sTIfhKHaOvAodhl^g|A<}#S5QS+_W8} zErv8NXcS9TAz0Ujcj6Dc45!^+J`Vp@p@1NM!I0eW>rPY{C}&#b-^Alv35UmzdDr)6 zh?$j#*VdH^eN5E1d?Z&R^yx-}InkSSMUerUMy(i9k7AncA?z{GCF3`xH1Q!!dB8nk zneWhmvd_|;FT55w!P986B|+nT&tO%gDxt>#V8g@U(`9MtjWFFv zSvl=tha9-5HG8M>D!=Ea_a3a4FMQNLFeOsXAl~woY#-#h2*K%%s5#%74WSqdsJxiu(x!-B-jN(JSJ#B0ttaQKM(#ZQUXgXfpK725q3tNf;Vt-b*qkOa4e5 zns!gfL!X#K$V3*k1=MMr0Sy#JFy6XlIP{s+hg7EwvS;31PNLod z?I!;Rm1M4Pm9IA_K0|ShlH9aBk--naI3cE=KMIdN(#6EJp#gx^bQwX+$BSZ!X?Pw0 zH)x#`b3(HYUc5g)L^EsJqL=c9)xY=zLx1@%>~0C{XyC?E= z_>6`by1jf4xr|^O2aV?=%zqiOx&`O~L`i$%3G56uebtgwRQJt$3~MG%k#5%YnarHN zxKc0vGZ{h;yd0;CnP6`Y7R^=GSHx$epSklbzD}e_`0a?)lJ66-vm__F6UH8}H-?EA z0NA)0L(`<=_pOZ#z8|_6T9SO<{E$w<(1!4P-oJD9!odnZEzejmE@GOVBZzNPS=NKH zQU86v!NE_$m;jn z9GmB2o(0-*(fwe(J~W;z#_|8mc}74hk;V%Rdw6~eI6vsH|L&>dj)px#Wia!_gVS2Q zu33y!J*}QDi32`gqc%7Pge6j8XMEuY!&ZMP3-8uc)Kx!89b1^P?<`a3EGtVNWWehd zLW_J+R`sOC2-%DMyCQIhs@lq$>%4z@PVxFqsUAuk3_5CX#C(Shtm-vz7}}p>O7^0O zbSzvOW#mOlwyxd3U~63=l#fk&`+=DexyXQ@GYLg}2WApwR4vb@<2natW?_&L2bLw_~t^=*={je@(ex=7$rOsQEMd%#svMhv5?XvuF9CpK48QD0Gugm_=*iaSi6LApE9 zHZjC%6v zIA8*w)Ha=F1U_%)>rIp{-N(87u}?3*ruM1Mpob=S!p~fPnu{O8PXA+y+#v}2O58~^ z9J=hEQoL5Yj_pdFnmH+|!4s}`-y~%HkNE^y?L=R9N#pOEeOAgpI?E*=7f9;6=26eM zJ$#z%w}aa@A|q(Rpn^X7hIz1R!B?ioN=LD z{&h5AujgY5BXKJ(8bO@KLKk46%tdjn(w|-LuB4)4^3~wIDh(#srb61Vo=5s?syn84 zC_S>zKuotwVgK3z^%3Ug?pNP?!Qo^nwPm+04U~ZG1)91Tl}wJKiO5qAKTFFXVM7BP ziwxkaMV}KJBp638Y=9IMy*iH&z%-3Eo~o9Gy@s<{PciS=52)0vm+r<&Yit7-z(|Jr zF5!^xV)K}FarKPL>3>YdWsNdb{S)Ix9A>=ZHF_HvR<85Er#|xi8ZHbaElMucHahUx zNVXFamoOO;w3BL#;}Yam2Ky7Lu5b7>rJQIDYi67#Ekuq*@15R$_;7-rkryfC>(kiU zB7Kvv0y1zu`WY{Q^MF_ZsUV4QwDTZxNVFp2%-`*MLglEaTW?C{#vK)0Ir9GW_?*YS zJTTui`!hTfV0E3#2zJHd$+IG^@4pJ}ri&^U>;e&?`ESvZ|D$Dl=>KQl`MBw<`2yG^ zH3mBmXktJT4r?tKvffV1tC{a+XzX;6Y%eVk#oL>*I=n_{IeBY>oeuhel&FV0DJeSKi<@+$Fc;$kH>SS*p0=)oRlzg3UwK<%{W0Wi|+==q2I_ex^eYz4$zl z{kUBx=yH)1o1Cj_w#ApQ3vyeUp9|g=usxOCdaX$w!spHkG)wB=Xq?^hp2#U)JHSxn z=aeU7A~cgIBj;V6;8kMy^<*wQ5ox zmX&2r9>$*L>K*z=-$33r-+TLBL~|&p zW8BPr=oYkMaM(anN5q-KVYh*|>K;{P#ou)yx@lJX{%CEin!uI2a@hghd32e4BF~8z zPwp|~72D@+l=Q20UaN~m+f^MR-kU_44tRaO8Hlp1Nm$t0xS!-W@3Lhm9&Pz~vhnFt z37L)rLltTUc)_b+$M2xf1dITF+9|-QA|B4`du19^A@qSU)No75@vr*gqoWVzgE~7Z z`H$V{Iw(HQ68duss8OEOGoA)`pRat_ax!866%5c|fnqH>KHzD6Kf;ADR%paIEp2#Y zO5m?t|NOv0bWC-%VtSQ_5@SIbs~v{ixN3>1l#fL5qD5((kXIqi9KsdqWcvDFt#T?{ z9e)pK$G7ZWD|H*Pzx`5P%KoMFaFv_{SXeCUFV7c4b#fsYW3J+B)s^KHRdLvjY!AJt zwSuhvNZ@}E2gy1BaegbTda-p8xGyBEI3eDk&(u72;8Of-yCVOA@bKax9%8)PDXv7N zZsyXvv=AQ+U#+v#s4x)0M$?7*txr_WXnr02wiaJIqFDXA z<_WM2JIsx|e+jD+_}@hjnmWNcv|He33`7pFoU!wmCWJnaZSOI+ym#0tpFgRrWFlLu z6_$SeL$z7Wd{sM;mT6BT*%*NKR^qU>av8^^n+qME-V1TzNT^qcWX*#4S=u{Sl>KaV z^_CSc#zeg3BYm=KCpYK3Y4#tyvj56&A9g)~M9l}HErWD(Dkmtu@lWxPL|*eeDB497 zbtlLfgOyz+F<+l6ouzK~1XZapSl{#HJ+3KNn}N~zfN)=)glKZCLYMkZO_4)f(0PZoUjUiZ2Z31>!b}(GjjszL z!sL(T*U?k#_DjO_}c$ZE04iEzD4dUaY!+hujnZ5AH|SVdyw zSAFYgUd10%hDA8S!`ynANw}t(%4bD*dDL+6)5-T$$@>+se=VgVfO2Q(|N zd;L%xpfsS5ITFf!*7m14F|B)z@--({iZYjn%^PorW=YE+DZT`!-b<$6(H0{-A*f28 z>oR@ccqAQO@ZI&2P-kMNivg8m5lv)T^ESV9-R17Q=rge7omnleN7A^6|#dS)i$8CDN~EO3^=t}$bcA0;r;oJ2RU&;J&$BX%;?SI2j_pmR7HpF?Yg@n@2c zNt_R|(q_2|dtl-vDot0cgo!}XH(AyXK7VuQ?|%4?iEwstr=g9R#@k~ z4KDh1zUTmh9to1kgICJPT@-7hBtw6H%NN7o>;kV`u&Rjxr0<+%03%?Fo;Ov2;H_0| z-U3i)B<)nktPkx}Z)++`NTc@iE4+2RZ;R}|$95RjB`rLY{o8uYaGS#*(|oFwShXJa z>5=(?r7xCJOd1NxnRGI>OmSYze3h5JaW_~!E5=8&KA`E|-IIntnEcK6lRyNyhH$d> zp-j??X?E0WD}9KOO3n6ggP>uUL*IIMq67CsT6pI@2WG|tFhu0w>xuPY zjWx)FRrf+}Exs*r5kY)h@8}*eIT|nnxnibHL!6{7c{L02)4UUDZcUOxH>V#sD$=-% zm+d=rP5;w?nq%MKIU5BSzhLD<5v2Z>E&UwG{D+^E)&ADI4B55J?(*(QHPxX6y^Xe* z7trTVRGG`}cc22NttF>si=$Gp4ORNvpMeYzfgZD)?)Xu}LXjhR{js*{D6UH{#Qaw&Sj?yh!O6NKoUSx25%gf>r9^M5Q?U811g};}t*stgfMv?ARQ*}#6 z-5h6)e1T6bjP!M2eHM0wLcJm8{$na8=^aLt_fs_W0CP~kCs5qI9Znhzm3Md>I}ZQa zCEvX=o#W?bJju0N1g%pMjBY}8@{dv?Ci4DiQ6*ArE&HUU&DG^g*2~nwdZDioTLczq zafhRZW{I|W&>^tL`Ix5D3{^~%=59peImjfB9Nt`?A3VP52s_XXqqMp|S8UI))6l&C zq}4P*alW2)^}h=S1y0uEMs4u(ik1miSoW&`b5h@y%P>Qv=3SiFtNq3P39_3SnU}dM zd86j{=qWlInhE_EQH}(vOHTCJ0NL+lWQRAEOQOFII#*O)r2Ubhm=Be?j!auQ$Ja=8 z{)qY66gRsSX4m4?kKX0qLu|o&v{W6Dw=ZH6I-D7>eqQq(u!u*&`Ujs{!I!PBAS(IS zLhayLq@@k){l8h0D!;?HrV_L^UO6ic(4i}(inL#}cJ8~bT3r2o_$R;0Qw3I}8by}J ztK&(~T4Sy#`gbNAT+YF0Op^{n9lVz&<8lkxXj-MIuHUuudePGRTC3YP2ayNW%o{gf zmxeu3(<4lo0vD2HH{cyLf;obBMFt(Xbzp)H!pFYJBfPvEFP>gS*6WHsDH?e6IU!e{ zDwAW)re})z^%Q>{K}^GvOpsAnk})cl6mXlQteQ1V*YBz(p9tHm;=DG9BvA}!x&$tC z{4_LWS1t^|3X<#<2PXZF??sWdKZR?*ys%b)_t`=e#X)k(G@jrNO70&J!kti~;kPUp z7na@kQ>#9@zQ(v&wbo2TBIXbHVCPnqUWdzwb&#%K=09DoU9n_pkWv^XbTht*CC;St zowhXH8{>c+6PiY>-y`|!LYpL{*=@6yv7N)u?3c(TGX_s}7{$+0c!3#&n23VYzSe&^ zD2DjX>6r@fBIRG}&k2oQzz903=Km0K)F?@Vh?!4=7q}t)d$nj8xg0IWeoagl z+ou_3V8@cJsbMZ^PPs>}CU&&AjY&_#>q%xalr&M?bT9WysG|RSQB4;=Xkdr}%O0A9kt#Z6CpJSd)^VN@UUz`3me>?Dg_3j)-iU1rpc+x0? z^v@VaZM*-T=@9gO=2*DU@y|jJ{_fPL)yT;_v73L@rz4cDEHRO)D&`bjZ1OIZ0g%{W zK;*`^UshyO*_e z$PeKE1fAzBbcPpBE-qp;I5)qYt#ZX{VEUGP#fukzu}#Qe1aU+jy0jNj)itBJ$qKn3 z3H)^*wr^&QJa}N7(^z~oFouYMF89vxP5pw4(;WIMw)Z|>o)oqE9hmNv_At;J%}1B? z9cQqE)Z)k>6d**!%%a}ROAo%JLc1UfZFG+A*ullpJV2!V?;WHA&egue2&~wMy_|JT z%Oc44ET`41CvU0e$w|xz9yF)fy!*$bD3d0gSC4u{#sz`9Z(wuu)NNMudEk=pv}AVk z_|cO7zj1%pH&VE=9HtuENC=wHNAS;aq<3cDAbm6Px$DEGL-P9Y><5O<_%KBj3?eX2 z6w%`CLFhaGS6li;?z@k({=!L%UZ43uk);!K01ZsMbvKw_t*YR4t~(2P!|x zmG8z18M)xEF0LA_Xd&ND{tD=$8yyUpA*&}ahBp6g3y{jHphXLS4qKaK)||k*G!}*x z%_Qs%ZhrEFhDXpGkRlkzkZmg;Zo~I?hu0=5&!Xu$%QU+zMixUdLPD~e>($_Ju>y56 zf+XK22n(%1u^qpC-DFVjqk|6xS3YiqPmIWGNYpXhyi&gVag_6t} z++)r9&e|B+3G-zZ8sF@!K25$Cl9X@T6lsz8>~gGLHQrfYa?X)8^#w1*!wtU@xH^X# z?VXn0>%fmOH^GivCN`r^JJq4%YxO6oPnw&pM$S5?kBBDH&$N6v%_SACnq*FsLxYJC zULG&pSJp7O{D8m=AdX91(8b|q)Zs4v_gMgMr_vL@_xA!g_=g87r<-lr zvd$*iA#1tn!1&|ICKo5c>l_9O`GTAUU%U@_#N|9U-`ddRUcLXHuQlC`)6gvUOWjdk zlZOjU{}-p>|1l2WR^;t2qvptc93UR%G#sa%MgK+v^qg_7Mi3JwBTBq>VyL#pS>MG4 zM)mCeb&_(l%>7W?-yErgyVLc5BHx?DDI-#0kj8rkyZF>eQVy1OYH!+lqeuT?tgKO6 z!jBO}OII$Z$MJuQh1qh1VUl*j-*WL01-qzN)R~4?)EmG+8rH!TvG;V(hOAAi-b(<^ zM#~n{Vkly)%2<3JDKm6Uy0h@0Ftkj7sNkOA=AREOxTR?6I-JF=6f!sgkQ(fK9r%@B z##XzvQMr^`rnu^;h_JJXaHH>*hm#wcrnAb9u>KTKV>uTT&PUrp|3sAY$03|CW17v_ za%@Y^d1`BN{*m|V6G=&5k55b*E|pv>?x-C*91OgS%o`5{a6=fXYJd=F@dqx{JG0{| zsV%|sfqs*1{$6%`iJFmEQiA@hXSw1k>cR|?M@AL4@{%FxHW%j>x`lUAx$l3Yjh>5n zh6}@#x&0uV1om*JiyMDs`aU0+Fp7Tl9&wzr;ri>o9x)k@{8>+qcN(zyD5@=E5Pu5c zf*OOhqAHMWmQl|IC-#F!EmE9aP61a#mTY&4PA;;lb&$6$#sfo~je{=rGydR3LB9#I z4#YR%PsqI;w0}!W9-E1i)9E-W^8pwUp8Oc`lMU5#p{TNLcfecKq@w&*XMB5z%vuLK z?+HVR*DN-JXc1EHpgY0pTahFhoWEe?$%V$Tywb40a6J7D!j3BfRgGJh%wY7QIGe)P zyIobZujVf8&_VO}ujlz4&Fd~PhDWpQuimZFKZr&D#+UE4z((DO!{LlWw6mpxm~fif zbJ?Oy{_ER@F&>D}PMAo>1@c1JmS%Am+t0nn3ZeNt1R99WKD~YP~vA;uxl+)uY*nB8T$)xWk!b88z0APSZ76>P4SsXM>spjz8%iIN-=*<~SiVK1ewDLvsYT=vxG_ z=p?jiJ%Au$3FTEeF{f@Ncag(#oc}SA_k2z<(?1Hy)5@x)(}LkUSzW{%(_jvOC<}6q zsiV@!Lu=3Udt1v#@%U;@4Vj1O#)gYpyT5+Xx=}wp)=^dV8X_`y_OJaY67o%V`59ZO zZ3FENWIqt9qsn0<19?ciHx^IKg7N-OS!mmv9UwzmTTv-8Iiz3re%`IbQti4-M4U|FW0oQ*Qc{IsWUBt@*z~C zw|mZ6PnwYkKS4@fqf$B1JtSW@9V-NjFY^ySCr!<&pI@q9^CwxNgwpjjKaT`&X`X!1 zgA(Zoupz}Cr1LPgtj06e-6=-v7>t=gJQURmD^FfKM^$euDiGpgrfHE>(|N>Vysx_* z@x2=>e#dey{w@2L%LzXxTEpaT!U67QKiz z1m_DjFSt5CWlMSIOL+c5fauF$ZzD5yMnc(|18L4Z0v7UP79 zKCA|rlRDaL&c!}-Nk;a`OGL5M#-G2$Kk-hQZiW^`_u$KsP3km4sLfFl;my#Yy`Aq$ zzq_98DYU21R)c`uzYn7g!>ZOnmY>1Dsd!~e{evtVlL4yD!$<|janw=sHmSkOEq54= z1_$b-P<71dJR$~*X#oaOC>kG_rjdAscKM3PH|!R{e85>2l%d~wK22ceu28q9fw(cB z`p78_(DDI%%vGyxFLJYc1>S`TZm1p-dT-#~2+l|D(kQITCrc<;|?aH=%;AltMW=!b@Qc2?s z&LI$ZEPa{uFXD*n*W1iGI=j9-uXeziiDZpmBV0d&dQLk7C4y(PL=TMheuOn+!lnfN zm@t+Ha_R>r(P0`gAFzQbz_Y9IFcb%V4RopRqR+ckhs+#-i&VY;b_M?N3Cp}pyb&l< zUUy0x+6&)~U})Q*Qc(v6M#^21F2}J1`4*sw&s#f3in0q(2V;grt~_g8|JmnaUkjs5 z8H|~M{>U(>-i~t6Gb;Ww?Gl1Q$<+ot786TXE|)Az3b00De1JLm*JBED7p`4}y%c@z zv40A%2#BkQN|p}!&_*rbW9jH74286{SPatXMU6^18~H0>y!KI;zT(*=OH=Ki zh27$Q!`%dum&3Z!OWa{#q872SSp_o@JOJ=d}H z@*e+*(l%Z@PfhsYg0FK4zD)NX9Pm?+P$d!f8dDJ-0#^;V`X`&Lnm8?M%4u2gb-@Z# zdE1dX?xl43h18eNb}}3;iKd#7jQ2Q6y(sqwSzDd=J@={*{caQ+2$9EV#+D=p!&xm0 zF6@QvJd=GoqBp9d`T9x2JMS@9`y^y8)4I+1@dZ1vCsCi?uOL-ua{q#cnr_@4%O3| zQH%SjWFyeMTzm8G@a0MZtvIk)O|97|Ny<oNni6}oS5>VcK2 zm8Yt=k%XGn;HbCobIH}%E@v&lTL|(N+ANWJoyWC6Zi)zL?=9RX!qQ(q`Pe?By!lw?A&D#*7OLz_%=QFq6 z&*%e!aY?0d*~H|PHbL*X`a?_Fq3DjJKX)(LSX;^ zgAw_Gep$-7{h7Z-l*;8T?w;ZW@%t&_82Yy;*BHsy0U67d zrxRB(*=S#11K&38y{LKY2hqF@h_VU))vTuUS~v&+EmzTfSFjN({sZM9#ufeN8zaY! zE2~~7@idqIkPw=4Nc@$O(Cty_p0$5~m+%%nK#1#6-uxE*F zFeM&%9GD;Y1;9b{69I=>rcv<2dtFF3*QT=}dC*l}MRsu@XKt$lIy%0Na`$DXSkqI` z+_ost$Knc5oPH0}>%Uy63G#wFt%-YH4VD@InB?-+Rx`Fsh$~oEN7%dyX^&!0&qm9F zgeKISL~TvKunpg&*d+RRbJ6!B3KO6#QZl%&VP__2SD>b8Dj+DV4mk<+JkGcJUfC@3 zRXjRnj_LJJ9pRU_-JCQ^6g}Og!OnrSfD!TaljmMhqNn#8xfl<+NhF`ynyFTL6*~g+ zx|=S&MP(cp1I1`Ish7rUMcku_r?W;*wanl`izX|K7MhQb=vh{*Ce7}>I2n+OSh3gm zJ@ii15)&zBk<#C0HWx)ha7MPeC)`cTlCHp{wguv;EDQnxh9Ls_P$^<9pVD|K_s{YqU zC_W}zrwz6%lM|nu`|#n31Zko}zRi6@))rtz&B;i5B^DFf`nP+@UvgQD_JPXf5x+yp z#AhKOxn3}3@gI}zrriUk7wUzRL{=^$Y~zc($+W5+(A4>5EJmE`#2cT$p`r7k3RSIgwm!r z-`=Up%G5s_Gh2V7Rn7a`{@vst;gHY-xX_2_ywzdwtwUE9?+w4C7IFQ69gzvGY;`&H z*~#4L>zux6j?I*`L0F&qlWo?=ZJ4u{&C_CuYxSqBXQa&Wu03@WANe>|pYkqi^By4##S1@A<0n_f zwPN3-4lai6(iKW81}-l@n3@qGSKQ*6Z4%0|4QDwbY{%DZdYH9yOS{heufM>SzlL;H zHzk(#^T4PbTV=i;NWR!qfqRSP3qau>{MySHQ0ck0puB77@C5M=_emrw0kB+G!LbGT z!%%LV>j&@rtk4`*#)hPzaiv31D-GW}t$&LP>CFB>`U!89Vm++`KwLc59v~)q2m~5$B>Cim*6msBNdiGUWpRyStF6tyR5pGK z{mp3b7WVrj~=3t7Ro2?3F=lU1c3OZ(^i=?Y4b{ickRFIY(TM9S{;!rI} zqSQ#6Z#OcwooOsNE23vpc(yhz+CGSmS{5hh{$uh~fy_{2$uz+blF?c^3sg$gvuWm{ zgzhGXJI{6KIGm}gV;+rZ!_&#AGLK@}$7QuSV+1rdo;1qI75Xth8NH*9_Ns zA=Ltsr?~%7>o{HScOt_*B+UhWXkqHN=HYjR`>%+NgFAfGc4JNw#`8AhszOya;nZCoVZMlebVUMpnx_5w@qGykByq4-|NOeGuk5$D_`}bD( zN>7g_U;TSc^>Km)fqrEj^Jc$4chDXEEHi2fk5zhjdAnh9IG?mZzaqBN&|INJ;}ZAr z3+Qd5c6Al5=F4x+IWBzcjuVfm{S0not9)C(-PM6DrvuPY4ROO0KhJ`1hbq60AurzaptL6gON|xUE z@d-P*&MM-FVFG-GV}ATKiv$xuQ19bMIw%iFYHKK}s;?Qvoy25bN!m@#-qXBRIXO$) zQ7Dw@y*MEk$94vetTd?LCyPZ)}=~D)cQayo1`r%=J z*>~|l2xf*j`Y*VjcesKGtQC|2n*NMYMOj&>A-ll)Izuj!u3({|^h?mSWR4Rjdd*hP zE~5wlw9W^4LQfpU8&bemA=)Mp4ly+U=lD_XZoFz*reWo~*XXtC6iWb__woqr8yV!S z)D*sboc#ES@hUJ&p^5k&tnt_?yrfB6qdg>pfK?PML8ubD0dD4CB)@AzRLX`tqO2QlkIcDVsoLls4*bV{oGWghiw z#c0n|Rz^Gy<)FmtDwnkf`IIMV8%m_oHv3xjHzvfG|l+mrlNovn%IXPbQ`5 z2h*-Xt7QDl8ul4fI4k1Qy4G=_(utZjg)TQ3Jy;~+q!tmNdYd!faA=8x)PPG+^6X^7 zsttF)mNf6D@F5i^?@{g&0 zF^y}e!EeI+!{Z7{>Autc!;QT6b`lgcooj9>R(ZtVn?_<;HbhOqK((%dq#*v#@)?%` zu9D5)K8^cYMiu>hJn(Ld=ULcBC)fADJ<=kBl?$A&BFNnq%it>+cFp?7lyuOT73hWi z2KNIC@UE~0HnKW^d#M=N9 zJm`4ZJ0_chOAOER9a9k3f*Zk8bRfihb#rehgO@~$e{rGY?^;GOJom<1MYk7*Me=V6 zm3eE0Bu8s-kL#20VX1N)en~|R&j91_rFeq0!wAQ4sCk=4*W`Z)&um{_Gf!Nb8A)rE z!M06R9cTQ+mxj0CN2$LoaF<2S!`4N<3z^jPv~4plx!S*X%Ot+UZP$U5{1XkZ#z8=V zMZ8VdE~%mF#bn$3*O1A1w1F&Lwyc^6XT0iBNCF)wYiX+F|5)@j5K0g&s35pRMI5OYvef;TMmANb!9h=R%=(i%QW2L{HSB(w5dY8&k_T@m+CX_41FY z$Z5LD47JMVF_Ln5itrK3Yj-xkg;O^=Gg6u&U2g%s%Z!wiJ?5{rwn%b#NY8|8`zjNW z;Vrl+jM~$@Qh9m|`GOk^KVkH>aY48{}Q9>RjWTI1{s__iTfw z6nURDc1H3BN<$|=*s{ksa6u0M5hQT0y~mW$-QRd-X=_sd9PHA7%Gn|B6@ScEH5tU% z)j(-#EA)M)d|Lj7jBV>r{fLLH?n9L zz(53sB*?qU+lS5LKM#>$=VdS+nsNERXilV4jN=hhOHA`S`p5S~Y?xdCjyph~@U)Gq z=R_j+G4l4GSK~<|hqD4+7**3zu*zh$`;_}-Hd9wxIBx#_-70lHD52Y!iHWNd-&@XL zXQP@;Fop@0h_{WhRtAeVV%k4vwEu{czg_W^TjU$FWJ=QK!Sd31DJ3Rc$@Hfp8wI0D z{qWS8ioj9wZOk1o5&^RYa?0Zll(->4!VnnUPnBp%Y5bML4->EJnX|X|Mm(v0sy^$~ zbLzx>e9EN6&4J2w)A1XB&-xd3{Q?6=BaE~XG>_iU6;9~B%H%VHAJk;95UJcY2I%Xb3ol|@)vM3)19k}3R5^y z`NU-XR4am5j3Jq>%0m~E*Zwi(`EKg=7kLp*`^McleMR-7_qy<{Qpu#11*H6mzbyjy zniKcuqaDt4Y*_*D_k;^L#0kFje!lnR*?morlP-u`uRaB$C$&4@gB@cYYXHk%PkoMQ zd5oSCoQw;TWAsnM!qaFP*+(>D=7M_1UGJVi_`9od(}uz*of%-}@6&{3h!#R@QgIyoGmL=1Op%p?{-|O}Mv7TGw&& z)V(M~Ee?`L(s+pIf%MaLbR6R8qT;B`|9GyQhuC1zFQ-o@&RTQpJIz@0in?sD z?mN6zoEWMWvdnPxo{o>O2N6RXMRG^bcm|;7i}&e`m?!j0obabk*fY;45pm`9B0U8^ zA`ye78ufSJ*KSSZzsm#ubghXBUifmw11mQ`j}=?mUMp64RBGX3LJKlq1%z{O(|F9v zLlRsabAw*|pv;YiCMaE$QfD8iu&;*3&@x{f&spwmjApY+6uANqhKq!Q{1$ zK6|DvGvyS6vm5MsjuEM_oq{hxbclfGYF3<)4g=A~c*T3)Sp|Iar5764)M5?aa@8Z4 zV4Cyv84&JXKc=Su%G>+aJw(NG*YB}6L{C1u8C8l?Dyil^&EU-F`MFH-kRTuJn6cC! zdZE?7;$m}>=2pJ%1*Zzkc6y+mfBelF{j9|wg z3FF!!6&K@|mE~n+5sNkHuPmf=c};?UuNklxC^-Rx{A3z8p27Z^d`sUWk0$xOuZZyN zpl2}sb7|lMp^tqS>Vh(=4ncAO52^FA2e4e4uEUiqxGE;ItI(X)pY`mbj!A#M<$#?E z_;b~FUu&pcR0&afVNB`nOWaTOtk#I>Ql*R#!>_xTKKS^hY2U$|b?IlAqX-TRaS-4z z`mHX)|Co3KT&9s6&4gF7Q>!Mm$bwf2H!y64x%30{rdLEc}Z1#Zc7KQub z(7&@T)j51icTC|a7)8-LcdngD>bKq>eHrYC{`zC;T}YGi0D8#~8}gofz893)w?lOA z?_@EfkGwxy?=8Tba1PggeF%Ok@I}=F(zbv49p~8pC-s)HEW}B)V1w#@(R( zSMW8`a5R6yJlJQ3NBMx6o*soemFv*`x$lIwHCfKayTu5?J^A#)h1n24r^Y9XcWVpH zpO~%aT(RD9hU)gP`4s9v(-zSXtn>+!x+*-{F=h5eoyhp_X3BUZxNO<<^DHsqua08@ z|7~)~F6M5Bp^aM^_ppN+^EcB)peRGt5N(U{xSo-Ah8J^mwC=z@5%q@IOuIWkazUGndGLrmS zZJK{Mes0n5J;ct=XEC&Re0|g^F8p0k&l(f;bM`NXU6GoZJoLyAgw`;C#XARklBki) z6F?#%cT|<40&Sp2)X`tSU0ntM+wqkQ5U@31%PCZhdVpxPaE?n*MK`5?6IovKD|+|i zwidCnDT8R`JDkKbi{OrySjt2)X%#qVv0k2{yW`CtY5QEK6Dg#jtHkiE}UGAo70Mfr$_ zuk~B=69>r+5}^JS_1ywiPB?mI1?wBp6Bmh;eco)ZalJ3AtS9V=z3R`9^k3X;p|0j{ zIutqNbu_|y(|WHH`da%!xBJa6oh^^6?-5?U88i>Cla@sR(k@7;>k3gtKdI#n2r=xa zta8b&t;U`-u5^RjwV~bAU;41Z-L^2L;px7166NpRrdZ%e);bJ$6Yo< z5Z2ZIGP|I0i}0Feq)oqN6&|+^g&L$Es9PVPcizW^qsS%5vOrc8@y@CUn9D9lJJHZD z&r(;OUZupVt|o@vJ|T@OK6^CL!hG@RCShGre)G2v8_dcEvE#rRpXnT#V~azSY-fgH zfY(w-7h80gm11S|Cd8I0KX~6UrsY6+1oFqiA!HCSZIwU3+nyE?S)hM(GTOSPuT<}C z0Yzm>BG5Ul(Wg1H|Ls$yKI+!-Tl&P2zV1$Rd`IKkaSaOwHS6cX2pO~jIsd``6}xVqD*j5T^F$|IACkqKLT|RzxVb^uRcQ3U3cp6thBpiON0GxING+TIKG5(MB;84!>TQJ2tRWN32 z9H($hv1#^RZ)(bv9l!q+Jt41CT=v}AxYVDJ!YPACW6zf(RV9$o1wK0|7+`; zutbD(*(qfBBsyd*>lur@_1~VPFx83 zX3`SsmM4WmXYI~iLK~Y1Huitayeu8vDj)dhg{Aqr_T!C~$rh#*5~Oq_D*6puHMz-9 z)oEt%YTxTUxNn6qV!!e!M8nN35+Gd20V_PYNpuoWMsaRPcw2F^|E{ME(ZF~`2_F34 zH7Y}O=GC4x7&T~;#QWkPdb#k#Q6+d05rCNa|E@3e3siT>P-zt{&U`?-ED!AEb+)i9sh3K;Gl(=Riw>qFCR20T} z8X@DRQUH^zst44h3bu+{ebiW6C+@kwlb9|uKy{*jSGvGv;$MzbVI$je#V?~8vY<@0 zMm&5*fxS17d^2AE%oq3clP-TbA})2$U5Ou(F}XjVHPcni+TYB4febbdlM4+aEC*k8Lkpsf!_bE4H9~^{g)#Vr$rE*$}Wr882@*Q z_pAziBk=l0>tzM0eQH|{w^UcZ2(t#=-%g@pG^uKhfWF-V{1~Sm04a7Q2iKb&Ju*jr zw_6NvBu#L?1^mhPgMaL$?;sr#Hx4=+X#d0%aa^}%cB+>7zD# ziyFg+(%UeofweO^i zyK(pGQ8D@!uze95=xMuA5qHKZ))IfOn44E;_w=sn`m$WB%O`sMFI*O~%LgpZo0lxR z$T7iK5oPjGr(kAu8grQ1-;ywecxmXB*21Hrs);T~jmvF)ZuN}WbFFpd~4$%tn2DKTzX zFdW$Z*h6rp;|Wi`F2L_`xqR-n>p)iI*o@_~ibq@f3)|jZesT_t;CLkBYrqS&Qwigb z3M4a4(M8*{rwIC^s1K{Kwl-ZESrS3{M3iFMxz57qNnPX535T}j`?DX|YpE~IoN9f9 zs_bgs)36_4V^Qx1O)QaQiS4weDw7Xu)njA%I4Hb?Y#?hmz$<_3*SYd z$X$5q#m=ET)Lmd1)F8Wiun^lZMHEzH%F7=9*%%!h6nO!$>dDu_DWjb!xRt*x0ishZ z(B&~=T*8jzlD(g=RIl#G95+4v>u{S{{$;Duy&Qiza*TyW@s)QURcoWbu|HS!iOi~q zKg16Flb#<=bYKP=G@f!%c}^a{Z<`Fr)>FifFrJi%+TSc~!=o*f~3yN@qWL7Y=JWEVHm42jop!ulomzG7H z` zx9_VzW3R_F4$Io$;JE6N%HEZ%B4X@^6kc^x=`eS>yt(15Q;^;#H{scabr89$YeAump6E?pXf~(etfxOLvhgXRChry>(Ozk%}av3BxB^l zW!_)kQ$xRrF}@(DR`13p?GphJ#jvGwRzfjd#`;68$yyyl5<5r|C*CFEf=pUg!^9n| z=;`%JRVMOwq~Y&8?q<)CL1#{$S&R`K6!?JKps-?z?xavW5NBG-VS2(hmmcT3#i!bv zm{*MS42^rDOKiq+&D*4UJtD0JXYQi^3|!(;$nk-9_*kmo@c?AD?h;!bs?%npjHW4j z%<=+^ZmEw3BwV=3SMK*A?OWsCk#enVm3`8WE(-~c)cn?+Jc`Sue1Zf$X9Jn2_bgk7 z-{`$`1XCQx-WyH+r<5`6mr7G2Q#wkC&#;G0AFIT@x^Gp{>y^Pbc)&zLFTE=;tnDGn zPpS=b0EpbiVxK~FW{}sa1XOn(a|;L`fzGrehnx^`kJs$UA!dg<$){W0Up5J5>DE*M2(-Lp6i})W%of06bh4%sQ_%5K_q7g58 zn@$_<@A? Oa$4t)F_-J83sIG&pD1amFY9L)?x>h-a9{NrW)-EX^U6jOM=8p0^~~ z)TgC&({HKPW<|j_?s&J!%?Eclwg{VC;QFkAj3Vpxr%@@90W_&O-tk zGZ`M(w_m~zStDr5%djuuJ0;g1nolCkj##9)zh(XBuq=a{&YVsLyb&387g|uwP%eP& zbF`zgb^i(cu@R0(uPz|QFKlwjuRSu=BpaUaOh4H7-mf%u!^p1xS3p0Lz~k1*8r8$~ zXb~32RQ6oUm3-3wdWfl7XrlYRXN-;h^f79OGXirhT;ZS$4V{msUPKKg{{SO$l!h$K zF*xhjgjjchmoLS3L_ytPye88`$d31#gqUeAF8IWL#|e?V2DKeirUuMOMUiV`JwGsy zQeVVQ=IdS9yWiF`Myw9NUSC{t6}~S9zYZikol4iwP3pfPawd&1w4;oV$c+liJMk!! znTnBPpqK$Pb|-w6DNIs zl#|yhZ-qmjn>+8CaB-osF$QQcb>!O0YxLmHor4eUHLKsTT_%J<9EKorfuZ94S4?%u z+vY%g%6l_JQH}ZVc6BBjCgd^|Ku5d=EQX8Bvr!}TTyh6L!m6_GOSaF3*ueD@s95Mj zYYVk6ya={Vwri4ZbL+Nrhz7}*#q+O46Cfc4{W23azcx(rp8+7BFD=Zd*Jp59(y3z> zHNJkdgyp^ffFN-C93t;>8C362i@w!Yw#nd0^mO{mf!R*5_arPqK|5Xd&-HB(6tRaN z^ulheRw%w&$D0dT4C6aU&xzzgKv?yAdd(iXMN6+(>bvbG7JqmrB)v@mxz!)cL{YuU znL$8u z0eIgZ-ZA|D%bonnGjswFwLp&C1)^dA>_IjnL>9nU@qoUGo4#gtqTa2}&tbb`B8r5fP+XF>j; zj=3@5$?QN*zDQI3@+YR*dqvyM<}h-CdnOIx$U!Hp7|~-{U^eRkZWwreM`C~DJfOu#Si^p!@zmtFZs}f33)UA)PSuAU z69-QCag_(1;2ZUq`7Z4|@$J#$1-7)$Xk3k91rO0+N?iWOW*_=R})f{eZ-w8q7U4cOL2mW+P2xtH7Tj z(<`Xupk=#yu}_mFS1)AxX=Gu??R<~fr{?E2#S&RBEf2qX*|5J1?sAkMM7j{l2%*pC zD-rH?e*#V~RrJv#=-2;squS0oj7~b&-GjdSd|o<>eRvMEJ4*AnkHeERs2Np)XhsTS zyx=gkv;vrL?R&p!8>=a4radu6HuCAV4{q|T$V2Wx@A6#lY}wjz_x9ERN8YEHp-Z=x zS{~erIX{!yb>e4-6pv9!9`_{!_3b;X6DKcn9_&14O7fmWpQTrh?yTuk)IMY-v@QnE zturl*Og^C48OyrZH}CyE=A3w9+)=d%=uI}-K*aW1T(3M_CeN`#zo5)UqnMqIOeeR4$908f21bH3mUJxm|I5CaD;rS)ivxveuv1du z$HieJ&Sg4j(&Pu4cWiOT>)qW9*Ru9sH??SXVXVFj zu{XpN!bO)({d{%N$!k{({sW;28sVCb%cMm#rh1keW!ea-0DJ7_{c*Zp2E@v0e-E~4 z!?u@#2BH0owy67Jh?jRe?-FX2oJHd+FJv&E;E)363a@| z9%2;&)?6ox66#lnk0$4w=VG8r^+il>z4zl~FfiqrW z7HmJ+$|M|MUJ_dHp^%$$KIiqxGqSNyoXWXEZ(eKP;>Y#|B%qTW2GsP=`7M@%Z3*Q#Lpm@4lb# z?khF4);ny?!AkNb?2FCKPn%Zr-$HW7JZa{npo@vr^QfV(V4S=B#Y@mlLZj-@2s#{X zJ*FX2$b~4f)4GUAdt4fo{xvIYFPr~$=Pn;4UPOUD!-FqBkVay7Y>zo|pEo;nQ#+88? zBT|__A{cRMPCo@Rl$tf|^w8g2URCTi|7?$4?DxZ0e`@J?h1hnI+OgiYcB?GQCAcN) z&X($pJ2Qpnyd0qQA$Yn1|c5hatbwh{F`hN ziJ=FDhxp+ihA90}8`;OcSal9aae?-PQ4rg5lC)KUCp^IaILY4Q5DkX51bpa`S<%Q9 zCaKkZd9Hk7ziLK&#G#FrZAug@H`y`#jnzpZw6Ge3ZB~&}RZ9@5f4PlA~TcxZDlmWt?hQo&Te00B-w$D7H5<`=y7ZQj!TKN~0 zw$mx;^~9mvX2bo|@>4f~X!;^_`UYTS=sd%Os z3iLjv2(SXGS$zu0-C&qOEp8J2-nB|nr9WEh$Q>(p6@29Q=xoyA3QX%EbU(&$`ltiS zvJn;H!PrJBG`>R?V*y7l4{kD+ega65o0jMTAY;0Maq8{nmTkcSUh>b6L{o^I2vm7^ zqfq1w5V8MDrlyZOl~KPWmtUo6WyqQgZ@^;@L_F?TJZxx-^yO7GlK0>h)048`zijon zMs8c`?x)KI{W^sXEvrC!Rq8s3Z{4abY(5EEvc88(U}a&8%T#12Ncm6OOAbg$De3v{1KO_ha#5#c5(Uq4jeIA)tkt&Q zb+MydVmLKBJ`3g=$9*Ntyq`W@nBr)V8oZ%YW+^uCY1-Y>AyL^nI$0EK0g7>Jxl=!j)!^{O$;@?mqM@%+5WjEov zu6}8?z)So-eWh*hmDn#aoWnGU{w z;|rgx^vu->@$KS|7>k(T%Ots-K$2hb_Q3M&O>!b1ON>_(tG6(Iz#-G+HC1M zA{q?Zl3+#e$Sue*Aoo(2_7etA^aBPK?`{{v)`U$>au*!Z1o+z~OAi*u*-RW=)!1M( zk*Kx^SW!C`q)i@}2LqF&!R1|}7R=g~-8xLAStRHt5zC;s{)A?{7}F_V&eyEQ|7s#? z^~HO_0$PpUc}D}jM)7)!N(|cg7IPH!;=MIU{5$mPs~8rmnwqixb~OHO;^6ABnzYJR zeViOVU;Ws(O6jLMJBU9zJ5hQMqE@axe{#85z&w9U04T@j7*udPR5uo}LY-2DE}^I% zi2B3RV0c(8kEXRIJqjx{XeSGr#tAL%Tq-=^%=+X2waX~6c2gf2ghR*hR5!JH!AvGM zoK5SQQp2MMJx|{nio?_>k~;N5qR|||PCg^Vg1SN3erC&*fX3*9gtG2$3|xP}gNS1S za`PXf;Np4Q|7}E_aO5?oYgN)m*}fGMkssKZ%tt+;$A4z~81P7l$kg_;?TJ(yLR=x^ z96%@x72&cGZ><*7{{NmihrPK~@?6Zsxwu{An@>w8dcmL8GKGCdwI&myYfml?X-$Z9 zdv6B%KjgEqt6oMONMxU{94O8Uh_I=o@f2&^c0ijSZR{zZaCN%3+)^&jF5OvuJ|yeb z?<4ctY!+i8anInVrN;4(QLHW|6a1kHi~(-6UpNh&M{%IYqiUms>GG#f5K*X6tYFt^ z3&C5qi+)7+n#$MH%Ucf%x{-<#Jr{W0_h-Dao<}`}$i4h`nN_%#B+04ggo!Y-?ko7M zU+d#JB*rEh(8QN7r=lFu&+}L$9v8W_-VcONdV}kJsKzVZeQz937Y@s9N~`T7_X*FS z-k~|6x&r*y^ssM`G+f-Q%gkIoRPn92+#BBQ7kh|#lz^crFa;CXdxaUN0_@mAD%U4p zFD1`ls$4b=EUtdn7C6JzrY(Ecl4;Vm(DH#Wf_sSHjpNSG}-o}4B zv^hP`>%fgwWSYY9Q>ZA16QB(u?^6?b>m&O6e#ei8Hn?5~C3Ip0*fL4Z@2u#hi|z4# ze#Gp!{j*<$_sU*6_ZZm-JP}75WDii$6yh|rAKqv3jmnPqdIDUdHej)R?hki7^94m* zpPst)I_R~je7Z%Q?~V)r3~djxYW02DDmVYd^oZm{Jg)`o{7S@3$Yb;@KY9VA47+te|NAm7j7aCNLMCNAQd%R5p~f=U zYTbCJI(#!WVlh4UFUO&U&Mb3g{x+E~A8b8s_T4ZZ#STMv@?%oz%i`bO3mcJaL=wJNM6*TpBdn}#@BkY`xv`M6~XNnPj+_%#R zqmO17#go|;pzl2f9Q>NMu2o0>*E{8!n>L^`eJ;t_G5jj;qktypQ*u2Rgzy7sxc0IY zFb%uWyUWBkbAF&~kfC1sMRa$vmqB)=Ka^t(@#3k*$l+hu9z69LD#AggZ*skaz;BNf zT0DbPG|IX+r81Fsv@>3?^J+^e6i5=%zlpJ%V$|HdErr>BI9FmBr#_FEpm$)Mn?*xv zPo>L_7)?O04GD4%?FdBz;SUx-kgh1Vb%q|hO&DWHooPv&LPa*)9~qQ!EaD@2y>$u- z^!j@I=iND;+iyD{!7%~+Pfna0ixXs`Z`u91^h#ENFG<>^H?UT^n>b~W5zZ)dcI#l! z*78s;iRs?&{1v~(;9gnJ(f2GWt(?M$7kx~2$nBiYU+*~D#Zm`nF~K&ivB!wz$meX4 z#j<9I-_@J-eM%1Z#x?0mp}hCJ>V5UmUV&W=(7%l4X+S(Bpkgc7KYO-+t6dvbGc$t2t(w*Ym zC_X3|yV>CTdTM6GREqXML8ju1VU*I=t#cO@)rXPz%jQ*Izbs99@GMO^N2nCxHEtB& z4so`;trC>{W8ScrMyR6?{bqSvpnuP*+uTF1WLYV!g2u!hV@Boz`Vh7YwWc)^Y>Qy= zFD-L+S+|f%5X{(t26meG|FXqvlP&B4+~7KzoUW8RuK6N^e{u7Lu@B%=)*;)7mjws! z$W{E@v7WWZMh5)1#W1Sh@!dYN?)gz+@y30u?-DHp6>1`Tar3pVW0n3B!akt0?F@wz zOHJxJ{K*#0vFg4S2zTmR=NPiS=fIf^40h$#ZKdYvL?@n_ktle_O6JUohKsnM8+{+A z=%_ZJfcg(v-GM>NcK0IdgIDc=fK6M*#cI;yNtWw%58d!cJ)qo z$K;K)v+tow)t2jMkAXVIbRmWl9?oS&*mmQ&2Mvw;cnz&Os+d*ew;WfgCdY?-osd1! zh^F{o#~F$#H*nMQSi43EjfOt@D~E-tS_x`rq`vc%0jgeBxH#FeU(-FZ5O{6$RTp<^ zME*l^i}k)*aph^U(}6V+1S_P6fxDxyl1_sTsee&}cC6-jN^+E1WoY)j|n>qp3&dcrYCvyVl7;!J)qMEWU-Lu zSH4UnMMp3uiU~B^zbMc*vDwCqe8_<$x1eKvg8p!)aexshQp$h6_wu*ot2I8-SMwe? z4)MXl&FPBYX)mgd5gzTiLV02MGi(J%+oBV0!lTm!iWCvPP%^-3+%55wt*}g7A*>Vd z7SfVy70!xuA9}Cd@{pI3psZsQ z>kr)K$CRPj1-WdIcK+L(@e$p7{a2vVAHSK>GhfRcoM{FQzO)b~@pUM9h)gdNQ?V1Hp5k^E*V%6Y=ujWUTELxD+2SV?1O4LOj3-5K{{^hzLd90K$p_=+( zxFh)C(5Ps&r`-FE`VR^l1c06%&RxDv?WIq%Dpf%LItp0|JJ*jGx4cyIICT}h&$W>9 zvHZbpq;mVC7;?ybe=-U7a_SISh>jlx%B9aqbhul-`^tpLlzO;+* z2S@XbaABHSM>Ft0o+|wTE&N}NSaDs1Vu+f_42+*B~ z|BmnMdFUzd&3U^Y)?@SMfOwh?0s!^41QXls zKu#0SyaGxhJduSL-K3LpQ$^EPTSQqyULw5LyEdk(KTjN+Er#zWf7HvPmWdf4rq}BL zi_8@YQ(=BLJcMztsjh1^M;S#8^0JbXR-j+Gljv&!z;E7}_4G-=IhuBqzI%arxjRqw z*YarguA-q;WONEUsn(&?eywt3| z*1-KnL!ng(ac1PfmCu5byKYqy)=bgW53Mj(S8CNHXsUB_XDE{DjYP+*y**|2yoe^A;{y0ui=>~2C&;mn+Fo(RKqo`(t7h1{c zf&S@-%DxZI>2UpG_f))g0oW^Nj2x9PBz>RkN6>){fHLN!cs+U_^LRInj@s|4rF-Sz zL0S7Zdv`gG?m_Ww6gXx>GP41Mf_nWxm_he3x|_x z?}0+?p9cKybZXiBH2m&ZuA;A*3eqa!F9*{7Ml|>x=)}wA@b=coyx=ilp*nkNbRC4r z)3o0fE9lHKNQrxfzXaK#6H4F{_y7`v_d7g)x=IKVOV6KZReBbO0y2>iAcwTAQFM@;xe8B_@&QNUjB}ID zu;(^%P%R3adl?@!_zqCIrN{$`xsX3)q+|<^PxN+2YXksGBpkTu=k9iy)YXV;(aa1+ zp=kK>H$=WU2IQ}=mbRy;#D-N_c;c=%xkd#~&%r*-%{deM0olAmSRC-LeRYEK#5wI5U6~8`pHo`W!-cXizHKgi#(nTb^ z*ZrYVF8}-cL&qz+-#gBC$6gWBs;ap_`pNb$+Ob?%?I9Krd7-*oJkqA4m zV&(l1bP{m>-Mc(^0gHOHupD z4A%5-yfM4#Z3NGOW%dPXWd^J+WC6j=oZm#fr42ncUr{Gzb^6ziD|*MRn9wQ7c&}PA z_0Y}^u#W76{x>fDZ)6($5@n~6s77+YoIs&&mBC9wv|CmJE-B%kglMRk7DRr7=IvzZ zkZ{br@hB#t3AZH)O|c~Cw?H*ijMR%3etj(7{2P2$&Q5(YO zPIFLe;rY?Yi%)1d8*C^82Hz$3KowwgNBDsZ(I)tlDQ;&@L=G$o!W<7V0}QO}E_j7e zejYOD>A_Z1=3pHhTyIC0lf_rMsfFPp*PdSbDQ)d&ds`(ZVv__Q`*92iu+NpqEltRg zT+0YgQl!@*D)trJpe(kiy2*2U)5KM!Pp~_aS^FB?Pdj#ii~f+6K+s1{j^Y#=Ne5yo zHPd^zn%v7-hCVIlg!e=oPMo{QrSy#MysSs{Bs!yq1}DqwB<`-MZp6xXm5Rs-XsQ^; z&McN!Uu*pntn8-C$36){!6$TSGE&G>pXH zt3?0qgX>nb>9XXHtG7=F8NbKZzH)x0>Xp3v$*5P1;byQtI!&2pm#=P(IKe35XkCco zfs8J8Y^@T;7)CRWVmozP8a)H$dYB4i8PU15L!3AD+zR;e$yPnFjS*WuA36eNFKOkO7bECoG;V#>%c+>rdRa0BXV zOPK19N}7auaO35Ocqgwvw?<9l)Rh-|1?yxlI3LtxULI_N9bZ@8a&95{`-$`e^GeeM9BJ(gfE))?p!rs% z?dvZF8P`y^JBXnHo`I-kXRu(Us6-s(LHNhzT}A3^$;wgRByAZ=Cp?#Yfj=5s)6}Rl z?_3r?Og$5@=Ma`MQge?OwhEsRZ}OJ;w>m?=!o5?g4Kf$a6&ht_20YZ1RWlgf9o%VO zdgrgl7CaaAl^3mR!e$D{#yRfo8qR)#!P6u9Q}TKpSq7NC+&@^WZGd_1?qSt1xxvK< zu--l)#!N^G+PIfhc~l!^AKos|JlN-bRlp)aVdv+;&6uZH-#aKH(6WiSM_tdb?D90SDxLz#i;$R`ey!s5y^J;f~EfdRDZZH!!f9N2LV`Wartqik(`t zUY2_ys@zEDV$zvPWMJ#_aj98yW29-~e0i%#NM=k(H`{P?@z1*_MOJ|`-^R$l)D>e5 zZ@q8OxIBB&b2-rKPx|}8sa2*}0v*Jx7q;*&tWdx%#xh%nwtQ;zY*&S8!IR!Mm6Zo) zweB+B8ZM04cRzdnY^Q~0Jq3lb_cOHyYL%E5tL^yroh?nGulUUgHlDH)htoRSqE7v4 zR19qd1&DPkJ_5J+Tw!FU^@>maf6 zoM`ATN7gF3$(t$poJdO`iJ*HMK=u`1eFaiv_hR+RNTG%&9Y-(pp*G$vt^cP7{;Bt$ zMtUA-6d|hFic0O+GoMhqF;4g$PB-MvWiTV>knP|^t9eYl-i)Sm^byTW-M2n_wIUN$ zI%i);9VguH+hNcTvWg9lV(Xxi7txH1n3Hiz?P+HmM%L=dZ(i&)lQ|8lW4@oL;`{kS zCNA<}-Q{w+253*!_2IvcO&pA{9AxrOyL;aLuKPl zOqp*TtQ=x|^IN}bP6Q!&E+FP~;0Xf+rMMYM^)sn=mmyIJwfu`C$y-o~rd_Z!3zHvQ zjTK!Z!~iw~u%;4X_%oGOpR;9q*1-{KUaLi$YvDQ2WwwP&iv;EN4U9os1`tD9U4O7X zb;XWhvY|c=)JnnBt@YH^X(L$?DjmBzbUHVL2unPEoU&~Veyly1>TiklKbA@JqyN^L zuMByS>QoZMQ(kI}{G{*umjgA|Qxl}GmVABSw4=8%+eCWZ!))+90|8+F ztAtTvOI30cNkZQ>P--)BDY+fN+A&Q;UEU6B#H!Q$V^nKLrzkhqt=>M8uTD1|#Ru+_ zW}P(1PyJYP`4-=So&#oo;!N*7*t!$MIVhvS_Q9=ph*;XYe?0lsdcKn>q&La=Pqcvh zIIKqwJiLB4qVc3#?+N6I=u0>I?%$-86xJdJWquHTyRqMb-DiLVW_R9V#|#xu*cit6-N<3E!~i(U zKLLw7VKfc$N;1O-V^BgWghjmBbfoj65a6fuuYbpb<(hU<*kCbIBP2fjx%S^r#J4m-sBZy)f`5bvZyG}_r>y(NZqNaja`w$-E}XO6?wcX zTk`;p?!F;c2a0De6Q&Tc!Pw54ip`zcRkO)|Z&PEFA)howyv63k_8_@{;6Eus3lhb0 z2PCyO2C4jUt9LiGN;z`z%Hx)Xh9h7C@{wZY)~*7$h5nkAW+>|ow)JPzs{$@qYZAK) zRA6N+%&@p9d{OOXv0Ewf^dEc@K7L1Le?IS?d}AasK##o#d^x+DsfR3j*I?uMR8FZ^FUlz|6G)#Wx=u1Ub(SXbtTIUM<8-Pa|L@f2OeBk1Apr zTPxt2eHyk7lM*pQN9m!$F@;kg=hguPIMd0H_8fR^%RXL!&vi#Dd$3Ehsd^8@^99PA zE!~PCO}J9NK3o!}w@nO4UAj3kpc!|XtjO`1_p3Bk)Nw{f-g;g0->GyT)A@V28YWRd zQqShq6Gzv(p8KF5>6`B%Qsw_XRzLq2AO4wG?YTWRx6=B_J&(t*sL4qcS}n3a9iXI{ z_xa;rj$KapJJWwT)WpZ2yQ9S|8@aorR_KIDxhC*7eDM_TkSg{8wHIumUX3m6#Nc2S zM9QPM&p-fIscQv?+&~C>v6}`?BLr3!Soo|9py}tKeTM1YpD~%^Y%rCaujAZw%eaUg z!@=31_v?zYzV&ucKm&Up$QiiJXw9n>d`wECR3*q^n_NN3NJ2S=?ze82Rr$xJ@>oUk zSV&>)Xh{XQ0lh-d{xeLKeDooNJE%neSQ9EP3v2uS@wDPTV_}wMtu9dccy=F%O4G9F8W=-V{H(|ahlGK^?BeDtl7hOnF5q+=ScaKcnj-DcY!j- z?u!E`VpllgRc7L=HJj^~qOk8uZ0=rvtwsRU{^JNXQYN`V%tYpwB zsyfF~TAv`E1!js05M^?D0L%*hhfGR!N8>l?Flw@KCO90_H3$Chy5>Eedfpq)-ix9_ zG6kF9A`;Dt(6id;g$D3($bjl2^grfd)W$*{v@o(pRHE6j>hW+Ptg#%R4R?02>}V2P zXKTI90-+5qbf-OZUiRp8ga2I52%5z)xK2W=Pk!mgqVxKjgBxZcLnPHflhsm>b0k13 zy2Nxhb~uq-?feJ6VX#SJwOFtBqATt|;lh))U#&-oKtb{u*w{;)DXC=Ci=g-CJvA8T zyf;PelO(?N_Kmkozh!*%#kKCHCGV=H&+kYINy)8A+?`JyPa^1;jj>8ZM#BLG#-{mFSk-V zFTlps;G4iaR=j>Eq&C&jJIiHjZ`nSV0i8kZsb69v)d?}_!GqgN@29z#8k+~3IF8GP z$?z5ieg~SjUB7OJRHtQjLk%)#9(Z5pOUa4fR0CpJ7H}%P9oWMUPU_!gw|@q41j24$+6P_vH|EmwKj)AwgINut(^-+bjZ|0T(JgE^pM{3z)| z{FkvIsZ2owAo2k)VD19|P!rCjzyAiu#A*(Pm1fA|r$V$0F$@)wa(71kHa+Xzkrk}V z7I%CsluLL9#+cMzV)ysd9uDAR%UBx^Q0~@ztuEL5m_G3YAsrbaap?%LvDKo%LCbh_zctscf?j(t>f(7r(^WRuR`|V(v*#a~_)och@KE0R z;Z^?a=-F#YZANXU8aj`k{B{oA5$on&Ia@sC`Dndy;&5dO!?N#KaBBX$vFc)TrPbG_ z5-(MMjm^Y8PIp%&-v4nYkxxGoj`!C)~Fp zJ#cO8Ze|y#x+#(ze}D=#yVO4&0o%~@ri4AH!28zV;hfiL#+fgH8>}8pP(Rdh{^bBf zhrb*^E3x?rI(DFFu~wGq6|Eg9bs0Y!wd0aoUKV;HbDnwvADq!v)?}2ZjMy=7l~hfb z>^Md;rfz0|VU0++(1|?*I+@0kmV#%kEQdIsq2K#hN1klYXF0q-lUAbwBP|oCa|5mn z)3%mvD;O}%0GEJUfHu)P*%_LyaUZHW+8pceF0o_kBkcllp>GDD*n7b4!;PjrGmWU9 z&Tb^Ff&Ig(SA`A~Cp7$ez_U$czm+nblQ}V7F;#n{JPID9UXKDFCmDH8K%ikW0o9rN zFRybafeFJ#sRx6b-aIafSnBsauk3hi=t!HD2s)+z=4$^z53Jptm@c^zS2N@bR z-)Y~aC#jI(fa`g11trB`e7WnABRYu9OpLhQuT+&Ed|bY&CjLm&Nt<_(e4mMpreqG? z!MD{{I;6J~9Fk0KRM_!ldg5qJ{ii$3jQUni+dr5vGiyKNu$TOf&s3x+jUbiJVf8BJ zRDArzr@;{v)shfNLKv!I$4O$qyhQQ!nQDUq7=JarEud&-OfBWc?=(Zx_Y&6M`J>=U zV#Y(Ax7MM5R2q6M?i+E0e00|*gWHQ%M2g^+b#7<-zn;!drdzDjlXc| zo&QKYO*4{OUYjuu$sr;ZkXH4l&lKsC+OULfpy9ZIegALshW>vW&p*|!WRd{p92J{K zwvezf`LrO}QBbB`R=mGOvwGg~wHDp!9zT@m&lEAD>(JDeUt4(`;d{~*k+vjy~7p{aBnD=C4VO6A9=PLaa%jyC;> zg#sUE)Jjrx7$he-iqeAQFJ2XN@Z{=&5DUq?4n{)nT-SE(v3C0)>8=Td?XCP%`wzEp z1Uu#!zNIi1@c)gU@EM{}dR`N=ZVQbu%SVXMIgl0$8&k<;2#Z{c59F(y8+nIA-Cv4H zJyBip`~0a@p^MXas!HvL&-J92f|d^dVEAD)Jsl1&W*;zhB zL3}$2|5o88G<+gAP74URGDT^p^}rT6$Uyxcvc5bX%Kv>=QIe3#l5HwP#*#|PHf^$% zX|rUT3L!~Clrc{#WStL%N=y7|)@=6>GyeP8!=U)K`P{WQL%5;*}(Y^>j50&4CCk8`AcnWzls8wC@G^uRm;l&35^ z|D8-DLd3FQlOkBN%P`@Ss{OnwQ90pg@Y-C1C!B@e`{x-8@}Woz&;V`=G3-A?xM%Y@4cz_? zcxM0}pk3e&oxKFD7x2Dt!O(g07hD_Ihc77R9Sf~)QqYKHGWP#mc}1%!&OBjurP8q| zEA=1+e`x=j(VG9NFE@<68c}nZRZVefZ}eAkGDs|;^LPn|;NjwzpL9IDbK*x;HK!b& z;B9;7Je2Gre{^4QE6BV2M8CD*IywH{k&EVGD`v6}J`W>izw`9<(s)PtX6SjYzZGIl zV>>a3xJ!`!+-r_aI|fAT=)>AL`y?O(#V&&whgTI)B4{v=J5qqX!$t_gAvGtq)?cU+ zw()-kJ`p&_r;u(sn6BMM<+5p9hcTQ!Xy0qvdkHOoPsQf94;D};UA_kVLLJJY49hOO z6Py~l&wm0j>OEwE=^(!Km$U#$f5^Li^aUf(v>{fr=-J=C{l6do3Y{l&_cVeIQ{*-G z1T@`$MPT6DiJ7p%Oc#OYr2iJZHvGL{Elc4uzbtM5;Q5%Fg~tYOeBg~?3{G*-Yu;(bYbng1MXY-$s21iQ%chT4hB>Sl0;QS&(19IC z(c(vBS+;S9{%~g*kW>Yn-kRmaz4{Y^Eco-sO6J4e5DUX(KtCi%{Xx(&C`?7( zIUpSW+FUJlos%!(@K{?lPJZxbji>2hBzO3}JXOFC%}*8`Eq$tlAIV(tJv9IhHx@9qYyn-dQlem-xUitAzcm-#Y`}LryiF1Jkpv8X5 zK%Fmx<9gw{gbEFQCb>n(7GjjMw;<5W-IVEA!Qj)BnYQnU%!!4eSTm=B`-c5hu5(VA zuaM3z&Xvi>Z>W1%oRjgq-uhYfxcr{wmJJ_YI1E?jes!a?Q_ka;*T)EZFawMI8jXvGQYLa9B z7Y!D1daZx$BkmifOaqT{C*zF_DN-A7Yje)uNnf-t8PD#%AH#f)I=JD{CNcXhGx)Mt za3W;^yRLY*4T85g8EB#h*FIBp zr?$CXl=B0GZ}iXMht_YZR8m5MeewpZAV}+^h?bj`+O=oTvejR{5qc|WMRQ=ufhYwk zs`Fq`8ydK@pE|aYM2R{ys8@&{ z3;mj@ZaDW2q`L43hK>w=Ru7gi235NFEMqA1wPYqr_}*{WPLPD_^sC@t+5HvKWhjAq zGC2{=J;FpWID8Cre&>0_+F?De!%8>r8Zhu45fQwu@%a#%%hc{#u&gPr7`p56wd%1Y zJw|A(M1RUi(stJlR62LWLqKAUd3Nf1@*Q5Wj5~ zYAYa~+oUEbM}LjLrSSDGVh+E#n<}b&(XPqjN>MH$X@(Y9cviurG$TpyGPD|&>?k|Y z5h_Z1f?D0*V?ZOnx#(B{#}i-FU;t8ShZ|6cn&OMo5SypL)fB6;;ZZPIHU+`$z51B8 zahdNHkM6znvti%tW@x+fyyp9Xh5TqcAbZFB7ulP|Sy&gH>hs6v4cbqTl(`@QZ8{5r zJ_7DCldlG~GAj0UUtTsf-=;ub#aQv`sOrn{sGobSR82eUF}JvJCt3Z^YAy$J5Zk~W zTzs8ZTUpG+ZK&$e>s7J@+=3`a#214l(C7xd{LI&Bj;Z(s!oNc1b-o=@7gMii3?H)? zi#Mx?|Clmp<};}T5jrGajS5Nj1dT2eI}s8f;?z;J9Fo?c;1$cJIpEGTr&_cG)aV4H z%t9tu`Fbn@)nCP$mo~HV=iZ?xAqCS1nLz1Eu^kj-%AaE2SUigDE5$9lE0t_tm`7Fe z@2%rrggGqB(8=7RPvEV7pZwsSa@H?AVU6Eg;Rj?jyl)a8uon0Vc|2b?cX&rTB2>3o zk@&~6=Z+;`<(Kq&Kf%nPHM`a$h@#qT`W8A2OR>reM)7jfZd*FCmkcCs%!?yE;N^bF z;+D#lp*qmaV#v`ru`ZkfSNIe+kUlrg~VZ(`jiuZ7cRaW+l6 zLwaYcb&L>QlYGVJAa~DmRe3IaOE#V&H6VKKNt2V|ql6c-Mh0EcWtF6wC67bHXF`*Y zxyRgZ;w8n6f7zu2g^+e|eOU{2_Oo8J^P;w$d(F2L|8{D=6A>;Hb7cb)$Z!+jEdmhg zwmm?ujx28wwyMxjw^y;OfYq>*OLryM?{kL_pn#ZsDWPY1mcVSQ2t?zPG|)z z*~Wbm{1q$4aa?h5huA;8_E{ABGz*Xo+3D$o1u&{)|#Iu|%WG zrPnvO9d@JePF~~tu=NBu^0eD=uySd^D?+gV+X$l%F?X(bvB8}C8oqJr|CC^VeTp``WZ(S36D1No?>Nkbr?4&#vP8`<9nX|?6o3>GKnq#(4*ZW&JN> z2WBOLPE@6jO%_VoeQ`whhF$PZ^^}L zD&DTxK6b62;vP>JU*?N`pvWt0>JFaT`Hz%%zsRG*_{PDvv%d$zjqF+3KF2LEvmN=? z)9M}8z&C(N z4d95+z+!ri8aAWjjm%PhxgIGx6b6UC_5nwVDLFE{8gK<+3AEs!pvXjY3}!p_>L<$# z`)`iZY{Oosk?A_H23WsJ6qRNiwL>$hohp$kyIX8Rf5Y3+ZT+z>P;?peeiGpOT*EwA z%#McEwOQEU!i{2}s`nRsq>?9h?K~;QcQ;^R__AJLw>BE`B>(ZX=U!wX>Hd*HDCt{O zuFYfpoORcHQh7LeWso};fx!Z2$d6hMHTNT$M_nPXQhL~7 zp1+rGMaaAfatZpS_q$kLN!?Zf=t>{kgTox1Cy4S7UJ%AwRaEx!r3&f|=I)nM4m-~z zne)n@l?yq)jT8CGdvCF453&Gf-0AZBjhYrE+4s~`&)txiS%k9Lkx(k<8Ml_mzs<#? z1`G(&u4O&Ez%Tlbk{VSeZtROycO2cVesPyq_xGuH6;H3+$X5O*2U!iKHdm|!;5eu+ zfYmxo6xrgVP@KudVCB?|Ja2{D{c=0rsChrPwbGa}!r85~GxK>azF^h5>FC$eLrz?b z_099TKUe`BL+6YhB)jLaG`AX;KnSuZ*#{F3%7~x@|5Mz{1%QF#%*D?&Ucl_%eWuO( z7aOVlVaD$2s$Yw^5L&C0331*EvamLx{r&L4-n}7;)_;E7&K_J)CZ-a&Ol&OOu_pXn zgA>zVwXfjScl!0{D9SQA-|gzufcvadQ@)SacJ;}Z&lnesOUidiiS$kbl{MPpprm;?Hy0pOO~aG@^il(#Dx{7O)wT zr)V00`Qf(byUD;uH%v0hf&S5V=0cSaDSPdL@QTTm7l0~p^7>2D1Vf*E@0=T-4okca z*4fqc7pD6!sICu&!MhOp`}lOBLNo_@t;d&BBn|#u zb_${KO5gRwnZa{M_428dZL!J{O*eh7M4j5Q&vdS>v!PPUq;S-@@!GqbxHO$#KlX3^ zw0E(~Vo5`wiP9TkkiY8vWw`>+rm{3JuoM?&R6JWTC4}+Jy1FRv+7hY1oK(Q_=PPP} zyRV1ULS^OvAA^8?-f89N}BHnHH|g+##( zvK1`CnTc&#_i9iS&)=Cb5IK<*{(Q!MV#1uSAG=Oy0Nga_H@Z&688AkukLNt%=6}4F z!M{N<^Phsr7^&b}Q)pQfZx?|^RxMB4)HBlw+aj!wi~7i$y`-#7uZ20R_hH=yaVSp& zo-8$QCeOEHNk)J4k&~8Bk$9nRwK;UyU;EaV6Qf+W{q{02230;e$M)mxhx;;7f;V$| z3UGhJKxyCov>kp+YURhm?mzbyCx7>#MkM^9$Sl_&TUbBUwRBCsV3H;#k1MhAt!<~) zZh{GOaXzc0K57b(I|z@x_i+vBN0vJaq2&x@_v|m+c8xR^p;0<%H7FfmR{!j%-L*^{ zy#(F^O1`v z*&ir8>N5AK;Ohj>f*H+L7Z}U}kcH$?M|@E*jUL<&mBM}0ooetxOxYRmyzYC9AeH3H z%8k_`z>yn|q+x_+i~rlHq6(a-;VsECVG`Ix*MVK`3wqU$spnK4Q=H!8=3q ztXXiJkIu?`R=!V~>_KzCjcLi}R}aYV(IQm8Sr*Dfj<)`a(oMk7txzZOt?%A{{n=sN z9YhA1f?`=NJdGA)+{QUgWML)(Fw>DB&+-8XB@)NA4lkn$%w}ZPh;gWSkRo>QU#+}B zz)O1QMby=r2YBwO;3y^si3jBlOIU`t#NNQBo%e|l7Hl3}j_)4>y92!}T}_fK5&I$iGzI; zi}N@w?5(IBZ$x51 zlf18g5C(P0<84sA7D;8xSSi=R|D%h>)nMmY-Z0g1qSD)fzZod>RJF(OP3hH_!x_dr z-z}n7b=56j;IvN=GqH9&UtB9rN|fhJd3`!%ru=z$5qTl~Iw)#eAB_0LJhb%(5MI_0 z%TX)ry~ZQd4gcS>bhj360IQ(ENs+SZS<+ef?)i*|9P4>lur^>-+^v>RJ zMd)pqBq0)ai<<{;ZRBZ8ZAv?vX4k)E>h|1^YXD@lIRRO(RW}2dgJFezouSCOKwDIl z&ER+(!f`YfEs*>~khob!tv%0Q=Ksjap!xV6Az94thz}ZNF){->`j;*<2wY@4Q@vmw z-}w4Y%=^7lZ6YP#P_#=nCBR#+|L2INfDG6cRJo05uwiutNUIVr?*18_>YGYf?=OmG zQ4x9xANcJ?MEUEgtQp89L5aU?_$$ zBz3+8`>RHa+n%1X+bePv_lCS+xWr)>4s3V>r$hk!4>xjM0d3@#;M-C`0CDj$s(>g1 z)mwoHBUN1WE&F88O*HLGX=U0Xqyz2&%Iq3A*dNF7y?RMzKA0V(12d6H!QzTJWs$`% z{VwgdR9$Y(6?T7YW%|?N(Q#q0b_8Hm;&wrfvsay-+)fJLdO~oC)V+tUbkx=GQhIeb zGcPy#*G;99&T`&19?-Fsb@0j}xC%oA>oWdvems+{Qw>qK{)A=LDlnID&6rXH{9G3A zG8f5-?OCSu?PaB`4kpCgsvcq(o~Zu4{rGLcNH1r1nb?(vo~MSls1eOik|Q0D`{mJ~ zg1khNklautdg%^1EMSkI2f6wNj{!oLzUy!f!_?N*+1{_K;!cFcTd>UFuo%r&g(WfE z1V)MJYLuDmRVS)VM#(Ls!is06s9>40d?6^v3jdHh*g|Gu@PpK5rw3m;$aE!EW6FR= z_mRm5asp}vI*9PvVf?DbhW@v;;|qxEc!okSLi;|dyyM;>BZsxGxIdR-$iD5d&x5I> z5_jQ{rT&W0C3q_VbZw$sydMX%oqwfIhw>?HY#I%`a(|l(JJb(k=I8sq&S)tq8nSut z-~sY{fXTeuRnLFsy+V$Asf{I5kB0S6U6n;&7T3_i2TlZVr>Q$=XL zOB3b8E|JR zjCbZao;qjfRC#t!#mj@8PGf!}W)IB#BITqoSrE$^XxQT$ej+&NZnJ~9pKh{WSj%LI z2IVSnv6dr#OTNs3eWzKj7b-%vI#zSeZG1{y8F5 z`qMx6ME8+{^I{X$7HtZ5JY9-~)g&1tU^{Nemj)CHQx=c5@Qieve~ERT99eUm9)4)j zt~&W&p-aU}=WE~JJS!%$^Zb6sW@%*y9ZU&|4V$gHH@0?LbY!yA$$bB>qJ-!4g+Vt%R~s>|K@tA&Ewg^*@>;|Lg|R)y~rIl;knK9W^ueQm;$!er0*7gsu+j7xeq67&Dsl`kc#(*`Qo}xyQRFxZcC=Xjw5?I42WX0 zt0SNhE=)B_d=;RcGhEtMZh12@)=Dvd{Hpx+x@F?Dl;l!xY&~x``5_|h^x`k|HsA{w z%2W-clb;2Vw_wk7U>!aUhL<;<>(ROMN#`>veA`ZBO{fWWs`>$OWti*AK3s>{jZ>LM z?85E7^^dQSWVMenGhw)Tv(@z%p6Fs_c_o=^S00^|flYgt{p$f+L6}n{Ae3GOk`5?_ zMGRIw*Y$X^v#G|Fsnnc5AmMbtT)H8c8R4L6>Vhh=GE8 z_i7~qGJy>tO-iJmL)`6auW;rxcj3;B2SU+^hKs-?)5*t5WuXm4tkgeo6-`DsQmJKWPCS-^INK74(Ba(SKPhENr|3 z{3yIwt~D`z>V0!LX|TW?iGqzW2mbOV+d!ziV|jajg0-EgmbE+4-DSp~ad?@6GbmZ& z(t!bEm<)fsAavY>kc(;`cgv9yyITxv4=z`%sf61eD&kzjFQjoEfzBrDuf)1&=&4nB zH`q+>$9gjM#h3dA-aaBQL76`o2ky%^RtEk_9zIm-%e~I6U^}#smCQ7`(W7!rzQl50 zl17iN0dnoiZTr^~rxwbLw6@W&g!kHAY<}=lhVx2v>-9r}HAcOwCMFcaf|!cZzbIKO zK*^2*lq`xQOH5<6H6|^$yz)kAUaIcysw78iUZ!&%6NbB)7Vt?Yrj#{oP{apZ$ZoDQ z*N*)g&vhZG1Hq!F)OA_p_fTPt#NHXIk|h-Hw%_da(|?85mBamcN8z?J>CiLG&LYl- zrqt9OdU?YSPBI0*+eh)$7_!MsqWA{eV8kB`N_EkDJSE`RD^1-zmNdJOvq?~aJBaH+ zNeAPjFk&XIg}>cG369C)gW)O*bIGc1KkQOzEqnJo-m-U18QEJS2EGFkkTMHHCq1Yv z0KOD6sj$<`!_0L53iyv8D;HGI!El<7yfTR}Igsg~p%jiBEeIvX4`du8&W?rq=}KgA z-UIZAE6qai6)3z=-*?CPehl(vE^cS9EU4fBf&P(&cQNi#|A3kBUy8t7_Ts$Reync@ z%vW%WQl;=peuh#D?F5M)?yrl2!#v%^zSbn|O?X)Sc`HgKWz>N!dNjY3IhD4i+{5+S zgbpSA?2kav`WCq0sEIcDT63MqZEU)fcO{k6H6X>^>ip+Dh(G9jcs32i-2*uE>2;@$ z&*5a8XD;uVaJ2<(W9{%tqHpR`sP2Nplz$B5L;f>b-<;NjGwjsKFlzFDdkCVI-2k1~ z3b6ek8rSvR_bPF7vkLfED3PLs zSeJ1%llke(zW^OQQ^MZ_>b-emDpyamd4)xob6`k4?c1>-n317(DK*&b-LKZ(F;k;E zFDQvk$gTeh9j$`PA%E0>9z%+0OqY@iU1Dq{89JfLkhTnkeMe?t#MLa$|@ zEFh;Py~K%R$<0wglv#7#N2$s-4H@^1ZO*9vs(Lf~Izsx4RA3LGD*Jha8zNTzfqh+f zfHf&O5?wsYuK0y%W%;&Ux`C~o%gdyUz4YbVGaKSI;5K@eSVe0WEy1wp9vVW zW_w-ur`dD7jK%Wu?}U9_LO9EJqPd5YsZCAKe}CQk{+%t4c&R5oZoQzRz&QEN% zc|h=`-ZET&&kXzMV%k*HXQdd1$iGeK`>1lvw5oq16-YU3(qZ{!aH!&;_Nx~v23O}l zWq-i}{)$qFDu`*KQZLX5r0#nEHH|0SxNcrs0u|Uq1enD9DhbG81Fi4dqQv9~PC+I37Pku(wE%bLgsE|BEf0eNd zMm3(zDO&gQ9SYqB%mPJLyKA7;`tV9V5>sH4Wh*xXC-E&lP2nU<6{6!7{Fc)L<88vU zobyc0e743eB6h&tgc9o0v1h(2y4~rJ(7L=6QB1%Xx;T+%NWjbz$4p00ZxV|g#iC<4DbKN0f25_M+{VOCn=xVb$ zg2s!hhSt2)R`YLDS(?ea(-t4{lirm&dMCfFMr1h=_`9IdO_U7Dlx*+5;;(DU*_l>J z;f4=d6SytNdJ`|66WpPKV)eE^;iHKU2K)x*l6RGrKK&A`ArV%fFw)RX4QoJ11A`at<_^|4(SDdQH`7rKGX;-%&L*V3l|z;^JzNj2VPoSZ4r6yKWEIVQtahO96wKijzXLtxEh zd(Fv%m~toXJ!B|;w{&}mA_=(1=h@P>E-t+}ofGvU?aY18p5wD>&<&&gG1*)%^HQ*> zUNrjob9Ywa$jHbp>E>Pfd(I^63U9XCa7FS-;)!NR5y7%2aJBOiaJt+ySHWwPXRYOQ z=H@b=Qs!m*a)0Wqh?H(n;>lI*r4#}6aQu!D=S{>}QO*azCI)d`qS2%*PM4ivn^U6= zjvXOtGXq08pFTeu+#@=2w#!mz(Ncyl0=E)a=g|YzoI-9XSt1CMz;N&Pwfa?+7ZiX^ zX}`RE*^o&kVRKsLNiew?XLfr)WC%|G4W zDs0Lr{Po!%-YfCOwpa@avQ!Tctn*emCk1$tBYcFLUh%$ zw}mM(yYsg9H~nmsDt1(Qm$4<#&SH|#vcTP^$KN^1fTIm^@pL`zsg?b%#kY*MbveIm z_vg`R4qL1IyN!Yp-^gOFSu# z^}DV7o;J!raCIp4%2K1E;QcgLm5uFPL2Q;c7!3jh{DGWOV7#C+`ngvy zi~FprsEqQ`v**^O;JyC)&|HHBx(i8C@W_Dp%pt%ptwlwv=Cz#ag43VVo8Cz^co3F8 zp<&5P?%Pa7K`d60^+oKc3zyIw@asE)?3?HGqib78)0cW6;sSBip?VD03*>*OK`-iF z^O(zN`mFElhxRK~Jxk$tqlY~!gZap{i=fshfDaO5pj%SW2w~>_rM;|*Xs@0eyQz|y z{!K>-r^o-a;Np>td-lacXJ(2%QHLw0!j4UR@WkTFNK0$Z+0%pT7oFV{I}A{=p;45E z0X4Va`J6!IoGSMP8>GldO8%a$aCBx;HR|xiURA+KQ1m8ZB-82s{GD<%N|ah$n{UEK z9mjlm3a2Ld7K#TXS*Bmgs}5A+qs%q~15@lVFym@WP=rlQP`Lv>pYIEL-y;DD?<2CR z8B!iCR_Uvm2?;^nX}CO=C(k(r=&`Lqe;Ew_^MUw(g{Sdj4Dw?X_qabt9JLAMjYJG0 z=z?6Jb!HA}*h>@VnJeEe2Od2p&iU=|Lpj9qIf6l42>mcSjZwo5QojKC5Grv46BnXi^~fk+_3@!CV;p;V2u36M7Svf;qt# zzb|g(@@?|0yeq<`*E#O#ml`hDRLq5&k;~-fuG0u{uUDH;^0Vy>@#1_JD^H*6KHA0m z)sH>@aW~15IyBHMIwDdrJXK=@GW9>)lU_-5gxJjL7}VZfe@jZhTKZQ=3x04y$X!i1 zei_CF%^l0V<3`EBwK+t5E^~3)Z)X+?aD;U2tb=;8Cl z%RJYne$2YIbALN-KdAci;QlK3Z2Ct>?>f$xFY%-7uS-91fuq8vwPAM3zL&eP26-8n zrVwc#&Lim4*Wg$bZ<o-~(Y$BHu>kwk~%D;+6C3HH$ zWL+h&XbCL2WcuP#MqxDp3FW1;nZBc85#Z<+GkvlNivsh>KP6m;veSAwXzpE`PIb@@ zo-@D7KgTl^2H+SfO4G%!VOiznC`-vpT?Z<`Bwgk7wLvmPjUIwBmQ>%AH{OIha z=D9_4{Q1H$Tne%G{iPUz2(H73ltL#y$wWpsIq3T^gWugc*ifOp?~s-H``Bj@?60q$ z*_heNJEE_@e1sCOj(=TeZ?;YCe^cBh@ug_N(U%`LXZ4^_T?#NxQ) zMQWdC+z$4s9Qd6U%`DN@7;HOxMR*GN-A4b$82@KK6(sDL+)nhL=G&2X!JqplRo+~* z*d=M(Y3F-8rfRDuOi0@WS>v7#2VFN;He$!l)*!&KMj>o3lbRz4d8e(9_w(7^%$%>l zGgRGu-TrdX7XF=<&Rl9E@!Ocs-GIs@s3oUY=C?#rU?}$;#AapwLge9% zA}2&Y6NRN2!<+b9&-1%X6-;&;vfJlTt?hKmoEXoa?;`!3HA>fr3wHG2RY?2D-Jr5F zvK^xt4J01dAM|M8@Tk8G?^z3^a%I~ak1pasqi1{WR~J)dw=&lrcMtR|q^)11@>d znBMU67S!|sLZBo-ChS2EINN)-H&bI(6gnKp9J{iS53Q#9hkolq3OqoDBJvep@wBGi z8FGni)uweGr8nqz;qtfi--Ssd$1t9E!)KQd1AjBQ1z60PHTvTTEQ{tadoYWQmfghp zKzMHC)F4=CvIYab|9G(3g$%d>@DSH;_2~^-SFoO40ep)8y5l>)xj( z$CYFt?@vg@bdA0*ie^HG=NZ)^*!Ms;6JY&Vod_?>R|O5Ha>^-@w4+%$Cj9q=?mXL3tKYc2DTrw~zyuR~@C9Ljji_Tk{D)D*D}-8Kw_MXJ;{!lRR*Pc|y0Pe)Aer z2~tOR)&6o^+aI?px>m{_NEC)2bs9R`rfn^H#$D~&-Qrl@U$rOJJpxmra0-2$HA#n~ zvoKXmo@;M&GuLIsfwKfYHoI!Jw{O4eW9P>r z0cVWNMYKfjwiv73+7;YlMeHMT_sj96(_5f&^VBr+8)8le zRG;m=emnEVAiKe6g84f7}~DJgeNYTV{q zc10@Bem3)gNq;kW14-KlL=Lb%n1{r;`#oz}(A-My(C{s}iy575j0{(g&r#bgTFrA_ z)w(2}DV%NT#AM$5y?QBF-^P=;hPeE+QS&(H@_OMUYPlgL!xw`MLn8YQTk1kjrDC2YFzTrL|E3$A|Cc-S z_uqg$?!hWX<_UvE46~h`2Jc?r7;@9uuMc432vVW`7r&O?Axz^=>wp9oaqgRFUs+>k z@cw*Cf#t8lHTNIuhzWgOIQDj>Ad7sP8bg&eZ*=){3s&p#_}nHd?osl^dBc;s?N3Fv zZS>_SgX8ZG%AFb+!`lKiu(^1;tKQW<6K96_Y`WqgPg8e)V(Sm}WDBL_>_cxe#ReZA zehrI-3J-PR|b!OdkG_U;?L$4O8J?FQP<9Mdu3&C_Ti6RHE zRb^IONQttU>EYk_tC1f*R=(;`(jD`g!dH%A2btxwAezC8%EcQ!WxQ{Kke5Y8wx(t_Nf?T0 zn41E~GIxRiu3cas^XQzMMa;q9qLJFT^+M z4Hh#|_SkpY&(l{sL=G=6pE}CixM@q_p(N2g?g4Fn;%j;gKzDR#bY>Xg{X?)Nn#a)YALr}Piw^Y- zr>VHkqJuRM%~cxZ2re^}lgA6jbw4Z`ip%rWzxtlP#c!_Yjb2`A7ey_Ox#Xb3+ly;M zY$x9);$IuqVWQ;|Coj3H_YTH4dwR+pc4-yebRoP~P43=y=_Cx5Iu{M%dPVROb%VNf zU>dQt##E&0ho$VKYi)vO&%6h`f9d5-1Jmrd+wNahG~$l^y2~}TNS&K&Tm0JJdmADc z#^Uzlx}~q`vlG8EY{HDo)!r`+pT4s;w>uTp5SGQNnyQ2|3xe4f#TWQG8Jp_DT5Bz|vT} zW+!|^jjy}zt3n^S9etSN!g+o9dk@OI@4*dgfcYjKgJ4f+ff2EMRzB!h^6^7pZlSsEKQ^2>BkvRf{q3 zhxB|%EdarrshpkIbpYC7g`M+9OC7)>S!U{-Y(yI>5;yywBJ>`L_5n80hwg-8ELktm zzXa|M<#$ju_D#c;>oV(^Kbjd0h@wlQ^cDuJ_^At+3h8^9Qk!7AKBOx zkn3O-w$k5*`L^ui;h4`9UPDe(a#Wu5k-?S>$cVS7bC|3xliXLMc-hcckD1h;UU|+H zy(TeZ2-a@%@Kx3%yyY;s)AVS0q@Lcj`UUl)-AB1#1gC^+rIt>%C=*!uB<0ZbrjG(eo31>`fmO&IZ(fC;ZcJ?}R6 zsRgQuu$zl+{Ks%4QB0ROiTS8gqmO8Cf@x-Z;SSUWSn7-iw`9~$v|%d4w&=sapAFph zeGYY)9l+cN)YRxGb$TC3&nILtZ=F8s;y=4QwW9rr>6Ca>{Z_!rj7> zGy%qMz(O=ZTw|a_w9f{>E1L^%pI{tc33sw7#E0rIi-1LOZnRAd+ka#N69gfp2FH(t zg;SQPtU*}p75H|!TH zrB7EiYDy6ZkK3g=V zMvpFGPzK4|lCAQ513e7OPnJ)c`E@FmoBr3C7^H|!)>}&xFLriAjGBqb(T7hRmJTfI zUw!ok;a3Jb!u#3yIKCf9Ww*KQVD>=-*2K@hn~+zBB4a*yp_Xk=BHsJ#XeRLG%l;v@Q=fs)a$ln{Oozqa z%RQcdYY!`VR^k0o=kOo)ELSf<=RW*HSd%*@x$MwE#~yAw!1rnE{V!8u*&E}m(pqW3 zL97MHP&v>Y>W)`_B$ZI-1~d5xcrI#Wo-=NSS8qK9E&Rl8Vk^Sppw8ZOj{90I&YoNG z!RIq-IL@-Xr2D>2L}c^Y!ZaQ;O`)43CKrkK6Ew}8#PpITTMIMijHUh+ zvNy{hTWd1caaZg=9@R(|Y>ahLIN2`D$Ftt`{gIL3KhvzALsCf!xK>P=Xv?((7M|W7 zQ*$mkyubTMdi4kPEJ^d#iPc|7u<*DQst40RT7lWYPJ-KKuGIs-T|SlSE1YysB=lu( zWmO`8QLzpg7{dl2Qff)*&3@>pu*ALA{#^@M#HK#t`>@C@PWVRhvntnLF`uU8Bz(iJ zKiC}`y5IK0RNeM4!Eq%Abyv1ev#stl=lGD4>-$}m%xM@}r{<{RUPJW>{?^9(3lE zc4HBY-|?9{Sz{Jg`%lnp{KYjB2SW-Mzhq07)sPQ9NxFKYSUUTB&#NmwRmm44R#5z% zhv6+Hqxb=ybqRNv{cC2lfh=|N8YgCpU5|mCXjF74a+@d|CSx2ubn4)&0Bi*jSk~=- z&=!JcCgChRP3lol?P2HVXI_Vik4LxnWG=SJ9XwEz7EW1Ei@frQb?}FNMXca<`4dal ze3EkQmuC0t*SDOxBCd3yaeKqZWxF$OLz)O{_&m6HZ!xU9a`Dlz28c-1=gB0WJjZVx zR*dv#sSM}T%Td1aYMa%cU1O%&1@74`CND(z=`&jVl7G(A<_u|cP9wUJ?)@nU(oxpW zJbYq|{(vLX<^38FOWG5fDIwlyZ>UgSRA75a-1(T()x4DU;Y7`Pb-d?AOfePADrk=| ztP6TERU!}PD}1uJsGsNC{60~Q+A9#c?j?_B+sr=lEax}j5-^HqquHMBWBGJ1)9X9g zhIwwtb-PZHpJN5PWLJ-}@#PfKDd`=SyR2&> zG3vZBoC~fOTtxpt3yxW*CmU+QOYF~~w?QidEpd8-I(t`NT=$GH7EM)iVZUKx-jMam zQTmJVIKn^6%4_;T-lN{f%!d^_3oc98GZB4YXNn4o7t;%Lq%Lx$dE2SsB^Rn3%1LaO z6YuS38hvwLTW%WP|QR`GvbF1m@uF^2(TKyILs|EFau;x3JWtgOW!ja%=l?P@_oz}8;70$`X>Hf zIZHcP`bh)98i0#>n?uo!un;RTjfURpl#`nBZs6o~Pk+Qv?7)CQtY-fo=@TjqWOcT} zmCvEx*EHu>aB=>#-*~#a1Bv#Iub5v@uTfkhJ0$GAzHSIUi#Uuyt7Ye;hD1CTi;~KZUejC z*AusvjsCEU9yyWmh)7Wo2uFOWh-|wqc4x)R7u6i*a)n?HsK3TQ(*aXb@Sn@#H|3_? zyJs<2?^)Wan`-*qwWpHiWpCSqz4ur-41f=5CKZ9&g0*4uMnNef7HVqM!*?ifnm#(Y z`%2i?RMSbp(;-KWkk?z{N(sUs{Nm z#Ks&R%i5;J1EdP|ij}c2?Cz&E<5$;Js{xmb)xEq25Xgl7pNH^Y>GJ>C2E_~Ln05?b z=_~u6znIM9JMA>2Pe8zE`*xoJs}pxe%!~cbo-(a)+K2FJhE8#neYv1GdO|ci^eKlO z2CN3fuimXX`aF#2a#m{Fz|%Ir&EXwazbA4b>-A4!K#A#B@>CfQzY|q7$9j&+(EELw zR>41lVy3eiX{8;nipmzGr&+GKsPWGIS9Q}7b9F?@HKf3c9XVhO!pu~Ud(@dmThia-BLYJ&H5jbF{Ivf%%!IO1hqkVxW%NJh8XZaPjD<E zRBr~+au8(;Of*|+)FK+UEwti08}n>M=j?EqgxPshTzg=EYF!o)n^bl~E4yRsJRhWl z%~MeQ6%(fmGw`-QlGY?-^!qmdZSb>az%-7w2sEQl5blWWOWCh+(IjsDQFFN z!oxQji5?UGb@r?&ay@~C;wxnHZrqAc2k4b}gMyLDyEbP357El7Fb|#OW5mxjGT?Py zhiVOa5O$gwxMA)Xzt%A}PaG5-u=+G_7;Y_}X2CynHL4~4PZ5Rn&Eh_s08g>2`UGE*$HA_5{^LZTqOL_|PokuJ^9ks5mFRk}zLn)C!A1X6x;eKT+7&08~nBm~wX z`EtKo_Bm&ty~i?KIpIOm);C)zo^a=X=sJiGjDZ|Wzz(193_G^wZ4~}vrpV5I#RF5s z@7cw{x`927jjTZ+1*&&Nw`n&{?LPmHO?mAVH}Q^M{XwD-Wx!tdofdSzLS z&x~8zyUHXlfqoyk)6mBl(@?*aLWsa|dWzh6d+h>q5_z}j zb?sNx5lc3bsD}{pOI*;lp(0CxapMb5rM&mB*B_fK$uohuRu;<=?fgCevE5nSFK_rk z=ZzlER8Q%Q!8}Ejzhv=ZNzyE?EoQD4E*Mi;l|i;?i$AKm8S&8BnO(VwHC#HYP=GO2 z(MVU@up6}4<%q|iI)m}1y%zJo%=kH7K2!Gts25Rvg*Gecdw`;h5BJe_=?~ zWq`xzmG0oZGn}P1PHcMqy9`48<=Waaspo?Z0*whsdG$I6d4QTjyPAn`rgGusf9M&?P?FuS&v%KK z1;lR~MNP3gc)6^1B5}ueXK50%01XnW*F*_`c+W8x^kRHUJx8Z1E8E7|OmbM^cSq=- zCx(w>ZkA>X_Hfs%7zg+#8eVVYwP^0Zrv>*q-Nu#|m+kS=<8+^Z;(`eK zjTn$$TJTJ%7Y82x{3D5Jdx3N_`W7p^t~yYj31ToOciMSCb~=NmBFMY?7g3l7TMx}e`UZrvD2Gj06`2e&JGZ9 zEp(+&b$h81Z{57>A9V9q-n_A0IjAc9KszVs_VsA@M0KAP%XF8eAG7CrmAjLh2NIOx z{yf!WsGX>#z=k}Vaa>4I#FbK3FD>xv@)Q^LU+LvZr70-AnAth@jpk%4czep}omO0Y zCN77*_vQe6fdYu+CvJd|x=zcLg!3Stj$cfmJB#VZRlJcMAyva-JvZTp{eU=@K?fx2 zOyz7ljL!us^(7?$S{UfzH{IZQwb2-%4{0g7GfUTd+5lJ-5`SKJ#2f{*8tMIP1_07Q zZ;`El2HV*v!LTJKCV22GxGG|uspyu&aXGLcizXNkKr%!iJdb*7P8dPB`;u(J`#H6s z@o)B^K}O+U-?O;ki-Q`EWVN~fb|^ExH8GW!nI~a#yO&sJ9&2FLnF=g}G>aJdz;Mj3 z@qh8yD%ExPW-t-bGRw1Cq%z$@=(MRB2lg#;xKK6r<%+ZFA_F=S~TMArkt zAzIN%$)Cb?k~PY&2lj$5`wW*TC;zA9x0oX+5-rRyP1UPUla{DO$bo-fC>YQW3t*>u zOIj%>Vca%GpbOh8P`zW;onBV2(hu8QM;xOFQ)iiFNJa1&cYr(8jGN#a(AGOs57FiL zeE!5*0Y@dGS}*X3@cCvl7O!Lj&!VZQwM`e=*I&|E@`}?!N(~jpDtO+>vY*($caJch z4_Sz;{`I;R{Y>ht7an(8=WRh{Etk>aSu>NH{-O@ojMNXlo7GXit(0Ulc_#3quz_jM zp2aAN3-%Z_4V&Xn1R_q*vox%AD7;RmD&EKSKQ_VMCsGVci$A8Vuzx+W$G=d{^dtk( zhpI+_>iF!bj(n$6`1*%i1Fvw=PwG=m0^Vh0uj(QCyR97k8JA$E|KQzCMw#{u6)+lR zu;*!T2gD;lxH_JJ;|nF&cszKWjfR0URf1>Ofu2v9U#ROGUZviKi}DNP_9z3 z%p}$ccX$+zr(TWd6Tbd*RL#c7ppzx~v9rjuL=bxf7pXb*{im$dF0LE@hN!Iyi@M8F zYZO8?=ff*6lsLqbHroa`X@T%iAzS`8sgVt|gqIf3X38c!1HuI({KQS#=vih14l^j4 zG2*r?j3Wn(A{rvQMCU@N7978E`YgbX2|y7Nz%57g_%~i?FP_l@6I3-oczR}HJ6RJt zu-7gBw})iG5H+Wte}4TIk{1D`n1<(=){b`!7>5iLl=XNK0zNlWi^Lq-XbtBC1IHMB ziR^M&4QLz`@T;2-fOp4mA!_igsCD``+(KjNH~SfabJ!w>t()SYrx=RljUevh-Y$j3W`NIK z1$zWO=k44SPKO1E(#;@UOa|*PyzwcYy-qc`uOjX7&~}}S$9AG+ZYrwDkr%b19}xK; zn}1q^BdVK}7yyFpbGwk`n}s}rwy2%LcBnLKc(y#R+4AWR8g359O|yG+uUE}je`=_t z0crL09`dcl9-Lvstj4`WM?bmMD8qO_u?N{~*NvTk>T1X>?EzT>dV&1HuvLnu$Px}U zFxDU<{T?s}lSYfTuDqZ`!dr8Zg{bD|s2A?R;QAr!dn9xe?w|pN__Mlv@f;=7)Yo|56uU-cmpuTkYnAevP*euVXN({Wuqe)Kl+bT4d%QB$CYfc@D5KsrHquu%@L2LCy7XK!uE^ zHnnSfoyO$qz&M>4TP}Zl%XRYKXD^RbHJBE5IKHUVyYdo=Vzthajj@xhR37riY?u&= znisL*z%%O?xyQSiJGU4M%{*$jeAe$z?)CX+xNABo%d>H*zB@bEXpnz`OpjAtL0SNC zV2^Qkq&-q#w%wr$DWpsNbXV?p683uinx9HnmAKzby8PZzW-g9UhpNJLV1$Hc-zR^g zc@wAoeXT8>qeRL}?r$~p^_C`nqWqO~5eck{|NHQ>Z5DqlVr!zm>ahWr*e%RV-%L)z zPEjI1C3~!U8fsWbm*$RU-Cjxi$EZ<`r_fxC29*KW@ zOl6-b_egs5pL!%WASC(Y8voMXHQVD10Ca;x5FEvbN7XcbvPIKFlsm;ef}!W{m}8^# z+b2PR_=x*%#(a3q9J?ez(Zyb@&FDnbeEhCCYu?1!(7e@C0_-or)-VSvHSSZiRw{`S zlB=-B1G<*RKD=`Lbg}yHcT^1|{&*SrMrX~QF$y3t^8MMVv1zIdGo0Z`nVNEpvI^>1 z6B0^w+%cN{Wt{*g@m%)riu3<_y*G7quiA}0AC_l~-ii@3zF|3%;4MqXgCfc?ufeOO!~h*Gj^blTeS@bvJ1y_*#4n5_TI*YjgL zS_=Kz6c(eaC-O??4N6gdUqx2RwFuq2RCtR8^x!8jlP0E-xGKI$u$U5Qozp=iLJ+=g=)CdSJ~TykLjZp+^|oO4on-q~E`c3B(3$mF z=k5O%TmOr-*@1Nb|0})@Nqa-Z{*?ifYK;L%O(A_?>E56vdAlfmaUyoD*Xz8{T4VJE zX?{N&s966Ojj-6a_UmaEG6>R^&8H2to%UB3(#syNRzw8}`}M}oUMnz0eK2hU@+mKX zH7ZofFN~BF9_wCjm~a20TAGyt9M(HMSF8r+;6;AM!HaD_u~X`-#yJwLzxF|vX$|u` zIOuSP2f_eo#zdeXO6RAQA}T>|$`PGALJ!28Wm#PT!+BNrr*MVjQI-{ubcYd&an;pp z?PLQJCI;rTDTF7YIF~m)I-NUG|?Y78jYt65N(~@rcoz#@a`Q6+{eJ=^M{kG zS7a%e+g_l9gn1R=32qCxH9_VycCRb z^M~{p5bNOlejj4bPAq?D3!~K6RzHFfu7LYIwHFqcH3BwGgFOiI1yzP-mDKWU%Y6~o zNz8Bmkw?DCn$UI!C$A$yy-X2^l5xJUMn4TSvJCA(dxuV~Zq3lf@z8(Nm?E5utHB$F z768aP;*eVRseR_Zli70$0J*HhCfU=Iuho~ zID*r|5lUcXV(&nlb!>?Qa%vJ}W=_aJa9(g*;zO;#vTJCnc-JdF%>-STW8#y7Ba4r+ zu%<_7?9tg)u7PR2~)8gqh-r=2wxKcFkHnunTTROC0`U^bmU`--!tId0qVW z;cz8!8IOziWILN29Ut!jFQGsa3HA7Vk(ZDf1zgX(|IbUx4;I2c?#8hmiV0CJ)DdP+ zP59wjc9k6Roh_8@rus2&WC`F6lrr=MA=l)Y@*p{nowCsfi}sn)QO29$Dg(vJXMEHd zQ@&j?=ECB++2;wC2I;q5)fi;t42tX?7OgfLp@9KA-IKyTEd^=w&wUzUmB*~7*G$%b zFh4#jiaCN@b1%UUS$%qotA>&9KqjW!FeL+P#g1h6j@oSN+-mOtkAf90mmkyZYMpvp z=M>q`-i$c#7)4?A!V&5;#f+;kZmB7@wWlx;juf9U#yJo z7z-jnLz^c3h@3=?JfT`tUYroMR|O@^CIwWE2IH&-N&OwUkowQGclNYOHjEtef{&+= zEVj6Rq`W#B{V?%?apU{(y`u5FX0@k(40RIm{5-mH5nr=AAz`2zlY{kQrhwfG9s!6l z799!@LQ;$TnSjC<12KJZycH){Wh7ttj)y+(vRF1z-|{W8BM0sz=6|^< z#{%h*U_1u^tk>N+0K7d18oG6asXEonxO&sf+`>_IP?!$cUtf@ zitQ;#*$aztv){TGSD^69=_2Kr_d8pN%m)2ya;W#{aO5G1bSt;mDvq<>nPajNb@Bzj zmgBY#qAtbIS}^{Ps9>PxT(reL2f_jDa-VR7R6sVh@*rVDgwpfoluoN2ntJZBJnnPn zg8Wu-d;3qrN>ilZ7D<*I*q z;l#t!ku)r~(-)lc&g#~74LGM-^bA&J-?duol!EMO5#lNnQ*TItkW6E=E!-UgC>(lW zrdld?a;jBhDtj#}q*0r+4qcJgx9U68zhCCI4=L{T|Eao_kIoh$| zP)W*7_hw@o^ZEu`RGs#B*lqG}ZunmoBn0_5gbaXS&kF7}vFfkTrY&cC_^{vAx8gWyWC%x{$#0sAPXtQnt)XLc9<$7uBxn!wwjg zpz?yr0nQ{+?yHRGWV`E;GMPPJ$5IZTz4z36O*6C8wy>l@i!wbz#(sl6MNbOP@vmo4 z-i#3Wn^#TMoqf6j0@WXmwT$~}@So9jj(2it;r?}L;3 zmSXnP@sRcMSc8z&=T{t0b^qR;l@Y*hpJakK$3ZC=SF+n8j0%Arb4B^h8|J^bFT?*P zr2m&{IoPvf4R#&u**1`#;g|nBcIbjcZRtc;%X?!k)9#=qTB$-bY0%y(06E$yGFd#I z)GciN^Si@+Xx?STG5=b$%)u7TM$)-IhE6a%n!$$wCfpo!N!Cze7a3w*6{~dS=I%ID zAUrYNz_lr+PWM68=Z}x0iAm0EFQ?JV?5yL(;GnZ7qt{@>2%tHn91!S03N1YHM`8N( zjhuGzt8Q*hD@%}?_jCTO>cC+x6!5)(=`5Bc)lq{T2q3}F`h6t($NJjZ+0B?tn70J= z_f8y*)L~WI}0w@sP6!8~-^}cyi_Mn&K6ez4+h1yGI7( z#lHAT{eZ}@3yA8-Q;k^109-s|M*}8>eo4kaBX!BRmh9Uao_68GjJz(YK>sMzJXYa}P=cU}vp$A?M%-I4Cz&TlXPIax>%}IDy>Vx#nrSMr?3nss2nHOnVT`j7YbLhwvNMis4A8HArkc_z zzsC8A)kejNy>->PlgJ6`#|wu-&T$T#yqAl3@53x?5kA5`x`IrM;WO@iNRP z{iPIXyf60K>Ko?#`0nn#z-2c#NtIhmqSYdE)dz8zaM~3x;W<&yQB!h|NonHfW(r{E z#+Ur$bdvS|zAmd_T7PjU`oze~T(_(vxa3+Uq!kgo^4Zkg%=-TH&`SjN%p!6ME2$oC zv4RJ*W2!~T^%#eRIoG1cF%?e+3pMyjKolhT0yE2>QW>cznK1V{E+lWdkuX7~N z*70q>B`wQ~>@KAAsQoQX8tTA$=`poT7$S&Q_<_g0;URdh!K(8={f;C5QyT7n{rP`8 zI{eStFM|dqVIDxSWA)$#l3_+3Z(;(mtTv>~__ou~%L=ACU-F`E!L7vmDsr*fKc4v? z(>%_v#^Cc4W*r3)%3&MIVplQr>@VjMoRzoE4U>xLomiWkPpo6`<-CQ|Igtlh=RDvR ztqcdaLYADLUva>HY`4he+L4KJnF;#aYbcC+5Q+dS;bnXy)Qw0DF3rEiqL~XnaJA{? z`wErW;V8kwfoKAnrrO$`bXA%bPbnB4#9CI&@O%gD2jzCH=#C56b@j-NVv?e>Qs@h4 zgy21*?OE3iOivz23KK?Q6ZqSp3UW%@m&9})FPLfw`DDEFu=2+%*o+6EK}=PtGqNiwsCN<>|j!-+cWPH$aZo}aK>KZHbq z5jX}1i~kSJ=M*T@=e<9H-<7N%CNzEs2&=QQgdoPT2va5+<}?rqVP_c8VfHkOJ0pmz zq%#9>?~+)&>SH7DSep({_<1R%``jAr)TUy}piop)#rsEDtBu?$h|_1*X{Zrg*V z-h1C4*(2eaPY*{!=EQ0T4)`s#1L-}6OEyEFhC(SO6#voiu%DP|ntmEe;^01$N6*p4 z69$)0WqE`PGjJ zh*@`jb*n9x>7;JuZStwGoj!LL^V5lOdX9CR9rU5mWYBBPc?gS=t(Oeb-1QNLQ7(uG z&^FsM3eGFj?EtDUN<6NHTesb@VhV+vQQ*TpK&lm zD-N+;35c>SZ;rJxY7ce4s6-M`inY9Rv#j?`g*}FdBAX6+TA!@cB$4UZP9`VKkTOfa z<1yUQk$_~7cZZ@D_09dVNlomNFYD)JXN6C$tQ1{+XF{u?j1ZQZOgX#exTAkdjnu2< z>F3l>?+u@)u>=Owwlp%&C+dWtl45c*0`qL8X5J6$s`^j;X}1hKsvXE-ZL@xz{@{m3 za`u3yytlsz6?V^Gnle|49hqy|-pCoq%h-aiip9KMLHIRvAy}z=@qX=4l*pNQOd8?n1&)KKwT}YMmJ86^$*YZ8mg9^ z;(dUWNPqkKA4>taHtvGMm8di!LHHA8eriPwMmglH*pUCN^ZssP{{%JYOtX&TBU|*& zbBrtq-vnWV#6ea!^i&iOJ_UKxwm_d*rqGmm*HF=-;1+(@EeF2SdnLy76AbOUY2p5N z8J^~GA#fdDlD!QrlRiPs}bd8c9Z~{a>})$T<_(_ncE*aModVWnVjH@5qjmET;sF`Jnbv{uee=fhjFk_S4WkLe9Fe#BPaaSI+$0~(+a#yE z%m7@9=9e&6cnl1JbG{1ZsJ-;?CPS9!&OTG2E@V8lckfqsJ?i-#EpT%JQ-iwWMDlw| zVUJMF3MfL2R#Gc54>>Sf<;_UiKRWH6OH04g<>|zb=Vcvmezd;C{aYW(+HSo7S?PN| z6ykTau_psZZGgF~6+i%tX}$~GJfK&Y@!2?lP_*%$BoI8SR$qG56_6R+b^2XzVpavM z`=n%t_83F7neb*CWUGMowT-Jo>(o+Md>dbq`gpSO&x2k+3=9?Ks~vv{z+{HXeEW~j z#mB*y0Dd_vMS-kkk)ix7=nKRVYH+z26ie_SQ&j{_4GLhpydD<+~Pn z(`EaYTH0t)noyMc3O*|Jk%Y(d_6l z2vz2%ieo5;y}6$ZSfRe+ceN)R(@IJlJ@WQmvf3}z1#Y!jPJNZZR%(0Z`lZ;A9~Ni( z@!zvMV_93CC@iX*VG~lBHW;3Jsq{GM!nvxC2d1M*xPdIzF_3LIEKZA{A3&J=9W*MW zov})QRc&AUh4Nj0Skscz>oStriIYH`kly`E4xZy^#h~FQy|7X6V>`Dh(@t2vH+J#l z_Fz2kz9#oa*Fyh5+`ETtLGoMLbos*2;&rDY`29_l)R%H23)YaObMyzS{@IG%R_>{= zhhTFV?x$N)77I}g7vL%z#&#RCJf<&cvzJoIpM|Rt(HRM!P&N>7%`7#~7HHYi+ZrzoHHy>1ZkE-L zYeS{&6!)K-eOG`EXFNOAsej+^Zl0Tp7JU!CZa}|}5Cc*Tr{RdFw4QHKMklU0R9B;2 z?CfoMK=Vk}93mzb=OuHoP41(5JB7>ESd;iozmEkCRaNOUi!ap3*vkOJ*UC&rIRe&k z+0JGMJo-%zE=1c57}5Q){$@O4*E{)igwy5kFUjJxNR`K~!%mMrXplc%Ay`b~s-4z} zlwc~ERt8`I@|+P>;e3zR8^xFt@ve{EX4sY+tqx+p$O?ZZUwX%*j(Y^5N$lV>vX8K){x|FPQ#H|*Z7}U{;-L=;&896XL;IBM|GeQvxaAZ9X`ei>8B{aOO!Z#WF-fQ*N zh;2z)U*Am1%?x{n$U$=F>%W}Hg$TzsZM}a94hE|E;!|cJ@{;tr6(!GPhCJ7_d;8_r zG?;JUi^27$;rYbk;zo24V+ZU23WK*2_i}@`g1%(J5$cV**-oa3G41 zd>PpZW3!=#(W>(TF(m0W8!wccSKHLHSPx0QY@Qm)RFeOAG;;ZPX!Q6fglq+v>fQ{Nw@#lgu&JZ%A+qtZRWzl zTPObvF1EH#xrjuGs;fiqrFtxO#Y^_W7P3Cz&VPfTU;VbTxZ0bpbgH4fka<4vyK`jT z+ZMV|Km$4$>yGsn?($Z+?v%c`FlCtYVIyuP)~H;}NX!#spLe0a%v#7KK7X|~rz#@T zrpwpgUryKkpK;C8xS;drV}VKBUAEPB)D3!RNMZ!!OX19})56R_W$uX@IRe6{%1(=zyIv{ zsk&-x-4Q5n4(vHxK0RNE)H{h5=2X#2KPn*}``%&?Tx7EojF7Sqx&daX*|k;Vv~o-s zQpMR@iR;(vhljboY*Tf7tuM2EQK;pMX{0>DE$4@Js)7D=##v#i2rYn)LiX;@HYtg^ zR@WpTN3&k&u|qNWH?wuMcICSaEebn&zs4Tj|IJOh;y97+MArF_4J*;wzf~EL)!isn z$}+na9L-~7kEIL{lG?D7J*3%i=uw9K$gB~CKlFQY|JISELk=zysGeKd53K;%@!Vm< zLl&PbwwbaWUZb~M_x&LsZ)=x%Bu_B8!RhP>zRfj8AQ<=+W=;`Q$q!)|NStO?K_!Oo z65U^(IOr4nZmGGa2`}(6e^0a-=xkx$!}yS53?G_9CIe10&(|yR^KE+=_uF!&dDmW6 zr?xP^yY>=i(eaW)JaFJPIylZag=HO|Q(7~o@SvPfvAaMP%`f-Qr5_(Cnee_4e!yfh z873<}c%*o!kn#9GHZ=Am*svN7(c`-yt|7~u#{=NA785arJ?)ul~Bp_bFAVaQYO1lUIER}!;nkgn? zfFex{Z0N`$$G0UNa;{eR0>AwOl|qLu^qa78tZg;=X0BiHdg8}`&MCR@zHNsEto_Fp zH`G=m97WysaU!N$Y%%5c>%dTpf}xgg5^`Fvrt3L3gy7$Ar)19w?0L&(J;n>r&zIn> zu62cvFG(|aOhKW*iVkND`f81Lh?;cRL7Jm@!TgcHo@eIWk4u@K;So={6&ZX*L|Nu@ z>ql7qgaWB5M7-*5Tt|la-NOeA*q@8IvYmW;V4CeoV{l}7HH2zyv!nT1yTMqiY{IjO zUj_2XbAaTzrr96B;sxOVJMIPdTX@BXF&xo=mf6$hPF$T-W$G%}mfw(ctvH>1aHtXH zD;IobZr&l10n)TV`z#PgiZ&iXh>kR#M92=iSFFo-bV+JqBj#*xh)AESpk?2vy|QSo zAfXALd6lxof^VIn!;phGe%Leah$GOb%|ve|6Ii!2;(6hLvnsyNo{C{mf5lVS zHRlj_*gI}%_HqZRdbT6J7=6`*Yytw2ug)g=4^>``-+%YPP5OWl z=ZDBb0m*|l6U1=RvU$QWtcMu#6?rR6+dePK?*?@ZSOcG+YplY{P+^Gs9*`jK;Y^q7 zzJ{Cz7oIE36ElMlXf3I$mQL?#uK`iycfsixQ*a5QS05eFQ#`ub9@ws}Esj0xuvG9M z#ZWyEWUme}n4i?()0e3QWPGy>Pe;PV*_UH^CfYG7)7-CLLsvDYIi1RDEq0{nwy5St zNI9y#vW5a%8%2Oj`cB8;kS|=`*|+i4pXU3H1e#6qSLj#bK+i|%07I60vJ5pP-(K|_ z9^NSNGI89mwK=QBW^F|WIym{IwAoaK%1ck1GZmpJfo$n_ZAS{n!}7uHs2c+5)cvQ` z)lq%kkvdzQ=KP*z-?~IT$p1xbLJM~)N;dT}p;*Q4SU;Q!xCuetKn}gM_!pKE z-!h^Yja|SOJMYyFFLbp8Ybk9+ti@k^E!G3)aeyq14C6*!L!(G`!Y)VExS&+A_ZLh0 zPGIw7I(wFz=dipqL3$vnnWK>d;Wi8uwUFI|Hh%may4LP%G<-tV?2o-FfAJ?nT>@Mu z&$AdP<;N=vzu~EH$*mo4Fd+I7@|rTK&ibX3vyj%Z z{!und(@`RWDj^Q?(}6bm%J^lAQ8K&QDptUG-6ZE#c0I{xuPMx5-whTxX})juXu z_En&k$KKt8W(K?&InG+Sb@_T(&wYk}&-6D-1RYYjwf06XGsuv7Fv3cHTg04t`46_7 z0S<#h%gj24S#FGtK2495`z~}r#X0FzqF6y$rZPkqt_kuF#uOt0wE>uZRkvffR*-@} zOMh6eB*t6cw#dDHr{LDML}ZIV^1We|J41uXzb`<77@{Opc%#CIHU}XiXlZXbEKG3r=NST+pz(*q)iq5#Oo^%*96qu zLNk3Re@p-5oLnmRys+`lN-klu>d}fX^Of}#agL$S(>~ssk2-8+`C8EAl0FwI1J5|! zN?j@eDUfG2N@rIuOEWGYg(vg;B4<3}U**&5Ds?+!mk&ISmU%^q4qsiQ)$xSq=-rNu z>fG77{lHN}u-J5AlXU`wrH5F-R1?}NqyrLa{i=e@9h>8SE_CMLjl7i`OX-7hZ%gs# z7cI}SvHgxb*ujMqn@2+OKOuUCj(DZVOZC)A%>lZnbtY619JjEyz|0L5)u z1{rrh*mBVOn_hd)PY#)tH{@%Mj^c$`Y@JPoZ2IzZoBt9$hsm2%77b*Ljv9n72swps zqgCc~M?dkYC!H)16h*}z^Z6DR7_4+2hXzJ6IL29e$7~544R|a<6kUu3&9%4}H}&R4 z+Q9QA_VvWKWJxw|WlgIGY_FQFPz{(c+$mt$18>%HYI6@0`Xtm5R@79j?wiXy8#PMngbr!vz~PU!cuWlJp$8ieuY7-iK4 zVG|R!y_}j4=-;Lu(qd1CU3*|9cMpb~SoafTdu>2Neu3Pvm|I$ZM}LkmrzKG`LDT~T z(|#!yQcNwoU96`VV^IDIg0c4S+iqz+iRz6A^SdLwv3ZbWiw=g#5d8Ju3!$geg=dT` zs`nrUh&!}_Ii)G#x%aarxV4N|c{aD+r>j#g#W+=v8&j0J@4C&1Or@@H`V6XHKlL=h zr=uJ8n8k$XmgG)J^0te0t|5Ev zPuDPG0ZG8v$f?Z^IWE+n@p4u@;;{;|?eVU=q6F;1mB`DBh@7k!G@-_EwRQuJ_x3BS znlIZt9xZ0xs|fg5*n$N^TcSLN8}DFDbjW+5!lP7voVzWIkbIMgVPVAZp{0uX&(_%buLEtdU8Hp9bNi- z@{JF`K_KlO>32z#+dKw1HpBjV&-lS1Ve>STmA9LQL4XDP_XdQYx{A-rz(TAb9>^Zo2hZ70@(df z1fU=hP(i4Zs_^$1&Wb-3Y3!|I9+nf}8q>g9(tQLGGS*|RZnmXR(7uS9~1oCZ& zmd?7hugKS}lig6MmpQS0=aQ(LwG6iTBJv|h#QZ;knE$KeEqD$4?|*Tk`QKxSf4@5q zys);Gwd7v1V4e6^Ji~xm9?^-RYSA>KsnHHOGIdFvj2$RiF5U2{Y?}XdBQXHEo){in zqmD{18tikI*L@YLaicOJ^r%GXz?~VfU^x7dh?p=jl_XXKSPKgXA8Hl4%*;AHv->DXm61J2s8M z=ZIh$XkO46+VZ=RxT#3_UDZwOFC@xUB#sH9&t~u$go-$XM!X?26z8I2dEwnT8~5Wb zXC^tCwenoP9kYwMoV3})IO|Kx|!#f3uSs&u&j^&79bvyT1`K3Zgp4upD%S+Rq-nQ@V!Oo|#O z8vKgwVCnzK+Gu_?!_i3-cuX4?A=_q2qnq7F71N*7zl(vS)4DC=8`xbFz!j^(o9?s2*3kJ5n07ss_*a z_?*A+Q_KdQf8T9kSa50_|d&;^?sh1eu``8k1G)G zY~e@IR9}-(iW_$h(4i(@Q<_*}oef(X)mY~x^_W4z8zm1V|DUMV(ne;RZuHCt|6r@` zoxMKl^F^p&*BjBAhrFZTl5XrK84P>%u^Uzndf9kYmX(HHb@7v6lk~WGFjEHNZT^6b zgPqL_J5OhwRAQF0ri-4#OX0zaIN!$$F39s+16H<>5}PZBAPEA!-LIj~KL<{+Dw>uc zIVgD;`7V|wKHK3A!qBtzdi3ka0hS874MZgFIX{(D8@$^qR<=_2e8$bby^A|ADe^j7 zm*@JXChy)j*Jy@_fsSwKPECbe0I$$>IRgh9z^i6(7>F^b0eo{JEZJn$V{4s;EMXpRn{i5s+C|xIcM*n7UA`674CSf(Ver$|z>MEK&!nOgm+j2Fh z&#DjR(F2FPT%}vzDCC7z+>)p)EeTO5&M@IQA8}njQsgZ^#{{l^+gP=a zF9IN)pzSV=tkeoTTN41vp#}{SE#2#lU3_z21RR;YvQ0c;bGx-}@4l)Yu14sjJ{!xe z{s8ga99ryo+6j@7st?fBE&|k}{q8j~T+Dn*Si_BVFMzi76Z! zsP(-$1--(!`~&}c&~0S7Y~EvgX{uffLK503J^p^()%3n{cYaHfw&~)VcyVuWFXo$j zz9#Vql*~;RkP|Jz1qpPwcuFFOZ$z#*^BViZhe`EwkX?r9- z`NrVw37lH&`TIgMg{~M%D?0RN^3I%cYy-}eO#K?vpfA5$M;K=a&(iNA>}k%#fyk;x z6>@QG8p%#^z`QcLa`ny!`|I%?Wd~k-)wF^>D$?HwM}xs0;A3zgDt5*d~zq^rX|r8bD!4<^L;m5yLYI=^PErpv!<= z9g2H_o7Oj|p>Y#^8;6U(Lwh_6?X+wxb;Zqn^b>C+J}N4@C%(m|3kZ6RxEw4^eU}<3 zCP3>Uf$cTkDuu66rpyj*qTewT@OaDQ8>~U0ZnJZx#p&Ls(1e zmo`>Md8*tl;pDG?Hvyc-$^Yj3Q<^f}lA+W90#d92`oV~sd40Y*cIx6{2Va1d$LD$7 zp)ih&dpV!}V`H0T`?ptBNjKr5wVc{@=QdBdkXv7Wd?DnM|LvFE)U0mZZmE&T)~%d0 zqbGOVqSJc%P+4Lzg_g8n{fhC54kg!sL!K?Bkh-`VfO6J!Yu~HLQ~P`4L63ypsdfUT zf^x}Og3*g|F`cXfW%@GHbuJQWlHQ@PWc<}OA&8)OdU15re##rtx_Gm@U3X7dmLV-h z1ejowS*12G;XwU7%(`&dP$RUGs}>c0aRj5@p}&{&U{jUzP^kllc<%K&ixnJ0q?8~k zZAjZAuaa~M3T9TVMf(yuVA0OI2eKSu^Z50UksL2rFABF)4<*VUZOHt!ckzzhan|My zra9v(iHp0v5mE{gPHtI^Bi=ISYb$HshsIhyyGyCPL@1O!1XMq~Q3M>OM1!HA6gtG3 zgmV`C#nNtbOWQ|EEv_0p?EIm*kX_-BsDlWa6u()ygd@A>t&_D4iOaCZvk|*raKm#S z|F)Fhp1f@RdYA87s#DkEa|_H2zdfH6c6nvLrN_K9<@QgKQHo1=vh3pzM+_xCoH9$w zlC}M=*i+CU+ztC5eK^Zg5f=`&P}b%^lvxLVROV|aC8 z{zpuZ*6oGJ65GRU$61i=S->(~zfT&NMq4mLSYpf4_s6{oOP5|a-tkd(##R!4x=HQ5 zzpHA3tzL9((~lfRB#?#&F8!D?m$12|w0dJG(aMHPraQeF4CjdH`ajewO3A}-3N6D* zIT)v>{1_l{E@*>S7!QAGtCJYxpdznLnoF?aQ-A1hdg>_~=Yl!R2}eiknIi6Y7Y zzr%S4bBca;_|#^2ZnQ$*c47Z0L?hl+P`dk_3CC~O4J&jQs@djWGYsegO+|5~TqDz` zU2tbBdwf`cPhTxntA4Gq;kd4mx5T`O&@CD8J~&HF;^3ZNkx|8Bo=hDZv)Wupz6r7JG34Ghp zg$!F-b>T0UMv3w7Tzxw`J&p^nFKG&x-@KbIBy&8npy1$J`FSbf?Jaui7#kBO*-G|z?z+13t(BW`#lCi1FDnc}z5z^1>@aJ}(S+ zt6D1zG{6X-F=SK7K~(b<+%q~X#QH!ay{$H}XO?HB`-jW)`j4p^K|b<9hj*srh49~N zF56yCzPAH;8MtWB3}rU@ZO3UPAt+G?PjCPK99nI_M_lUeC6)rc(Fh^6q2l9&y>|@xRxvhy=n+|1t`hF_r7=1{C!e zUn1_bm2O}jyoEZhUaeg8p^r1Fbnc`2VLL&?!?&76c;HPM48A4&!T$EQ6)g8y_@r zu7=JNtXJQbuqg5}{#t;?IQ9tI-DH;eLVjmdECnLSSKN!T#F6!tXiDDWEdpNHNdqeJ z=lls9lh0lPu( z!uuj|-T9u*zAmpm%jt57#A%7`J>(c1r>#2(;`i47kKVpK9Llh7drBpQ7W*~|scgx< z&9qq}Ns6qKBwI+xHfEHNbxNU#sgPuu?6Qr0&AyKqLq!?PSjI3*-_`LQ?{U1}`#$gU zyx%|1AC7~2T=(49bzj$S|DEU5+eQwYOa*ES7#QHh&Q_y^;J2ohKq0kAZ+|@dNZq5o zAC|Xjv|YZYhv`7tri*TVa(gw&w@iHQhk=9{QG1DpJ3E1InRl2G4#Jdm9IDrylKGKOzTD??23ZKyLwN zN_4t>6JC&>%m(%_TPW4Gy#9=bZ4r6q9_QLP-AKTD4@pbg@MGc#iqJj#I(YQ|Z4RW6pWy=O=yZ^ALp=#S)c-GA?NVfz+5BxlaFpslvKSoemsISq( zP0+(R)B2QqYa2yQ7e@L@FdJdfOAGC}j`*=`p)>qiQFO*+HAzi_LS}@h;j~B%=owPO zJabzw`~=RKzC_^!S;ijfF$M|Rf#!D#_G%Ri5nsj~@%bUoP1XIW^Zp((`Qq0^rfeK_ zb)BU^>%QS~O=4pc6Mz&nG38G+bLt>~YBlU6+=^8O_v}Bw6o{1!$Um+{80qSG{=tq^ ze8qTLS+g;wtmYwZBXLMQQ3cS|6i2>qOs%xOpCd#u+T9^w$@}ps=k(Q2yB`g6NZ70! zfMx$q52HP#^tV6{(Nl+839iIe;-PP^iq&zae6xLQwwq2=Im^9K= zs_QT9a&OO+zsdxT#$#a7Ho5DcWbmVg?Nk~Id=Fp*rwnf^*PVB+Uh}y}_>P-QhFs6q zjz3f91GzN) z4Gn;^D1uS|G=SJP_(3Fs{z!`uh(3(yM+Xp_rS8gma%@@d6NE56ed64WA6Q^>J^leU zF&44^6^%0embrU==f=XeoW7=1hS|&koN9_)DdxgTN1(~qkSyu#uPBETdgF6| zg~CI5coyAaHX$2u4s0XoJrPYsEw>&uK~*3LHqJa#Gik>7n2LK&-IQHF%~hln)8VxgBSiS73wkQw4mN5M4y3M8`g29FaS(V`eB2 z1Y{cAo&dNK#BdB5{ny95z>q^KQ};bksgrr{5RRPMkJuSs2YQ`T<3d#jV!w~y7Fi<- zjJi|y2QpW%RxD}#RVUn{vNxCA6Zjo{nxGtC5FBq?PWc(>vlX zqU9*S^XJH)wb?OYH^NIlykLm3yBmb*GWR|P>II|53oW3RHWi*p{is48&Tmy+VB5au zW?RE&BcLTjrgA?4wm?gyhT~q7h%w?)!}qt3j~$0rb5!l9JLlaCA78TwWsfkO=ZJj4 zoXzRjB&17_WOPclT~AnuzeOiSNYMG*2WkKEr)@eYv!-J_nwLvWZTLCyuXKZQk%Fp%TuE7dmZ4agnj=u_yVe{<* zj~tx)l2%!j&RTtyfc-Ko8Y}}{8@q`6&S}Pag7)t9;=D|U&tC!<7Q!^Xx5Dh5)4Ocp zf^ZiEA#XM)GHs5t9ST`;5*thps1UY`Drn3Kh)A)(a>|}uCNzwt@y;IkaYJJp{d|46 zm-^6$09Cgcz?Y&s%#CSotQ?T>nYuiUJ@EUP!`J7mUrK5}zO}-aa%;Pw^9Tb=1j0J` zr34^_NK<@?*+dL!c&j0XiUNCu?tCe$iw`t-g>9(?2r^ki#EO)sDS%5HcBji^c>k^o`414gx{3`hnsb1?k zws*Adz$&S>zT<`DTkCS|rASX|Uaf-7(uaNi-`w^Dg2KmPNj;RFQTb<)DwowfQ|Y`Y-OeB5IzD z(|&pT`g(au=B7U-Jv#sXE#%Zk4hY}w2uQJ&J^eXly_qdDtS>%i9%eDT>Fx0~G%&Gu zMAgBu=5eciGg<5!MD_`90@QR#{;~L!TV=JS?6t(g{oN@B4|MvBs~c$_d)4f!U$0+x zsE^zgm1jiF%V&b{|VupOUSU@)ir zgC8%pzls>>9bj?db5Xg^$FffyOcX{B(7(1HNwit!Ba*6G5hBQA^heputMss7!o#X} zj%{IX{?O~GLNj03YcE<9?i+I^DX^r$fm$W>G!ZLtc3M?)U=&h^Qe$94d%%js-5%4wqBZ5|q=30s zX&2JYeKXK3wu4D)iiVTcMVMF>q?4ch?}-z326*dGi^*H>`fZn3_reFko&3S9Mv-c) zh&sJMfDe1IvO$jiC4_(F-UHdnA)y=F${M_RtGA+)CJvG{%e)gF6Bjb6#&g(K0IzUR z!#55q6d0Bn)~frheU3(3uIoDP-`MYZW7qlj5l=sTiZ~S|(SQeXOyO9jev$#{*>eY$PafjiE!m z@=ydsMN$ezblRwl?ti|5pe^6)`eU%hd|0`+NKXU8-UORtALFW2kK(FuR zF9VwyH;c)>GQnf*MbM?Q$|%RZ zt!TNd*o{dIevZVgP<-?(uTp!Gj$3uP<+|D3y0SbKe)*qIQ2<~BM>BJucyr$$(ap&e z9B1>c46v0i<+BHgpA!=sQs!c{9DZjBVD1Qo#)tn*?$9<1i)3DXb3x_C8WHA3ej`4| z-xAefpbKAz@feVD0Q0IL5<}{X`$koXMjFrdqSl+UR6bV^Rha9#Rn?Z)#=cGGa?x|r zk-0A_YO&X3kBejhf5iP@1CZ<&}ssLi^jK1RJV{0}Wtz zvIr|qnoDyEi!*?b;(opOK$^qv;~7{>4+do@IeP+3)F$l?jSve}-24 zfso-04fZ#X)*<1Sg-H_??lRVzK~z9`#U+9+EU)H3x<#0-i@Wd@W(zVE(Mf_dSv*(ETJ=c&L|YPS>h5dBkM(X50?n(>F2d{>01>%vZeWLi5&*gQO1}lkqf?h?bozHtz2s_S% zj+2q*Z#~Uw2()I`boot({B1{mz66e=+a`N%<4@$q^;V;BvZEl{nv7qAH3!@eNzj>H zR%!hMkY3r1OZ|SYS)$@RhhLWV7lO%YJUotuO`(=*azW7tEA5uFm%@ zfjdBmZ% zY)=`lsg`6@!gRZS=MJS<_9>d1d$T%2sUSYv)BcU^6=eapRd>`cbW^u3nA)UXW6kR& z&iKR7ybR}n2_~|Jt+KRZ0>1hKnD_CYlw zI9L5Mvjm5WfDn*6#0Teng=^}k>$qGR2_#A|4-W*mOZ}$N3_SNsCk9=TsjCSEmE@Nc z&q9K56RUKY=+7Gc^y7KWwFCY?CY|ho<)Zn{e?K(BbOVclmI`7z&{O;*;SPkrG4uCG z)ygrI!<_ARM&-R!Z=3#sJXA{V6A4IGCwr}(63LC}==RMW+)6pUQY>f~A`TpR;-tmp zza8kVa2}?>BNyGmgFMrB-Knt^dJ+HE<&nA`0uTb+Q|Nzp_yP7*OjFk0Vf<|(#PQxp z-No3C!B1Q7JAXF%12L|Y4bbAtatUab7MW5)pM#eRcku5;^`iGOFHt+@1RAW3HPRUj-^CK#tniTjR{z(a1f!G1tp&Q zWhLfY#6sCz2N8=3D^3#&_$#lLmY3`7Lo_MzCOx%%#;jg5sPa{lTz%i-*3mu@;G8yiQ9VemHX^?+82fLi zc;=H>Ih7?0wK8ZRYPAZwK*6>6(zWgQ_1Ih+eHN<07;`j&^op+`qh`JW4c*CG9%OiV z*^1@0t%~-O1E*Nnt;Sa>znm`3ttLW68jjw~RO`r691HC6ZekWNmg1j$!v|J&lFz99 zLO4f|#9kJhO3vztjPIMjmL_{EBvarubb(1VLj;UsGR({^77=XTG1hxL$$>qxGY+a0 z3i?Z1=J;O!v_JU=@;3UF)XhfDM-9_?U6Ll26Wr`E?d1)23fqCW8>Q$)+|_V~Za=RT z)xQ%ItzqVEq`@iW_FVdILRQx?-UN7!32tMZVw?~FeVWBHghZ2U8-{b29$IhFk`tU~ zza*F)34M@r2=SVl#gkl9Ty73^em3?qm&|}JlmN+ZO1GI;ne1V6^UP=JtgZ(2MA6M|j*df9ZlyWHV{!p`3axGB2n?glZb-~+oCTj9N7jnagi%USY2 zBw>eljrMQOom?zx?(evy?JE!LuNP=Hi)9rq`Vrd4!M7}0hI*F7NTxCDBBNMw3PVaV zd~;;!6vmWoa_!QwXX+OV<7`Cqfopl%ShBjYQPPXpd~5Ni_WlCTa{WT1W5Jh=YfIQg zbz0r%cp0f_h|U7$9HvDZ_XE%723Fdh>w^^;N(kCp*0Un%VcgrpPF_wDjwR<#a|EPm ziXJ_dM$h6qwEnUsF&TwK%7JH#fZyYfLPM0;hx(mA&@oOWECAq9VV!E*_SWH;Id^I7 zPt5zT%iZ&#wYz)QZc>Ao0<~=JGAbwgV5X+;A*6B5T>we+aV?-g|BXVA1j;<|(}HO7 z*-X`}(eJL?Qu2G@UOFGo0Y5GxXl`BX28m1(FI)GmG(N{tSjJ7@0l*RC276{k1sLoI zS}5gPRqAOI00v@P%W6+?t;G?PO`q37P`?d^QhtfHp)LPFK3g!w`DgJw8ZoUFQ^PkA z2NA`S!YyL@7|De2=4q$mt`FL?IWW7(KM*BZQ+6Q?%G9HyDKDC6aQ+67#W5%7KtMIK zz+Mt2P*oMhc_-<#Kq%Jf5>DZ1-R0~{y1^JDGmqUHhV&c;R<3 z6W1r1SE${2t)+dfRbgAd?r)rPBE@^DUwSgWa98*G$p~~)32&YWZ(C)C~ z!8?kr)%`6F{aZyfbP+hMVD)Zrp!&&RWr{q3Rr%@qN> z91c|LFCHZ90b7fYo_0^VfIIFjZl*qL@-WA*J)*2uV(*PTFCzAaR*|_rK=od{Sx-2hwvR1aoOV&3M?gxtJwB&7y zbvNA?9?pe3bBN>@Y`19jDa?E6Zq5KGZ@5*{Esud1xzhmIzDIBO|5H4E;_W~BoBz*$ z;r9BUjQqdXF+SQK>&gRHBW z6Fp!9xOT3@$6Zoq1FxOMZN7Ba)fSBJ;E#ashkGDmY(S4!gKltWf9ni%#O5%;>G-V3 z0mbK^TiT8-d`~`89JBHq;ol+=^0a5!D6ih4%J;&8nvYaotW6S7>qn*NKr^sk+HCTG z4JZrbkS|IV$(szqqSN|J*RF zA_vTQ*{bTQhjBM^t`#T6cqyEOMMM-ThhJq?qINTk!;c&AvlAceDPaS zp#j$(8~X3>lyG><=dBg4w(v9t$RIVH{R8pBPhcFx^2opD&8DS#zaMt+xTZx=?};3E zeeVaa(46+nTU|_bGhV(QsT5@~a|^2SVv4=_mfk*`UD(^|8Z`(clZv>&xJL?+rb!Rs4%3^O+#od=FQXwVO9n5q}G$wq|tEyb2fFu zueL&z!~;iomPNnF6Y#mjM~Lzcp!ON&!L~>j3TL~RO}skmZSAjdE`?V6_q_*Uca+$z zafJp7Ki*|5gs6W}I6lAdiP7&Athi4jthi%aa)~W5UZ+slPa+E3{N_=UnEh&@vNrZz zx|T?HjW%CQ@j1vC%s_$VgLj8dEYKWT1t4GMOOS62g~yFr^(1-q1TwClLN6o^c~{3Q zNyK?=@33;Y7s6s|3O7V@=$@yo#H*jj6>wJR_>bv;OphIx$rs6v1 zQ}$jE0OnXIa2@@HybrIkpi2YdADtp}lRi>uwlk2&ZKOP`*IGwO=(<`fCS3iuo&L&I zWS=&UU;%2;PR8r?)NB6MW9eEyc29CimZ^wD2MabKuAXBmkg?HFiY0m~dzKyt--WcI z9yr~qCRsl^Pn`M)pnQ4ybq~T~5vs2!VMSVYM=F>YH{Ps|#;E{|6l{yAKM?O#`W2M) zO;j6^e`U)9B@#HIe|9iScEnMQx1{4nMQnU2i>TTtt&T*ijn>cM1(w*_RLytuD#FtZ z@A7|dotd6-`1)YoLN6`(eE7{bSX6ic4`}D#Ay+{@ZwSf_P*sCttZ^Do-Y3zsCu|;+ z8B6wW0CDr!_?tMejrp5nb>~f91LJsue`<-QGKBW!e{_kWXFr}x`F=8g%F}2? zJh1u!Ky{xi$>jVh%b4u1Kv|=kG|{tQf}gRC&Cny?`k)2h8CLP?D^-$Dvp-S6!rvuf1L(}hn7QKZ9;IG`5jLoCEoqn zK3be<=OUo=lyE*S&LW>1Z@OLF@PhTZK`q#y_LP-Z=SyvAwoVhW3yQ6<^<7?GX$~38 zsY~$_b>-cVwTTQyx1f$v?-mF}lDv2lO?Uh&6~au?JsfsUNh?}>7Ti%5tuWGOV3n-3 zu+`qQXDJrerM*Sv~%&++}?x8n6wfNWxAHL|Cd_k@?%J#;W z7e1tCk-5&OZaBphJJmw_$}m`M!a-(=W9}O>uT)%BXQ>;7mRe{dbc}O&WWsEAcA>uF zDaODau_5OuRj9-e`3eKgZ;yL8+Bo&)Ka_>84Z?C!$c#ml~ugt4}t(Fo%M( z*hI78!e?C?y(6&)RHc9St(^aa0lGK3ou0vcpJp2l#WT~kvBnpoT;-Ql4>K@7YH7e~ ziF8Emr9%k8Wwbu3B3oxlC`)M}EgE2mfvI*46 zxeaW2R|;v=NgGjjbFS_l>_Vm+KlUIYCW|Qh&;lCK*gi7azbs6f1&6$s-2zVNh)tFx zHRvg49MZ){lru*CSellZ^t|`T>w|>(n(jICxko9zO7a} z+6T5{Q8bpvLO-VA&@OGK!YjLzuIvr3ztnuN4&NmuAgop%M82Q7&q8nq%xTXIc-FJ* zVG#AQIA~ZD96DL2GQ0oyWY}ETBIbFE^L%Oi4=t;}z1N?P%w9TncrS_p7F7d1OE4F2 zG0$Tyj3P;Yq|L=rOkbiLPzmYusGu!v?XYN-|YI27Q zBp8ovz2nvd&4)j$`0MSET#0($}Xm>;SlaledXFj37h!~AWJWD(tz6^nFV*o z#5VJ)&3|}N#MBs(Z48l%XV_S94!V^g%AxIphG^`M7E?S9>b*xa$DuYVv8{EwlP^5g=>C41m4>_#1RmpbWo_2*3Uj;Qig==Lw~YI9Lq z@Cd^O|GUO05%+7&!|q3+R%Kc7c=R1x8NZdpXU1dr4lzQ=vb=qUjp`-web$!?cI!nu zrEFoF0L~|zDWEbd63rhC7e<0IjD33^j*lB(tFUmb#2;zq95h58*J_S?4&s)d!uLTF zW;cpBScg#x;D84umSHK-3STKO4f;d^_)T{!>`#~zp^%6Uw>Z@ylZ2{sFGMP_VX7a-W82n!5&8*F#Yl}1moWN24;BfiLA2}}=k2#ET(^b4IR&2gi<`+ zmN5^EnR3vBOmTPkuMr}s5?XxtY0!2{b(?jAXMe)!$K9d_lj8m7d=GjrDyk>H-U%w> z38t2nALj94Vd*>1G9I81mbD3PY^8m}cj8=AtjMU3Ot9>a1puTMOi(>Y7v0=re)l)5 z%*#eAGu^Oih~>2^Y^0GonlRwya+a@>(ND4j2boHD&mv?@N~%f|=2`IphK)Nm zR2ixko3D^nxY?G>|Afe|uS)79MulzFj{LOPGQH#>6+Q`^=(~7z7=Uf|!_R(1hzi-8 z+stLsVm#zc)c3y~s^m9JA5;Iryxfjs8Ta{NS8u?!{Iyy|xDhRwZ|GcH5-1ziq1(TM z$2Q048WOQvBUo zt6%Y74rty%*XsS{CeU6LW1gjMwHCR8#DKWNi2k{cU0W*^3a^zv=m46x+Y9T(#?>-a zz&y1FM`G~|2p~|XTU0@bdO?gPUd;TLsV2MtO|W6}y{EJ0MzI$G6Ftudy6$Mzso%O^ z6Z2>0%R0FCd(ZjO9IiBMGS}FH=*pO7D~7!nbP8JTo3x1*NC~G-ee1LC6EYx`#k-a} zu^lcURIlyk2Pi0h1pL_^ZI@k)(-ZWox~tQN`%poa6iEpx9k)A8Wy~cem_5lufR{Nvbh)xC3dqD zFNi6d(E=!~Vt3Y!@Un#6c)zN3CTd>H(>uf|NK5Tc6I4Q-#8Odes>ll0 z6`DOXJnj1{r0$k=eP>+O4ckoNM(VC4xl<1_IDVVWYV0Hoa{4a-*v&_=QTs1!txeMZ59dpXVoh0d)h;KAPe z?u=LhkWpGzL+aDFn;lhbDwBQ_0#C|Xod}+`xZ!#)pK~C=hUf~DcR8u-aNY{x5Azak zzAnFdZOfLeH37qZzvh=&zr@nTyCVSY67z@!EoprV6{2ZG6gK-Xd_;nFml`f8;WS`_4Ilkx~kKzFtXqXS9pWTSY8x(Ve{t!!E}??^prxze}S|3=5P zY-*Rv7qRnCu$1v{n6%Gl`<-_zBNx%9STs5?T=q?Z6){7d=;G~2+^9d&s9E$FVLNio zwi2LQ3uP%kb!MLU2}jzHgBlTZ4TagH$G1oD9?)3bs@UTqcgqHrNyE~6&|InLiY`#< zR`VHq*`{ZnzNPSvdj02yL7ap^Tv6RUG=DHW1?WVE8oEgt?rTB#wY<`B0I4;4d}yJ` z(caR6%u>(WO8vdqp1gPCNiA@kkE8mC6jS^JC(YI?Y!#@N1M2R93bzZ>Q{NpL!KIAT zv%l1-T{`p!QtWD6fqL{r8nSOM^0>!b&mYJuT?Bd?df~}z|29LVnyrU+wV-QQT&~cZ zImRkOB7I&tXI!L|Zu>pfY56UDsV;x9N3s-pNnyv8^&0N%JE=v}8Ek*f&b$e(~Ey?Oz8>JWV{w0v^$^uS#YyLaJ#{~b%N|m{p~U3x&^g~?7mMAWPF^6@0s77 z)7XB}9k*9z5LE4CM*OG44Ef*w{%7Q0H*fs2d!1Fitk7?o9H%IB(8@Hj7Z&I6)=01< z@`hVk+}ysZOu@lu(VM$-LJuXlPzpvYooxeQhV(=R2fG!NnIgMTa>(P9q!N96XvjvG z8k}_csR?`^?y0^Xu(fsr1!_enWj&<3vR>$8*M`kUH59_}CQ)6$Mmai8?iKt+T7TmT zCEZBge)FPd(QNK9JOe8LwpcNma%I{t-Ta6x^+STiw0-#bzKhVPwuGXc@h4tbiYF&| z`hf*g1}&&><_V8kwjK)t0gVPBF*eQ}cMU6g*Kg3;O}11*90)+TFM4*Ue~+GvIh`2J zy>scEp*%Y>H033HcF3Eb-k{fKoX3;_C$mSF-L_ba*<9851Hs}SgMw|9P2auDm^I!! zbkheYzsw)Vv#R!bBj26*_ZNiMYm@7Z&jA5xiFeg|04JTGr_J@?5v93_mFU&)YU7)h zhGz3`M!{_t=;-H`B(fNRa~(&5(=`E{Sp{j#dd!vTgU8gVY?@rv8Ye%jB<6Iuf(ozn=rddz?mrcR%pddo@{7RbEVIx@FyYPa|=D2lFKOva?SlUeAD_3pzlKVQ#EqoVf;~WYo%>;LHdLf!537BjXdm?DXZ=Qd=*I#9n-{vnsii@B@63%! z*3$bM(NPwLf+>mOqE%xrZsniI?T-E1EA;U{U`hW%lm7qyaQ-W>y??e7|J|ScD^?4fgU1@d5z$8lKfP2LJ}Wg#pwq z=wRYB*l{&;CUmTk@AijZEsOMyrJw1;_j3;1l|CxG z8ALrBC8kU8__gur-P1Kq@9%5BeyL`ZmX=<*BWssV@;0e&zRYYpG1f9W)@gQOtgElM zG-2#|2G*;-j5usDR$PGx*y6}cosyADrZ-JULAAyRXZ}7;I{$942M>V6)f~l{>k87% z9;Ztj5MppK|JR4mg|9~T*8a3g_o=?suH62*Dgo$%OMH6Z(rd5Xc(A8tMCA|78&qg7 z)7TnEGGbJg?cyBb+%^%cMJG2s)!dK9{9ManZyeXxvaHYz7?a~`3&IrmA}34QN25h? z6GgG8`&7mEnIvoTzFRNmYufAr8v|1jtBpU~1TQ@mXFuD!L48e{#2M%PVAc(EzMId$ zm_SRL{^C}v1b-1zpwoC)FZkjLGM(4Di2Fq&Yl!Buv3R5YpC zPYw^2?=O;=f1)Niai?Wtj#aU{Q8g#bTunJ7TC7s6i_Z7bUA^)Gb-SQi;x39MDp}J^ z;@1P*Bra2Y3SLx~aUOKs*>kZe?#wBDy1S8;iz?c?(sQ{mK&|MYusIy2dJPf=P3 zt%dr5wjNtXBu&aZU;E&5SW5|89KH{BPj&U)&;?s}`q)@|@%6lcE% zT#Hy@FJ3856fS*tctSQak4dZ%UUIdOoK?Ypirm4AZ>Z43nkIHf3nSSFodob6Eg zwo7iSEjKr05|pe6UV~>LtwpV<%7&wyE;^ZjRd?rnw!590QvT58`A3Jcqup`a`M(Gn zS)9Vg!phAgAnne7T^zW|iGxqf=kc$^;BpJcs$)T7c0Bat2(+nCzH0ML&nZ`-e+_;a z{!=&^%Ug+QUfK{8bpakMlvew#g@=Zv*ysYbplZx{@1RG!>);thcK$SlKIx+~eUI(F z#}WQ{;b1B;&tBc`&V7UYT>(mRd81V3${z4nF#?-fGjn_2Ju~?$o#dDMh*>`Q^8AH! z{9IR=A4_6Mi0SNSBurl`Axs9LHd-Iyewv)f8y6N~-CieSB6;|#Figer9Wt5D8T=2! zYS!xFAzxckdeJ~<8?Iv46QL#Mk9X79d3$sBHCHjA(@Ud$DuoURV10+)I!F$2_uaZf zQ0n$W`t&|&d8njRMZ{2s*0@adQM`vdlZhsWB-gBd_NK}BLy52c!1VzCm(gERoGleNwHy;lGqOf)A zbVRn;ZPQDbXBl7FY&gkFGLWIK7*Nl2g_@*J2>Z_Z&7csiVTEOZ;LL=^Qm74(d0i~M)n^@&m?KDN^x1q~5G~wL zK3Vu=CC24L6I>Vc&>;AD+cPCTTg5wOhORjLLsPgBGV$3QLlb<>;!fuj<(`@0)QK;; z;pTm%D4i)3=%f|`{Jn5lZ)TCFB+kV{;!S@=Aw`sWD+BKk;w*8pT4 zt)TEnpUP51yr9L@7#$55+x*p}nG#FG8rJtgRi3#ug=J>cIEhpFS&>}tgrJ1Gluke9 z0Vp9)BSu_-nEARn`^IQm7hUmi&?QEdsb8KTF-h-_QQg7`5MI%lqEq7CVK9X9d^qXIW@O05oA24>L4BkR2=RuN&X}8@%W>@jB#62 zO|N9HjKNz73jW3mi&#@Lqh-w)7^6923@Qo%b1VQ^aV_|j5Nqmy(JLlN%r0tF;)&KcZV8Z7j=J9K7d&rGDk#zYC2 zN8f&I^Mn0(sIvonRRmU^_Sekm!c|W5;6>mQDR#eOUS=@WiKyH&Q|Fw~=%aRASQpHCFn{2CHtTyw8>uxgDsV}_4_Y_>8 z9~33%6#``GCFpkE;1Z1x-+L$ML>!RIb59!IR>?4v!YlWnz;u$uTWHl3T8b>h;`a z5X&aPuOOVq`5+F}$7PtQjHyGO7ftacKCRf%-UzghzJGK?@?kw=;-G!!K$+FofZsdv zgr<0DivvvDepnjnN}u%(W1*{l9M-aeXU&C{fIaV_%2VX$ z?^sDuFj=yW0x%U6<(mdLV6;uiR*$dzVL8(=k@}SmA-#p3X?`wUtvFgBl~}H5@`1lN z#?C3-&G*@udL?bf+$QdcWnq-0bB4^r63GAryqVV6Sb&hKolO8 zBW3KVs-`^=?>>rQ!C=VXtLIj+t(Kf6MRv6khGPEmL$z19ILO*heeoE8G&I;HP+4ye z7N>UQZK>Ma7xPj+F**#n1G<-b0LMOV{kl4J=~cUbAZIPMA#L*n!TNw~O?ePGss}bvwN5VPU8HNetJtX$PsB83`WfC# zbMQZi1XaX$?NI*S$e_lAQz;i7gEW!auoYsBQ`kl}92gx)l80KO zO2;yzUJp1QRZTL5DeM^&t;=i-+X^2=G+CrAd`3A*QM`G-r8*#|!ose=!DT<8BXGHh z^>;YuL^E2N0R$5VgviVr#Frx&t(A_*r6!Az>shYEDht%Y_&Qa-5za5vprF*hT{3F% z7*HG^J+|~2m>4jw{#gSfXv?71;5sIme(|+NIA+R}Mf~Q6FrInf0Fv zZ0g)iEqZ%rA=ilcneJ7N1{7yVI+C38?C})&ATF<6O&cY@&QyVHoNIMc3Ex27iF7d$TIH|3ZiGBNDX)4LI0U>adMMf^#w>uq;^e;mmW8@OYS z@0VH}>AM!0vk+i%Ks-H6kK={ei4N-$!C`K3pf^x3#p5wx=jB3jb+af6Y9LuW_!RoD zI-Wj+W4cRGd}nrg#tD(JVo(C!p?VEtMmOXFhYY}^5n!h~xM?FJzgyzpyx%j{!ljsv~Cc$`K#s)R^KM$&4;c-S<{6AbZb}Rn8bTb;{l6YU^^zy^{ zv9z0u^e$Y^WnoyB)#O-RmF3XWgcn}4*2hMG|AuKi1bobJyE`Y_duGV|!dlyfG9|c= z7EQp6aBAWNMhn^A5mpb~lahPk8XD5s=a=m}o)9091xQ@Py>o4qSBoFKD<=X;u0J;& zR4){IUpA>pBOCGRSE8xGi}dftXI^Hq3){|aU2m^j4in{_l89&=Q-;x|J+w36-lwq& zeLAZ01i`*x8uV?bao^1%1D(osOr6 zzjTy-xXGP`d27Xv9hG=(NRnb)yi5M)ou#eP*n)}C7B6j<$ncN zNaJ=Ys@=;Ui-!{9A)?#Rpjh3p5%ANpHPjVik{(oS{>ScuGEbe1S&wv8EKncB!&$M0 zCPvkEVRxp6`q!;hV-;W=^bWCmp`@Amh-tDAQduTeqrnKuhTM73>oiY_R~=j~gFrS= z)1&6!;@1j6*i}I69|6^@>^0P&vN4J-+KET#4jOc@&i-vldaMqV3 zAeWJErw4kpg<#IP8hXlbt^4WM55QEo-687u@>Tv#lPf)!(JTYcavf^0BL=y4@yjT7 zxz5!kt`$+$3rmdvwy-l0P4TDnv)>AXR|jt4(weCXHQ~f(y(Nc7DH@uJ_%@~;ST0J3kI* z(<`F+81yL5(EgW6oe+>rDtzSiapxoLn-9ocHV7a1Gmas8gFPNy(e^MwdPQ+P2K>x>uZZ%F%nH_swq`mpLrcMRc@&t@Xv6ctX-s4rfDXBo6zQk%b&Z(MMX?O|>6 zvv8RRWuijXaGYV`WrTAI_$wvm31{A0zMW|id|9?TQ%QdkER^d)7pIdD(-UN6MkFCR zY}RZhf&Fo`d%teI)0*h>k7wFj%Pja5B!kkKt;Oe#Lit;EDQv{7-yGln2M-dtv0K7) zn}|JvFlS`AGI8_;H8!9H1zh3n8gL($1uU7S^Q*!p;9b$G`zv$xr3f^z4v7zg_K_uc z<3-Nib8%<462J>TvvEn7bIC~}<4bWSP1o=eZha@$P&EuEiJtDsO=*jB8Re6mPCtLV z?{2~!ZO{EBXgmc(?mdf_7dEXaTYRF=&!3(m?4~EXL{~J3KG8+5C`$mnUCNws7wPDu z7p>cJIfmcNHYGwzE3;LhDUX!`OfKUdZR|WG333Kdd?);GI2P%NaK@))LaxUj3i(ED zCw5E_iiLrf{E!VpTvyF`8RR6~Qaz=4>Tl3xOutCYG5h&L;(av*wO60FI=oC1n4Gy* z`j8x6T~(<+*ptMpY5r-f#oY^`NTa6AT^6f}%gcpkbDKCutsW4A^JVd>o`7YDmq+mRF#hKwVDr2)_UEwoyAh`EH@Cu{+&UNwFU6(` zyq&PPaOg95N|xlrU)X^(SJSP!pPxBoO%IM=BCqJL#xc81M4n zw_|1$pGVkFQ}sg)j+4CXo{Oevwan94(f)07wF5z;L^Ng1ZPo8}CJ)7sF)-|f(F1Jn zq3teWp@O+~klYAtp#z%CLnFCdZ<`K;;ZpIQZ`$LhF`Du+{y= zrJV6~X%8xo7VPn8y`Y6=au#80EZ(-CNVhvD)UXmW^gL>4Up1IABc41k~xYsYon!5zGFpe z=w904#2@|ae2AK$>so7`m3X+^T|;!sx|inufCrX*cpEh_;quH(A0J{yp_5YC1&wj0Q%zTH zl(=e4DVofC6@OU~d}Mbn{oP*3(x*M>dSt_H_;bD&Z-%eS^Jq7AO$6$>1}pi~Az!<*X+UF|_3b+#<@X0b(bO_EgBf3w|CZslLbA34w_da4mHBakbH5XglTVnk!-6 z#~Fves(rWJX9nG21QKJhCRY!@oNHZ)jKcqS2L|YUZ{oPSS~M`=Uy`?cw(lwb!g7p3 z^5L4a|2*7_q-W2lo{2Vj+(=dN_*`QhwWeygM`X!jYW0qfn{{et#4V3hT2PSCh;X1mz?0H z1v>5Z^~!CYLZ_hd7#`H1g(9%w53anrjHVpXJ*l@dW5t84pMOte?>(~`$}fhCf8M5h zB)hA4{L^9+ja|F^&AmNExZyVX&-n-J0$$X7qQEBudl)tfdcgPGJhMA@dGQ;?gO8LR zwQ?OL*H<6ddJlOCkSjg&(z->8&sXLw1}11@qI#hNf}!Q0dE5y&kio$OO>n$R$@kpg zKw>Be74~4Q0GLm122?rCY z3l%@D{c=)y!_V8(QldHxj6i%SH=&Ze!x6zH8KKRETCMp*P z5<3R$VQF{FW*C8=Uyc@!wQLh#+ES$XJ-R~m4|*}~vC${zFU|M)pA%Bc#45duFZM_J zNXa~k5xF?i5lJ&HEVf8=lrKd@SiOR3W@R0mH>?lbk3OyRk4M7fRvh!+67^bNlwqc1u7=%(6c? z68;^Um0cyT*30nxQ<|p1v)ag0hB62+LiCZ8_lU!!f;4kWa`LILN4xP8dr z>FR4a_mbIpis+3NUS-KZnFv|6^b~Wu%O+RPU2ASX2m|k*0nyqdpmh%q=7{&d?*hKZ z<#U;DjGGesZqlCe-WwlYBBT*nwPjS>T8rpcTM#Y77(O_bTI)1h6gDibwVtPLdbNmS z=;@DCv82F1ogS2C5hYYd&09C~^LB_?6Y-b($n3_Q>?TGS^j{`0)FbPb=r~9HA0?Y3 AX8-^I literal 0 HcmV?d00001 diff --git a/versioned_docs/version-0.7.0/intro.md b/versioned_docs/version-0.7.0/intro.md new file mode 100644 index 0000000..52cae09 --- /dev/null +++ b/versioned_docs/version-0.7.0/intro.md @@ -0,0 +1,38 @@ +--- +sidebar_position: 0 +--- + +# Serverless LLM + + +ServerlessLLM + +ServerlessLLM is a **fast** and **easy-to-use** serving system designed for **affordable** multi-LLM serving, also known as LLM-as-a-Service. ServerlessLLM is ideal for environments with multiple LLMs that need to be served on limited GPU resources, as it enables efficient dynamic loading of LLMs onto GPUs. By elastically scaling model instances and multiplexing GPUs, ServerlessLLM can significantly reduce costs compared to traditional GPU-dedicated serving systems while still providing low-latency (Time-to-First-Token, TTFT) LLM completions. + +ServerlessLLM now supports NVIDIA and AMD GPUs, including following hardware: +* NVIDIA GPUs: Compute Capability 7.0+ (e.g, V100, A100, RTX A6000, GeForce RTX 3060) +* AMD GPUs: ROCm 6.2.0+ (tested on MI100s and MI200s) + +## Documentation + +### Getting Started + +- [Quickstart](./getting_started.md) +- [Single Machine Deployment (From Scratch)](./deployment/single_machine.md) +- [Multi-machine Deployment](./deployment/multi_machine.md) +- [SLURM Cluster Deployment](./deployment/slurm_cluster.md) + +### Advanced Features + +- [Storage-Aware Scheduler](./features/storage_aware_scheduling.md) +- [Live Migration](./features/live_migration.md) +- [PEFT LoRA Serving](./features/peft_lora_serving.md) + +### ServerlessLLM Store + +- [Quickstart](./store/quickstart.md) +- [ROCm Quickstart](./store/rocm_quickstart.md) + +### ServerlessLLM CLI + +- [ServerlessLLM CLI API](./api/cli.md) diff --git a/versioned_docs/version-0.7.0/models/_category_.json b/versioned_docs/version-0.7.0/models/_category_.json new file mode 100644 index 0000000..67c0cfe --- /dev/null +++ b/versioned_docs/version-0.7.0/models/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Models", + "position": 7 +} diff --git a/versioned_docs/version-0.7.0/models/supported_models.md b/versioned_docs/version-0.7.0/models/supported_models.md new file mode 100644 index 0000000..6077615 --- /dev/null +++ b/versioned_docs/version-0.7.0/models/supported_models.md @@ -0,0 +1,13 @@ +# Supported Models + +ServerlessLLM supports a plethora of language models from [Huggingface (HF) Transformers](https://huggingface.co/models). This page lists the models and model architectures currently supported by ServerlessLLM. + +To test a model, simply add it to the `supported_models.json` inside `/ServerlessLLM/tests/inference_tests` and the Github Actions will automatically test whether not it is supported. + +## Text-only Language Models + +Architecture |Models |Example HF Models |vLLM |Transformers +------------------|--------------|--------------------|-----|------------- +`OPTForCausalLM` |OPT, OPT-IML |`facebook/opt-6.7b` |✅ |✅ + + diff --git a/versioned_docs/version-0.7.0/store/_category_.json b/versioned_docs/version-0.7.0/store/_category_.json new file mode 100644 index 0000000..78b547f --- /dev/null +++ b/versioned_docs/version-0.7.0/store/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "ServerlessLLM Store", + "position": 5 +} diff --git a/versioned_docs/version-0.7.0/store/quickstart.md b/versioned_docs/version-0.7.0/store/quickstart.md new file mode 100644 index 0000000..02330ce --- /dev/null +++ b/versioned_docs/version-0.7.0/store/quickstart.md @@ -0,0 +1,294 @@ +--- +sidebar_position: 0 +--- + +# Quickstart Guide + +ServerlessLLM Store (`sllm-store`) is a Python library that supports fast model checkpoint loading from multi-tier storage (i.e., DRAM, SSD, HDD) into GPUs. + +ServerlessLLM Store provides a model manager and two key functions: +- `save_model`: Convert a HuggingFace model into a loading-optimized format and save it to a local path. +- `load_model`: Load a model into given GPUs. + + +## Requirements +- OS: Ubuntu 20.04 +- Python: 3.10 +- GPU: compute capability 7.0 or higher + +## Installations + +### Create a virtual environment +```bash +conda create -n sllm-store python=3.10 -y +conda activate sllm-store +``` + +### Install with pip +```bash +pip install serverless-llm-store +``` + +### Install from source +1. Clone the repository and enter the `store` directory + +``` bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/sllm_store +``` + +2. Install the package from source + +```bash +rm -rf build +pip install . +``` + +## Usage Examples +:::tip +We highly recommend using a fast storage device (e.g., NVMe SSD) to store the model files for the best experience. +For example, create a directory `models` on the NVMe SSD and link it to the local path. +```bash +mkdir -p /mnt/nvme/models # Replace '/mnt/nvme' with your NVMe SSD path. +ln -s /mnt/nvme/models ./models +``` +::: + +1. Convert a model to ServerlessLLM format and save it to a local path: +```python +from sllm_store.transformers import save_model + +# Load a model from HuggingFace model hub. +import torch +from transformers import AutoModelForCausalLM +model = AutoModelForCausalLM.from_pretrained('facebook/opt-1.3b', torch_dtype=torch.float16) + +# Replace './models' with your local path. +save_model(model, './models/facebook/opt-1.3b') +``` + +2. Launch the checkpoint store server in a separate process: +```bash +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + + + +3. Load model in your project and make inference: +```python +import time +import torch +from sllm_store.transformers import load_model + +# warm up the GPU +num_gpus = torch.cuda.device_count() +for i in range(num_gpus): + torch.ones(1).to(f"cuda:{i}") + torch.cuda.synchronize() + +start = time.time() +model = load_model("facebook/opt-1.3b", device_map="auto", torch_dtype=torch.float16, storage_path="./models/", fully_parallel=True) +# Please note the loading time depends on the model size and the hardware bandwidth. +print(f"Model loading time: {time.time() - start:.2f}s") + +from transformers import AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained('facebook/opt-1.3b') +inputs = tokenizer('Hello, my dog is cute', return_tensors='pt').to("cuda") +outputs = model.generate(**inputs) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +4. Clean up by "Ctrl+C" the server process. + +## Usage with vLLM + +ServerlessLLM integrates with vLLM to provide fast model loading capabilities. Follow these steps to set up and use ServerlessLLM with vLLM. + +### Prerequisites + +Before using ServerlessLLM with vLLM, you need to apply a compatibility patch to your vLLM installation. This patch has been tested with vLLM version `0.6.6`. + +### Apply the vLLM Patch + +1. **Check patch status** (optional): + ```bash + ./sllm_store/vllm_patch/check_patch.sh + ``` + +2. **Apply the patch**: + ```bash + ./sllm_store/vllm_patch/patch.sh + ``` + +3. **Remove the patch** (if needed): + ```bash + ./sllm_store/vllm_patch/remove_patch.sh + ``` + +:::note +The patch file is located at `sllm_store/vllm_patch/sllm_load.patch` in the ServerlessLLM repository. +::: + + +Our api aims to be compatible with the `sharded_state` load format in vLLM. Thus, due to the model modifications about the model architecture done by vLLM, the model format for vLLM is **not** the same as we used in transformers. Thus, the `ServerlessLLM format` mentioned in the subsequent sections means the format integrated with vLLM, which is different from the `ServerlessLLM format` used in the previous sections. + +Thus, for fist-time users, you have to load the model from other backends and then converted it to the ServerlessLLM format. + +1. Download the model from HuggingFace and save it in the ServerlessLLM format: +``` bash +python3 examples/sllm_store/save_vllm_model.py --model-name facebook/opt-1.3b --storage-path $PWD/models --tensor-parallel-size 1 + +``` + +You can also transfer the model from the local path compared to download it from network by passing the `--local-model-path` argument. + +After downloading the model, you can launch the checkpoint store server and load the model in vLLM through `sllm` load format. + +2. Launch the checkpoint store server in a separate process: +```bash +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + +3. Load the model in vLLM: +```python +from vllm import LLM, SamplingParams + +import os + +storage_path = os.getenv("STORAGE_PATH", "./models") +model_name = "facebook/opt-1.3b" +model_path = os.path.join(storage_path, model_name) + +llm = LLM( + model=model_path, + load_format="serverless_llm", + dtype="float16" +) + +prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", +] + +sampling_params = SamplingParams(temperature=0.8, top_p=0.95) +outputs = llm.generate(prompts, sampling_params) + +# Print the outputs. +for output in outputs: + prompt = output.prompt + generated_text = output.outputs[0].text + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") +``` + +## Quantization + +ServerlessLLM currently supports model quantization using `bitsandbytes` through the Hugging Face Transformers' `BitsAndBytesConfig`. + +Available precisions include: +- `int8` +- `fp4` +- `nf4` + +For further information, consult the [HuggingFace Documentation for BitsAndBytes](https://huggingface.co/docs/transformers/main/en/quantization/bitsandbytes) + +> Note: Quantization is currently experimental, especially on multi-GPU machines. You may encounter issues when using this feature in multi-GPU environments. + +### Usage +To use quantization, create a `BitsAndBytesConfig` object with your desired settings: + +```python +from transformers import BitsAndBytesConfig +import torch + +# For 8-bit quantization +quantization_config = BitsAndBytesConfig( + load_in_8bit=True +) + +# For 4-bit quantization (NF4) +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4" +) + +# For 4-bit quantization (FP4) +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="fp4" +) + +# Then load your model with the config +model = load_model( + "facebook/opt-1.3b", + device_map="auto", + torch_dtype=torch.float16, + storage_path="./models/", + fully_parallel=True, + quantization_config=quantization_config, +) +``` + + +# Fine-tuning +ServerlessLLM currently supports LoRA fine-tuning using peft through the Hugging Face Transformers PEFT. + +ServerlessLLM Store provides a model manager and two key functions: +- save_lora: Convert an LoRA adapter into a loading-optimized format and save it to a local path. +- load_lora: Load an adapter into loaded model. + +> Note: Fine-tuning is currently experimental, especially on multi-GPU machines. You may encounter issues when using this feature in multi-GPU environments. + +## Usage Examples + +1. Convert an adapter to ServerlessLLM format and save it to a local path: +``` +from sllm_store.transformers import save_lora + +# TODO: Load an adapter from HuggingFace model hub. + + +# Replace './models' with your local path. +save_lora(adapter, './models/facebook/opt-1.3b') +``` + +2. Launch the checkpoint store server in a separate process: +``` +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + +3. Load the adapter on your model and make inference: +``` +import time +import torch +from sllm_store.transformers import load_model, load_lora + +model = load_model("facebook/opt-1.3b", device_map="auto", torch_dtype=torch.float16, storage_path="./models/", fully_parallel=True) + +model = load_lora("facebook/opt-1.3b", adapter_name="demo_lora", adapter_path="ft_facebook/opt-1.3b_adapter1", device_map="auto", torch_dtype=torch.float16, storage_path="./models/") + +# Please note the loading time depends on the base model size and the hardware bandwidth. +print(f"Model loading time: {time.time() - start:.2f}s") + +from transformers import AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained('facebook/opt-1.3b') +inputs = tokenizer('Hello, my dog is cute', return_tensors='pt').to("cuda") +outputs = model.generate(**inputs) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +4. Clean up by `Ctrl+C` the server process. diff --git a/versioned_docs/version-0.7.0/store/rocm_quickstart.md b/versioned_docs/version-0.7.0/store/rocm_quickstart.md new file mode 100644 index 0000000..49eee56 --- /dev/null +++ b/versioned_docs/version-0.7.0/store/rocm_quickstart.md @@ -0,0 +1,174 @@ +--- +sidebar_position: 1 +--- + +# ROCm Quick Start + +ServerlessLLM Store (`sllm-store`) currently supports ROCm platform. However, there are no pre-built wheels for ROCm. + +Due to an internal bug in ROCm, serverless-llm-store may face a GPU memory leak in ROCm before version 6.2.0, as noted in [issue](https://github.com/ROCm/HIP/issues/3580). + +1. Clone the repository and enter the `store` directory: + +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/sllm_store +``` +After that, you may either use the Docker image or build the `sllm-store` wheel from source and install it in your environment. + +## Use the Docker image + +We provide a Docker file with ROCm support. Currently, it's built on base image `rocm/pytorch:rocm6.2_ubuntu22.04_py3.10_pytorch_release_2.3.0` + +2. Build the Docker image: + +``` bash +docker build -t sllm_store_rocm -f Dockerfile.rocm . +``` + +3. Start the Docker container: + +:::tip +If you want to run inference outside the Docker container, you need to pass the port to the host machine. For example, `-p 8073:8073`. You can also get the wheel from the Docker container after starting it via `docker cp sllm_store_server:/app/dist .`. +::: + +``` bash +docker run --name sllm_store_server --rm -it \ + --device /dev/kfd --device /dev/dri \ + --security-opt seccomp=unconfined \ + -v $(pwd)/models:/models \ + sllm_store_rocm +``` + +Expected output: + +``` bash +INFO 02-13 04:52:36 cli.py:76] Starting gRPC server +INFO 02-13 04:52:36 server.py:40] StorageServicer: storage_path=/models, mem_pool_size=4294967296, num_thread=4, chunk_size=33554432, registration_required=False +WARNING: Logging before InitGoogleLogging() is written to STDERR +I20250213 04:52:36.284631 1 checkpoint_store_hip.cpp:42] Number of GPUs: 1 +I20250213 04:52:36.284652 1 checkpoint_store_hip.cpp:44] I/O threads: 4, chunk size: 32MB +I20250213 04:52:36.284659 1 checkpoint_store_hip.cpp:46] Storage path: "/models" +I20250213 04:52:36.284674 1 checkpoint_store_hip.cpp:72] GPU 0 UUID: 61363865-3865-3038-3831-366132376261 +I20250213 04:52:36.425267 1 pinned_memory_pool_hip.cpp:30] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes +I20250213 04:52:37.333868 1 checkpoint_store_hip.cpp:84] Memory pool created with 4GB +INFO 02-13 04:52:37 server.py:231] Starting gRPC server on 0.0.0.0:8073 + +``` + +After starting the Docker container, you can enter the container and run the following command to test the installation. + +``` bash +docker exec -it sllm_store_server /bin/bash +``` + +Try to save and load a transformer model: + +``` bash +python3 examples/save_transformers_model.py --model-name "facebook/opt-1.3b" --storage-path "/models" +python3 examples/load_transformers_model.py --model-name "facebook/opt-1.3b" --storage-path "/models" +``` +Expected output: + +``` bash +DEBUG 02-13 04:58:09 transformers.py:178] load_dict_non_blocking takes 0.005706787109375 seconds +DEBUG 02-13 04:58:09 transformers.py:189] load config takes 0.0013949871063232422 seconds +DEBUG 02-13 04:58:09 torch.py:137] allocate_cuda_memory takes 0.001325368881225586 seconds +DEBUG 02-13 04:58:09 client.py:72] load_into_gpu: facebook/opt-1.3b, d34e8994-37da-4357-a86c-2205175e3b3f +INFO 02-13 04:58:09 client.py:113] Model loaded: facebook/opt-1.3b, d34e8994-37da-4357-a86c-2205175e3b3f +INFO 02-13 04:58:09 torch.py:160] restore state_dict takes 0.0004620552062988281 seconds +DEBUG 02-13 04:58:09 transformers.py:199] load model takes 0.06779956817626953 seconds +INFO 02-13 04:58:09 client.py:117] confirm_model_loaded: facebook/opt-1.3b, d34e8994-37da-4357-a86c-2205175e3b3f +INFO 02-13 04:58:14 client.py:125] Model loaded +Model loading time: 5.14s +tokenizer_config.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 685/685 [00:00<00:00, 8.26MB/s] +vocab.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 899k/899k [00:00<00:00, 4.05MB/s] +merges.txt: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 456k/456k [00:00<00:00, 3.07MB/s] +special_tokens_map.json: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 441/441 [00:00<00:00, 4.59MB/s] +/opt/conda/envs/py_3.10/lib/python3.10/site-packages/transformers/generation/utils.py:1249: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting ` +max_new_tokens` to control the maximum length of the generation. + warnings.warn( +Hello, my dog is cute and I want to give him a good home. I have a + +``` + +Try to save and load a model in vLLM: + +``` bash +python3 examples/save_vllm_model.py --model-name "facebook/opt-125m" --storage-path "/models" +python3 examples/load_vllm_model.py --model-name "facebook/opt-125m" --storage-path "/models" +``` +Expected output: + +``` bash +WARNING 03-13 09:37:29 rocm.py:31] `fork` method is not supported by ROCm. VLLM_WORKER_MULTIPROC_METHOD is overridden to `spawn` instead. +INFO 03-13 09:37:35 config.py:510] This model supports multiple tasks: {'embed', 'classify', 'generate', 'reward', 'score'}. Defaulting to 'generate'. +INFO 03-13 09:37:35 config.py:1339] Disabled the custom all-reduce kernel because it is not supported on AMD GPUs. +INFO 03-13 09:37:35 llm_engine.py:234] Initializing an LLM engine (v0.6.6) with config: model='/models/facebook/opt-125m', speculative_config=None, tokenizer='/models/facebook/opt-125m', skip_tokenizer_init=False, + tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=2048, download_dir=None, load_format=serverless_llm, tensor_para +llel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig( +guided_decoding_backend='xgrammar'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=/models/faceb +ook/opt-125m, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=False, chunked_prefill_enabled=False, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs +=None, pooler_config=None, compilation_config={"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"candidate_compile_sizes":[],"compile_sizes":[],"capture_sizes":[256,248,240,232,224,2 +16,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":256}, use_cached_outputs=False, +INFO 03-13 09:37:38 selector.py:134] Using ROCmFlashAttention backend. +INFO 03-13 09:37:39 model_runner.py:1094] Starting to load model /models/facebook/opt-125m... +DEBUG 03-13 09:37:39 torch.py:137] allocate_cuda_memory takes 0.0004572868347167969 seconds +DEBUG 03-13 09:37:39 client.py:72] load_into_gpu: facebook/opt-125m/rank_0, 8554547c-25d3-4a01-92b6-27d69d91d3b8 +INFO 03-13 09:37:39 client.py:113] Model loaded: facebook/opt-125m/rank_0, 8554547c-25d3-4a01-92b6-27d69d91d3b8 +INFO 03-13 09:37:39 torch.py:160] restore state_dict takes 0.00017452239990234375 seconds +INFO 03-13 09:37:39 client.py:117] confirm_model_loaded: facebook/opt-125m/rank_0, 8554547c-25d3-4a01-92b6-27d69d91d3b8 +INFO 03-13 09:37:39 client.py:125] Model loaded +INFO 03-13 09:37:39 model_runner.py:1099] Loading model weights took 0.0000 GB +/app/third_party/vllm/vllm/model_executor/layers/linear.py:140: UserWarning: Attempting to use hipBLASLt on an unsupported architecture! Overriding blas backend to hipblas (Triggered internally at ../aten/src/ATen +/Context.cpp:296.) + return F.linear(x, layer.weight, bias) +INFO 03-13 09:37:42 worker.py:253] Memory profiling takes 2.68 seconds +INFO 03-13 09:37:42 worker.py:253] the current vLLM instance can use total_gpu_memory (23.98GiB) x gpu_memory_utilization (0.90) = 21.59GiB +INFO 03-13 09:37:42 worker.py:253] model weights take 0.00GiB; non_torch_memory takes 0.62GiB; PyTorch activation peak memory takes 0.46GiB; the rest of the memory reserved for KV Cache is 20.50GiB. +INFO 03-13 09:37:42 gpu_executor.py:76] # GPU blocks: 37326, # CPU blocks: 7281 +INFO 03-13 09:37:42 gpu_executor.py:80] Maximum concurrency for 2048 tokens per request: 291.61x +INFO 03-13 09:37:43 model_runner.py:1429] Capturing cudagraphs for decoding. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '-- +enforce-eager' in the CLI. If out-of-memory error occurs during cudagraph capture, consider decreasing `gpu_memory_utilization` or switching to eager mode. You can also reduce the `max_num_seqs` as needed to decre +ase memory usage. +Capturing CUDA graph shapes: 100%|████████████████████████████████████████| 35/35 [00:09<00:00, 3.73it/s] +INFO 03-13 09:37:52 model_runner.py:1549] Graph capturing finished in 9 secs, took 0.06 GiB +INFO 03-13 09:37:52 llm_engine.py:431] init engine (profile, create kv cache, warmup model) took 12.80 seconds +Processed prompts: 100%|█| 4/4 [00:00<00:00, 50.16it/s, est. speed input: 326.19 toks/s, output: 802.89 to +Prompt: 'Hello, my name is', Generated text: ' Joel, my dad is my friend and we are in a relationship. I am' +Prompt: 'The president of the United States is', Generated text: ' speaking out against the release of some State Department documents which show the Russians were involved' +Prompt: 'The capital of France is', Generated text: ' a worldwide knowledge center. What better place to learn about the history and culture of' +Prompt: 'The future of AI is', Generated text: " here: it's the future of everything\nIf you want to test your minds" +[rank0]:[W313 09:37:53.050846849 ProcessGroupNCCL.cpp:1250] Warning: WARNING: process group has NOT been destroyed before we destruct ProcessGroupNCCL. On normal program exit, the application should call destroy_p +rocess_group to ensure that any pending NCCL operations have finished in this process. In rare cases this process can exit before this point and block the progress of another member of the process group. This cons +traint has always been present, but this warning has only been added since PyTorch 2.4 (function operator()) + +``` + +## Build the wheel from source and install + +Currently, `pip install .` does not work with ROCm. We suggest you build `sllm-store` wheel and manually install it in your environment. + + + +If there's a customized PyTorch version installed, you may need to run the following command to modify the `torch` version in `requirements.txt`: + +```bash +python3 using_existing_torch.py +``` + +2. Build the wheel: + +```bash +python setup.py sdist bdist_wheel +``` + +## Known issues + +1. GPU memory leak in ROCm before version 6.2.0. + +This issue is due to an internal bug in ROCm. After the inference instance is completed, the GPU memory is still occupied and not released. For more information, please refer to [issue](https://github.com/ROCm/HIP/issues/3580). + +2. vLLM v0.5.0.post1 can not be built in ROCm 6.2.0 + +This issue is due to the ambiguity of a function call in ROCm 6.2.0. You may change the vLLM's source code as in this [commit](https://github.com/vllm-project/vllm/commit/9984605412de1171a72d955cfcb954725edd4d6f). diff --git a/versioned_docs/version-0.8.0/README.md b/versioned_docs/version-0.8.0/README.md new file mode 100644 index 0000000..4a1397a --- /dev/null +++ b/versioned_docs/version-0.8.0/README.md @@ -0,0 +1,39 @@ +# ServerlessLLM documents + +Please find our documents in [ServerlessLLM](https://serverlessllm.github.io/docs/getting_started). + +## How to build ServerlessLLM Docs + +This website is built using Docusaurus, a modern static website generator. + +### Installation + +To install the necessary dependencies, use the following command: + +```bash +npm install +``` + +### Local Development + +To start a local development server and open up a browser window, use the following command: + +```bash +npm run start +``` + +Most changes are reflected live without having to restart the server. + +### Build + +To generate static content into the build directory, use the following command: + +```bash +npm run build +``` + +This command generates static content into the `build` directory, which can be served using any static content hosting service. + +### About the image path + +Images are stored in `images` path. For example, we have an image called `a.jpg` in `images`. When we use this image in any position in the documents, we just use `/img/a.jpg`. (The document sync bot can copy `images` path into `img` folder in `serverlessllm.github.io` repo) diff --git a/versioned_docs/version-0.8.0/api/cli.md b/versioned_docs/version-0.8.0/api/cli.md new file mode 100644 index 0000000..7769797 --- /dev/null +++ b/versioned_docs/version-0.8.0/api/cli.md @@ -0,0 +1,323 @@ +--- +sidebar_position: 2 +--- + +# CLI API + +## ServerlessLLM CLI Documentation + +## Overview +`sllm` is the official command-line interface for interacting with ServerlessLLM. It is implemented using the [Click](https://click.palletsprojects.com/) framework to provide a flexible and extensible interface for managing model start, deploy, delete, and system status. + +### Installation + +```bash +# Create a new environment +conda create -n sllm python=3.10 -y +conda activate sllm + +# Install ServerlessLLM +pip install serverless-llm +``` + + +The CLI organizes commands into clearly scoped modules. This document outlines each available command along with its usage and configuration options. + +--- + +## Getting Started + +Before using the `sllm` commands, you need to start the ServerlessLLM cluster. Follow the guides below to set up your cluster: + +- [Single Machine Deployment](../getting_started.md) +- [Single Machine Deployment (From Scratch)](../deployment/single_machine.md) +- [Multi-Machine Deployment](../deployment/multi_machine.md) +- [SLURM Cluster Deployment](../deployment/slurm_cluster.md) + +After setting up the ServerlessLLM cluster, you can use the commands listed below to manage and interact with your models. + +--- + +## Available Commands + +To see all available CLI commands, run: + +```bash +sllm --help +``` + +**Example output:** +```text +Usage: sllm [OPTIONS] COMMAND [ARGS]... + + Unified CLI for ServerlessLLM. + +Options: + --help Show this message and exit. + +Commands: + delete Delete deployed models by name. + deploy Deploy a model using a config file or model name. + start Start the head node of the SLLM cluster. + status Show all deployed models. +``` + +--- + +## Example Workflow + +### 1. Start the Cluster + +```bash +sllm start +``` +**Example output:** +```text +[ℹ] Starting services using docker-compose.yml... +[+] Running 3/3 + ✔ sllm_api Started + ✔ sllm_worker_0 Started + ✔ sllm_worker_1 Started +``` + +--- + +### 2. Deploy a Model + +```bash +sllm deploy --model facebook/opt-1.3b +``` +**Example output:** +```text +[✓] Successfully deployed model: facebook/opt-1.3b with 1 GPU(s). +``` + +--- + +### 3. Check Deployment Status + +```bash +sllm status +``` +**Example output:** +```text +[✓] Deployed Models: +- facebook/opt-1.3b +``` + +--- + +### 4. Delete a Model + +```bash +sllm delete facebook/opt-1.3b +``` +**Example output:** +```text +[✓] Deleted model: facebook/opt-1.3b +``` + +--- + +## Command Reference + +### sllm start + +Start the head node of the SLLM cluster. This command initializes Docker services (or other configured backends) that manage the API and worker nodes. + +**Usage:** +```bash +sllm start +``` + +--- + +### sllm deploy + +Deploy a model using a configuration file or model name, with options to overwrite default configurations. + +**Usage:** +```bash +sllm deploy [OPTIONS] +``` + +**Options:** +- `--model ` + Model name to deploy (must be a HuggingFace pretrained model name). +- `--config ` + Path to the JSON configuration file. +- `--backend ` + Overwrite the backend in the configuration. +- `--num-gpus ` + Number of GPUs to allocate. +- `--target ` + Target concurrency. +- `--min-instances ` + Minimum number of instances. +- `--max-instances ` + Maximum number of instances. + +**Examples:** +```bash +sllm deploy --model facebook/opt-1.3b +sllm deploy --config /path/to/config.json +sllm deploy --model facebook/opt-1.3b --backend transformers +sllm deploy --model facebook/opt-1.3b --num-gpus 2 --target 5 --min-instances 1 --max-instances 5 +``` + +#### Example Configuration File (`config.json`) + +```json +{ + "model": "facebook/opt-1.3b", + "backend": "transformers", + "num_gpus": 1, + "auto_scaling_config": { + "metric": "concurrency", + "target": 1, + "min_instances": 0, + "max_instances": 10, + "keep_alive": 0 + }, + "backend_config": { + "pretrained_model_name_or_path": "facebook/opt-1.3b", + "device_map": "auto", + "torch_dtype": "float16", + "hf_model_class": "AutoModelForCausalLM", + "enable_lora": true, + "lora_adapters": { + "demo_lora1": "crumb/FLAN-OPT-1.3b-LoRA", + "demo_lora2": "GrantC/alpaca-opt-1.3b-lora" + } + } +} +``` + +##### Example Quantization Configuration (`config.json`) +`quantization_config` can be obtained from any configuration used in `transformers` via the `.to_json_file(filename)` function: + +```python +quantization_config = BitsAndBytesConfig(load_in_8bit=True) +quantization_config.to_json_file("quantization_config.json") + +``` +Then copy it into `config.json`: + +```json +{ + "model": "", + "backend": "transformers", + "num_gpus": 1, + "auto_scaling_config": { + "metric": "concurrency", + "target": 1, + "min_instances": 0, + "max_instances": 10, + "keep_alive": 0 + }, + "backend_config": { + "pretrained_model_name_or_path": "", + "device_map": "auto", + "torch_dtype": "float16", + "hf_model_class": "AutoModelForCausalLM", + "quantization_config": { + "_load_in_4bit": false, + "_load_in_8bit": true, + "bnb_4bit_compute_dtype": "float32", + "bnb_4bit_quant_storage": "uint8", + "bnb_4bit_quant_type": "fp4", + "bnb_4bit_use_double_quant": false, + "llm_int8_enable_fp32_cpu_offload": false, + "llm_int8_has_fp16_weight": false, + "llm_int8_skip_modules": null, + "llm_int8_threshold": 6.0, + "load_in_4bit": false, + "load_in_8bit": true, + "quant_method": "bitsandbytes" + } + } +} +``` + +Below is a description of all the fields in config.json. + +| Field | Description | +| ----- | ----------- | +| model | HuggingFace model name, used to identify model instance. | +| backend | Inference engine, supports `transformers` and `vllm`. | +| num_gpus | Number of GPUs used to deploy a model instance. | +| auto_scaling_config | Config about auto scaling. | +| auto_scaling_config.metric | Metric used to decide whether to scale up or down. | +| auto_scaling_config.target | Target value of the metric. | +| auto_scaling_config.min_instances | Minimum number of model instances. | +| auto_scaling_config.max_instances | Maximum number of model instances. | +| auto_scaling_config.keep_alive | How long a model instance stays alive after inference ends. | +| backend_config | Config about inference backend. | +| backend_config.pretrained_model_name_or_path | HuggingFace model name or local path. | +| backend_config.device_map | Device map config used to load the model. | +| backend_config.torch_dtype | Torch dtype of the model. | +| backend_config.hf_model_class | HuggingFace model class. | +| backend_config.enable_lora | Set to true to enable loading LoRA adapters during inference. | +| backend_config.lora_adapters| A dictionary of LoRA adapters in the format `{name: path}`, where each path is a local or Hugging Face-hosted LoRA adapter directory. | +| backend_config.quantization_config| A dictionary specifying the desired `BitsAndBytesConfig`. Can be obtained by saving a `BitsAndBytesConfig` to JSON via `BitsAndBytesConfig.to_json_file(filename). Defaults to None.| + +--- + +### sllm delete + +Delete deployed models by name. + +**Usage:** +```bash +sllm delete [MODELS...] +``` +**Arguments:** +- `MODELS...` + One or more space-separated model names to delete. + +**Example:** +```bash +sllm delete facebook/opt-1.3b facebook/opt-2.7b meta/llama2 +``` +**Example output:** +```text +[✓] Deleted model: facebook/opt-1.3b +[✓] Deleted model: facebook/opt-2.7b +[✓] Deleted model: meta/llama2 +``` + +--- + +### sllm status + +Check the current status of all deployed models. This command displays the list of models currently running in the ServerlessLLM cluster, including their state, GPU usage, and endpoint. + +**Usage:** +```bash +sllm status +``` +**Example output:** +```text +[✓] Deployed Models: +- facebook/opt-1.3b Running +- meta/llama2 Running + +--- + +## Notes + +- All commands should be run as `sllm ...` after installation. +- For advanced configuration, refer to the [Example Configuration File](#example-configuration-file-configjson) section. +- For more details, see the official documentation and guides linked above. + + + +#### Example +```bash +sllm status +``` + +#### Example +```bash +sllm-cli status +``` diff --git a/versioned_docs/version-0.8.0/api/intro.md b/versioned_docs/version-0.8.0/api/intro.md new file mode 100644 index 0000000..f3a7d29 --- /dev/null +++ b/versioned_docs/version-0.8.0/api/intro.md @@ -0,0 +1,9 @@ +--- +sidebar_position: 1 +--- + +# API Introduction + +Welcome to the ServerlessLLM API documentation. This section contains detailed information about the various APIs provided by ServerlessLLM: + +- [CLI API](./cli.md) - Documentation for the `sllm` command-line interface diff --git a/versioned_docs/version-0.8.0/api/sllm-store-cli.md b/versioned_docs/version-0.8.0/api/sllm-store-cli.md new file mode 100644 index 0000000..1efe935 --- /dev/null +++ b/versioned_docs/version-0.8.0/api/sllm-store-cli.md @@ -0,0 +1,242 @@ +--- +sidebar_position: 2 +--- + +# ServerlessLLM Store CLI + +ServerlessLLM Store's CLI allows the use `sllm-store`'s functionalities within a terminal window. It has the functions: +- `start`: Starts the gRPC server with the specified configuration. +- `save`: Convert a HuggingFace model into a loading-optimized format and save it to a local path. +- `load`: Load a model into given GPUs. + +## Requirements +- OS: Ubuntu 22.04 +- Python: 3.10 +- GPU: compute capability 7.0 or higher + +## Installations + +### Create a virtual environment +``` bash +conda create -n sllm-store python=3.10 -y +conda activate sllm-store +``` + +### Install C++ Runtime Library (required for compiling and running CUDA/C++ extensions) +``` bash +conda install -c conda-forge libstdcxx-ng=12 -y +``` + +### Install with pip +```bash +pip install serverless-llm-store +``` + +## Example Workflow +1. Firstly, start the ServerlessLLM Store server. By default, it uses ./models as the storage path. +Launch the checkpoint store server in a separate process: +``` bash +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + +2. Convert a model to ServerlessLLM format and save it to a local path: +``` bash +sllm-store save --model facebook/opt-1.3b --backend vllm +``` + +3. Load a previously saved model into memory, ready for inference: +```bash +sllm-store load --model facebook/opt-1.3b --backend vllm +``` + +## sllm-store start + +Start a gRPC server to serve models stored using ServerlessLLM. This enables fast, low-latency access to models registered via sllm-store save, allowing external clients to load model weights, retrieve metadata, and perform inference-related operations efficiently. + +The server supports in-memory caching with customizable memory pooling and chunking, optimized for parallel read access and minimal I/O latency. + +#### Usage +```bash +sllm-store start [OPTIONS] +``` + +#### Options + +- `--host ` + - Host address to bind the gRPC server to. + +- `--port ` + - Port number on which the gRPC server will listen for incoming requests. + +- `--storage-path ` + - Path to the directory containing models previously saved with sllm-store save. + +- `--num-thread ` + - Number of threads to use for I/O operations and chunk handling. + +- `--chunk-size ` + - Size of individual memory chunks used for caching model data (e.g., 64MiB, 512KB). Must include unit suffix. + +- `--mem-pool-size ` + - Total memory pool size to allocate for the in-memory cache (e.g., 4GiB, 2GB). Must include unit suffix. + +- `--disk-size ` + - (Currently unused) Would set the maximum size sllm-store can occupy in disk cache. + +- `--registration-required` + - If specified, models must be registered with the server before loading. + +#### Examples + +Start the server using all default values: +``` bash +sllm-store start +``` + +Start the server with a custom storage path: +``` bash +sllm-store start --storage-path /your/folder +``` + +Specify a custom port and host: +``` bash +sllm-store start --host 127.0.0.1 --port 9090 +``` + +Use larger chunk size and memory pool for large models in a multi-threaded environment: +``` bash +sllm-store start --num-thread 16 --chunk-size 128MB --mem-pool-size 8GB +``` + +Run with access control enabled: +``` bash +sllm-store start --registration-required True +``` + +Full example for production-style setup: +``` bash +sllm-store start \ + --host 0.0.0.0 \ + --port 8000 \ + --storage-path /data/models \ + --num-thread 8 \ + --chunk-size 64MB \ + --mem-pool-size 16GB \ + --registration-required True +``` + +## sllm-store save + +Saves a model to a local directory through a backend of choice, making it available for future inference requests. Only model name and backend are required, with the rest having default values. + +It supports download of [PEFT LoRA (Low-Rank Adaptation)](https://huggingface.co/docs/peft/main/en/index) for transformer models, and varying tensor sizes for parallel download of vLLM models. + + +#### Usage +```bash +sllm-store save [OPTIONS] +``` + +#### Options + +- `--model ` + - Model name to deploy with default configuration. The model name must be a Hugging Face pretrained model name. You can find the list of available models [here](https://huggingface.co/models). + +- `--backend ` + - Select a backend for the model to be converted to `ServerlessLLM format` from. Supported backends are `vllm` and `transformers`. + +- `--adapter` + - Enable LoRA adapter support. Overwrite `adapter`, which is by default set to False. Only `transformers` backend is supported. + +- `--adapter-name ` + - Adapter name to save. Must be a Hugging Face pretrained LoRA adapter name. + +- `--tensor-parallel-size ` + - Number of GPUs you want to use. Only `vllm` backend is supported. + +- `--local-model-path ` + - Saves the model from a local path if it contains a Hugging Face snapshot of the model. + +- `--storage-path ` + - Location where the model will be saved. + +#### Examples +Save a vLLM model name with default configuration: +```bash +sllm-store save --model facebook/opt-1.3b --backend vllm +``` + +Save a transformers model to a set location: +```bash +sllm-store save --model facebook/opt-1.3b --backend vllm --storage-path ./your/folder +``` + +Save a vLLM model from a locally stored snapshot and overwrite the tensor parallel size: +```bash +sllm-store save --model facebook/opt-1.3b --backend vllm --tensor-parallel-size 4 --local-model-path ./path/to/snapshot +``` + +Save a transformers model with a LoRA adapter: +```bash +sllm-store save --model facebook/opt-1.3b --backend transformers --adapter --adapter-name crumb/FLAN-OPT-1.3b-LoRA +``` + +## sllm-store load + +Load a model from local storage and run example inference to verify deployment. This command supports both the transformers and vllm backends, with optional support for PEFT LoRA adapters and quantized precision formats including int8, fp4, and nf4 (LoRA and quantization supported on transformers backend only). + +When using the transformers backend, the function warms up GPU devices, loads the base model from disk, and optionally merges a LoRA adapter if specified. With vllm, it loads the model in the ServerlessLLM format. + +#### Usage +```bash +sllm-store load [OPTIONS] +``` + +#### Options + +- `--model ` + - Model name to deploy with default configuration. The model name must be a Hugging Face pretrained model name. You can find the list of available models [here](https://huggingface.co/models). + +- `--backend ` + - Select a backend for the model to be converted to `ServerlessLLM format` from. Supported backends are `vllm` and `transformers`. + +- `--adapter` + - Enable LoRA adapter support for the transformers backend. Overwrite `adapter` in the default configuration (`transformers` backend only). + +- `--adapter-name ` + - Adapter name to save. Must be a Hugging Face pretrained LoRA adapter name. + +- `--precision ` + - Precision to use when loading the model (`transformers` backend only). For more info on quantization in ServerlessLLM, visit [here](https://serverlessllm.github.io/docs/store/quickstart#quantization). + +- `--storage-path ` + - Location where the model will be loaded from. + +#### Examples +Load a vllm model from storage: +``` bash +sllm-store load --model facebook/opt-1.3b --backend vllm +``` + +Load a transformers model from storage with int8 quantization: +``` bash +sllm-store load --model facebook/opt-1.3b --backend transformers --precision int8 --storage-path ./your/models +``` + +Load a transformers model with a LoRA adapter: +``` bash +sllm-store load --model facebook/opt-1.3b --backend transformers --adapter --adapter-name crumb/FLAN-OPT-1.3b-LoRA +``` + +#### Note: loading vLLM models + +To load models with vLLM, you need to apply a compatibility patch to your vLLM installation. This patch has been tested with vLLM version `0.9.0.1`. + +```bash + ./sllm_store/vllm_patch/patch.sh +``` + +:::note +The patch file is located at `sllm_store/vllm_patch/sllm_load.patch` in the ServerlessLLM repository. +::: \ No newline at end of file diff --git a/versioned_docs/version-0.8.0/community/_category_.json b/versioned_docs/version-0.8.0/community/_category_.json new file mode 100644 index 0000000..8a81fd9 --- /dev/null +++ b/versioned_docs/version-0.8.0/community/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Community", + "position": 8 +} diff --git a/versioned_docs/version-0.8.0/community/meetups.md b/versioned_docs/version-0.8.0/community/meetups.md new file mode 100644 index 0000000..8e3811a --- /dev/null +++ b/versioned_docs/version-0.8.0/community/meetups.md @@ -0,0 +1,11 @@ +# ServerlessLLM Meetups + +We host regular biweekly developer meetings online. We will share project updates from the ServerlessLLM developer team presented during these meetings. Please find the materials of our previous meetups below: + +Date |Topic |Slides +---------------|-------------|--------- +February 21st 2025 | Fine Tuning | [Slides](https://docs.google.com/presentation/d/1rnw3mieAAbMabDIoIGS-ciMGc3hJ7AICYSaNJp-Fk4s/edit?usp=sharing) +March 7th 2025 |Quantization |[Slides](https://docs.google.com/presentation/d/1uSbP-LzGbbvPsemIAE6jCFsggYm_ATxQguCHDmdwoXE/edit?usp=sharing) + +We are always looking for contributors to join us on the developer team. If you are interested in contributing, consult our [job board](https://github.com/orgs/ServerlessLLM/projects/2) and claim a feature. For any other questions, please contact us on [this email](mailto:Y.Fu@ed.ac.uk) or on [our Discord server](https://discord.gg/AEF8Gduvm8). + diff --git a/versioned_docs/version-0.8.0/community/talks.md b/versioned_docs/version-0.8.0/community/talks.md new file mode 100644 index 0000000..66a2531 --- /dev/null +++ b/versioned_docs/version-0.8.0/community/talks.md @@ -0,0 +1,8 @@ +# ServerlessLLM Talks + +Materials for ServerlessLLM talks will be listed here. + +Topic |Location |Date |Links +-------------|----------------|---------------|------------------------------------ +Efficient Sharing of AI Infrastructures with Specialized Serverless Computing | University of Pennsylvania |January 29th 2025 |[Slides](https://drive.google.com/file/d/17GwXsqaDDS7Xw8nX_-RaKiwpaPQgu9WD/view) \| [Event](https://asset.seas.upenn.edu/event/yao-fu-university-of-edinburgh/) +ServerlessLLM Tutorial | SESAME'25 | March 31st 2025 |[Slides](https://docs.google.com/presentation/d/1ioGCVpsg0x3oCxX19EiE820aMiY22X5MG6jgImZ1W18/edit?usp=sharing) \| [Event](https://sesame25.github.io/) diff --git a/versioned_docs/version-0.8.0/deployment/_category_.json b/versioned_docs/version-0.8.0/deployment/_category_.json new file mode 100644 index 0000000..534be1d --- /dev/null +++ b/versioned_docs/version-0.8.0/deployment/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Deployment", + "position": 3 +} diff --git a/versioned_docs/version-0.8.0/deployment/multi_machine.md b/versioned_docs/version-0.8.0/deployment/multi_machine.md new file mode 100644 index 0000000..21583c3 --- /dev/null +++ b/versioned_docs/version-0.8.0/deployment/multi_machine.md @@ -0,0 +1,296 @@ +--- +sidebar_position: 2 +--- + +# Multi-machine + +This guide will help you get started with running ServerlessLLM on multiple machines using Docker containers. You'll learn how to set up a head node on one machine and connect worker nodes from different machines using Docker, ensuring proper network communication between the containers. You can extend this setup to use as many nodes as you need. + +## Prerequisites + +This guide requires **two machines**: +- One machine for the head node (no GPU required) +- One machine with an NVIDIA GPU to serve as the worker node + +You can add more worker machines with GPUs as needed to scale out your deployment. + +### For All Machines + +Ensure you have the following installed and configured on all machines (both head node and worker machines): + +1. **Docker**: Installed on your system. You can download it from [here](https://docs.docker.com/get-docker/). +2. **Network connectivity**: Ensure all machines can communicate with each other on the required ports (6379 for Ray, 8343 for ServerlessLLM API, and 8073 for storage service). + +:::tip +The **ServerlessLLM CLI** (`pip install serverless-llm`) can be installed on any machine that needs to manage model deployments. This could be your local computer or any machine within the cluster that can connect to the head node. +::: + +### For Worker Machines Only + +These requirements are only necessary for the worker machines that will run the models: + +1. **GPUs**: At least one NVIDIA GPU is required on each worker machine. If you have multiple GPUs, you can adjust the Docker configuration accordingly. +2. **NVIDIA Docker Toolkit**: This enables Docker to utilize NVIDIA GPUs. Follow the installation guide [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). + +## Multi-Machine Setup + +We'll start a head node on one machine using Docker, then add a worker node from another machine using Docker containers with host networking. + +### Step 1: Start the Head Node + +1. **Start the head node using Docker:** + +```bash +# Get the machine's IP address that will be accessible to other machines +export HEAD_IP=$(hostname -I | awk '{print $1}') +echo "Head node IP address: $HEAD_IP" + +docker run -d \ + --name sllm_head \ + --network host \ + -e MODE=HEAD \ + -e RAY_NODE_IP=$HEAD_IP \ + serverlessllm/sllm:latest +``` + +:::important +For multi-machine setups, setting the `RAY_NODE_IP` is critical. It should be set to an IP address that is accessible from all worker machines. The command above attempts to automatically determine your machine's primary IP, but in complex network environments, you may need to specify it manually. + +If your machine has multiple network interfaces, ensure you use the IP that other machines in your network can access. +::: + +:::tip +If you don't have the ServerlessLLM Docker image locally, Docker will automatically pull it from the registry. You can also adjust the CPU and resource allocations by setting additional environment variables like `RAY_NUM_CPUS` and `RAY_RESOURCES`. +::: + +2. **Verify the head node is running and note the external IP:** + +```bash +docker logs sllm_head +``` + +Expected output should include: + +```bash +> docker logs sllm_head +... +2025-05-29 14:29:46,211 INFO scripts.py:744 -- Local node IP: 129.215.164.107 +... +(SllmController pid=380) INFO 05-29 14:29:53 controller.py:59] Starting store manager +(SllmController pid=380) INFO 05-29 14:29:56 controller.py:68] Starting scheduler +(StoreManager pid=417) INFO 05-29 14:29:56 store_manager.py:226] Initializing store manager +(StoreManager pid=417) INFO 05-29 14:29:56 store_manager.py:237] Initializing cluster and collecting hardware info +(StoreManager pid=417) ERROR 05-29 14:29:56 store_manager.py:242] No worker nodes found +INFO: Started server process [1] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:8343 (Press CTRL+C to quit) +(FcfsScheduler pid=456) INFO 05-29 14:29:56 fcfs_scheduler.py:54] Starting FCFS scheduler +(FcfsScheduler pid=456) INFO 05-29 14:29:56 fcfs_scheduler.py:111] Starting control loop +``` + +Make note of the IP address shown in the logs. This is the address that worker nodes will use to connect to the head node. + +### Step 2: Start Worker Node on a Different Machine + +:::tip +You can adjust the memory pool size and other parameters based on the resources available on your worker machine. +::: + +1. **On the worker machine, create a directory for model storage:** + +```bash +mkdir -p /path/to/your/models +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +2. **Start the worker node:** + +```bash +# Replace with the actual IP address of the head node from the previous step +# DO NOT copy-paste this line directly - update with your actual head node IP +export HEAD_IP= +``` + +```bash +# Get the worker machine's IP address that will be accessible to the head node +export WORKER_IP=$(hostname -I | awk '{print $1}') +echo "Worker node IP address: $WORKER_IP" + +docker run -d \ + --name sllm_worker_0 \ + --network host \ + --gpus '"device=0"' \ + -e WORKER_ID=0 \ + -e STORAGE_PATH=/models \ + -e MODE=WORKER \ + -e RAY_HEAD_ADDRESS=${HEAD_IP}:6379 \ + -e RAY_NODE_IP=$WORKER_IP \ + -v ${MODEL_FOLDER}:/models \ + serverlessllm/sllm:latest \ + --mem-pool-size 4GB --registration-required true +``` + +:::important +For multi-machine setups, setting the `RAY_NODE_IP` on worker nodes is just as critical as on the head node. It should be set to an IP address that is accessible from the head node. Without this, workers might report internal Docker IPs that aren't accessible across machines. + +Make sure to replace `192.168.1.100` with the actual IP address of your head node that you noted earlier. +::: + +3. **Verify worker node is connected:** + +On the worker machine, check if the worker has properly connected to the Ray cluster: + +```bash +docker exec -it sllm_worker_0 bash -c "source /opt/conda/etc/profile.d/conda.sh && conda activate worker && ray status" +``` + +Expected output should include both the head node and worker node resources: + +```bash +> docker exec -it sllm_worker_0 bash -c "source /opt/conda/etc/profile.d/conda.sh && conda activate worker && ray status" +======== Autoscaler status: 2025-05-29 14:42:30.434645 ======== +Node status +--------------------------------------------------------------- +Active: + 1 node_f0a8e97ca64c64cebd551f469a38d0d66ce304f7cc1cc9696fe33cf3 + 1 node_3b7db178afb8bdb16460d0cb6463dc7b9b3afbcc00753c3be110c9b3 +Pending: + (no pending nodes) +Recent failures: + (no failures) + +Resources +--------------------------------------------------------------- +Usage: + 3.0/52.0 CPU + 0.0/1.0 GPU + 0.30000000000000004/1.0 control_node + 0B/526.36GiB memory + 0B/18.63GiB object_store_memory + 0.0/1.0 worker_id_0 + 0.0/1.0 worker_node + +Demands: + (no resource demands) +``` + +This output confirms that both the head node and worker node are properly connected and their resources are recognized by the Ray cluster. + +:::tip +**Adding more worker nodes:** You can add more worker nodes by repeating Step 2 on additional machines with GPUs. Just make sure to: +1. Use a unique `WORKER_ID` for each worker (1, 2, 3, etc.) +2. Point each worker to the same head node IP address +3. Ensure each worker has its own `RAY_NODE_IP` set correctly +::: + +### Step 3: Use `sllm` to manage models + +#### Configure the Environment + +**On any machine with `sllm` installed, set the `LLM_SERVER_URL` environment variable:** + +> Replace `` with the actual IP address of the head node. + +```bash +export LLM_SERVER_URL=http://:8343 +``` + +#### Deploy a Model Using `sllm` + +```bash +sllm deploy --model facebook/opt-1.3b +``` + +> Note: This command will spend some time downloading the model from the Hugging Face Model Hub. You can use any model from the [Hugging Face Model Hub](https://huggingface.co/models) by specifying the model name in the `--model` argument. + +Expected output: + +```bash +INFO 07-24 06:51:32 deploy.py:83] Model registered successfully. +``` + +### Step 4: Query the Model Using OpenAI API Client + +**You can query the model using any OpenAI API client. For example, use the following command:** + +**Make sure the model is successfully deployed before querying.** + +> Replace `` with the actual IP address of the head node. + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +Expected output: + +```json +{"id":"chatcmpl-23d3c0e5-70a0-4771-acaf-bcb2851c6ea6","object":"chat.completion","created":1721706121,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} +``` + +#### Delete a Deployed Model Using `sllm` + +When you're done using a model, you can delete it: + +```bash +sllm delete facebook/opt-1.3b +``` + +This will remove the specified model from the ServerlessLLM server. + +## Clean Up + +To stop and remove all ServerlessLLM containers: + +1. **Stop all containers:** + +```bash +# On head node machine +docker stop sllm_head +docker rm sllm_head + +# On each worker machine +docker stop sllm_worker_0 # Use appropriate container name (sllm_worker_1, sllm_worker_2, etc.) +docker rm sllm_worker_0 +``` + +2. **Optional: Remove the Docker image:** + +```bash +docker rmi serverlessllm/sllm:latest +``` + +:::tip +If you don't have the ServerlessLLM Docker image locally, Docker will automatically pull it from the registry. You can also adjust the CPU and resource allocations by setting additional environment variables like `RAY_NUM_CPUS` and `RAY_RESOURCES`. +::: + +## Troubleshooting + +### Network Issues + +1. **Connection refused errors**: Ensure that firewalls on all machines allow traffic on ports 6379, 8343, and 8073. + +2. **Ray cluster connection issues**: + - Verify that the head node IP address is correct and that the Ray port (6379) is accessible from worker machines + - Ensure both head and worker nodes have their `RAY_NODE_IP` set to an IP address that is accessible from other machines + - Check that you're not using private Docker network IPs (typically 172.x.x.x) which aren't accessible across machines + +3. **Workers can't connect to head node**: + - Make sure the `RAY_HEAD_ADDRESS` points to the external IP of the head node, not localhost or an internal Docker IP + - Verify network connectivity with `ping` or `telnet` from worker machines to the head node IP on port 6379 + +4. **GPU access issues**: Make sure the NVIDIA Docker toolkit is properly installed and that the `--gpus` flag is used for worker containers. + +### Container Management + +- **View running containers**: `docker ps` \ No newline at end of file diff --git a/versioned_docs/version-0.8.0/deployment/single_machine.md b/versioned_docs/version-0.8.0/deployment/single_machine.md new file mode 100644 index 0000000..7e064a2 --- /dev/null +++ b/versioned_docs/version-0.8.0/deployment/single_machine.md @@ -0,0 +1,217 @@ +--- +sidebar_position: 1 +--- + +# Single machine (from scratch) + +This guide provides instructions for setting up ServerlessLLM from scratch on a single machine. This 'from scratch' approach means you will manually initialize and manage the Ray cluster components. It involves using multiple terminal sessions, each configured with a distinct Conda environment, to run the head and worker processes on the same physical machine, effectively simulating a multi-node deployment locally. + +:::note +We strongly recommend using Docker (Compose) as detailed in the [Docker Compose guide](../getting_started.md). Docker provides a smoother and generally easier setup process. Follow this guide only if Docker is not a suitable option for your environment. +::: + +## Installation + +### Requirements + +Ensure your system meets the following prerequisites: + +- **OS**: Ubuntu 20.04 +- **Python**: 3.10 +- **GPU**: NVIDIA GPU with compute capability 7.0 or higher + +### Installing with pip + +Follow these steps to install ServerlessLLM using pip: + +**Create the head environment:** + +```bash +# Create and activate a conda environment +conda create -n sllm python=3.10 -y +conda activate sllm + +# Install ServerlessLLM and its store component +pip install serverless-llm serverless-llm-store +``` + +**Create the worker environment:** + +```bash +# Create and activate a conda environment +conda create -n sllm-worker python=3.10 -y +conda activate sllm-worker + +# Install ServerlessLLM (worker version) and its store component +pip install "serverless-llm[worker]" serverless-llm-store +``` + +:::note +If you plan to integrate vLLM with ServerlessLLM, a patch needs to be applied to the vLLM repository. For detailed instructions, please refer to the [vLLM Patch](#vllm-patch) section. +::: + +### Installing from Source + +To install ServerlessLLM from source, follow these steps: + +1. Clone the repository: + ```bash + git clone https://github.com/ServerlessLLM/ServerlessLLM.git + cd ServerlessLLM + ``` + +2. Create the head environment: + ```bash + # Create and activate a conda environment + conda create -n sllm python=3.10 -y + conda activate sllm + + # Install sllm_store (pip install is recommended for speed) + cd sllm_store && rm -rf build + pip install . + cd .. + + # Install ServerlessLLM + pip install . + ``` + +3. Create the worker environment: + ```bash + # Create and activate a conda environment + conda create -n sllm-worker python=3.10 -y + conda activate sllm-worker + + # Install sllm_store (pip install is recommended for speed) + cd sllm_store && rm -rf build + pip install . + cd .. + + # Install ServerlessLLM (worker version) + pip install ".[worker]" + ``` + +### vLLM Patch + +To use vLLM with ServerlessLLM, you must apply a patch. The patch file is located at `sllm_store/vllm_patch/sllm_load.patch` within the ServerlessLLM repository. This patch has been tested with vLLM version `0.9.0.1`. + +Apply the patch using the following script: + +```bash +conda activate sllm-worker +./sllm_store/vllm_patch/patch.sh +``` + +## Running ServerlessLLM Locally + +These steps describe how to run ServerlessLLM on your local machine. + +### 1. Start a Local Ray Cluster + +First, initiate a local Ray cluster. This cluster will consist of one head node and one worker node (on the same machine). + +**Start the head node:** + +Open a new terminal and run: + +```bash +conda activate sllm +ray start --head --port=6379 --num-cpus=4 --num-gpus=0 \ + --resources='{"control_node": 1}' --block +``` + +**Start the worker node:** + +Open another new terminal and run: + +```bash +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 # Or your desired GPU ID +ray start --address=0.0.0.0:6379 --num-cpus=4 --num-gpus=1 \ + --resources='{"worker_node": 1, "worker_id_0": 1}' --block +``` + +### 2. Start the ServerlessLLM Store Server + +Next, start the ServerlessLLM Store server. By default, it uses `./models` as the storage path. + +Open a new terminal and run: + +```bash +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 # Or your desired GPU ID +sllm-store start +``` + +Expected output: + +```log +$ sllm-store start +INFO 12-31 17:13:23 cli.py:58] Starting gRPC server +INFO 12-31 17:13:23 server.py:34] StorageServicer: storage_path=./models, mem_pool_size=4294967296, num_thread=4, chunk_size=33554432, registration_required=False +WARNING: Logging before InitGoogleLogging() is written to STDERR +I20241231 17:13:23.947276 2165054 checkpoint_store.cpp:41] Number of GPUs: 1 +I20241231 17:13:23.947299 2165054 checkpoint_store.cpp:43] I/O threads: 4, chunk size: 32MB +I20241231 17:13:23.947309 2165054 checkpoint_store.cpp:45] Storage path: "./models" +I20241231 17:13:24.038651 2165054 checkpoint_store.cpp:71] GPU 0 UUID: c9938b31-33b0-e02f-24c5-88bd6fbe19ad +I20241231 17:13:24.038700 2165054 pinned_memory_pool.cpp:29] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes +I20241231 17:13:25.557906 2165054 checkpoint_store.cpp:83] Memory pool created with 4GB +INFO 12-31 17:13:25 server.py:243] Starting gRPC server on 0.0.0.0:8073 +``` + +### 3. Start ServerlessLLM + +Now, start the ServerlessLLM service process using `sllm start`. + + +Open a new terminal and run: + +```bash +sllm start +``` + +At this point, you should have four terminals open: one for the Ray head node, one for the Ray worker node, one for the ServerlessLLM Store server, and one for the ServerlessLLM service (started via `sllm start`). + +### 4. Deploy a Model + +With all services running, you can deploy a model. + +Open a new terminal and run: + +```bash +conda activate sllm +sllm deploy --model facebook/opt-1.3b +``` + +This command downloads the specified model from Hugging Face Hub. To load a model from a local path, you can use a `config.json` file. Refer to the [CLI API documentation](../api/cli.md#example-configuration-file-configjson) for details. + +### 5. Query the Model + +Once the model is deployed, you can query it using any OpenAI API-compatible client. For example, use the following `curl` command: + +```bash +curl http://127.0.0.1:8343/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +Expected output: + +```json +{"id":"chatcmpl-9f812a40-6b96-4ef9-8584-0b8149892cb9","object":"chat.completion","created":1720021153,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} +``` + +## Clean Up + +To delete a deployed model, use the following command: + +```bash +sllm delete facebook/opt-1.3b +``` + +This command removes the specified model from the ServerlessLLM server. \ No newline at end of file diff --git a/versioned_docs/version-0.8.0/deployment/slurm_cluster.md b/versioned_docs/version-0.8.0/deployment/slurm_cluster.md new file mode 100644 index 0000000..5d3a476 --- /dev/null +++ b/versioned_docs/version-0.8.0/deployment/slurm_cluster.md @@ -0,0 +1,411 @@ +--- +sidebar_position: 3 +--- + +# SLURM cluster + +This guide will help you get started with running ServerlessLLM on SLURM cluster. It provides two deployment methods, based on `sbatch` and `srun`. If you are in development, we recommend using `srun`, as it is easier to debug than `sbatch`, and if you are in production mode, `sbatch` is recommended. Please make sure you have installed the ServerlessLLM following the [installation guide](./single_machine.md#installation) on all machines. + +## Pre-requisites +Before you begin, make sure you have checked the following: +### Some Tips about Installation +- If 'not enough disk space' is reported when `pip install` on the login node, you can submit it to a job node for execution + ```shell + #!/bin/bash + #SBATCH --partition=Teach-Standard + #SBATCH --job-name=ray-head + #SBATCH --output=sllm_pip.out + #SBATCH --error=sllm_pip.err + #SBATCH --nodes=1 + #SBATCH --ntasks=1 + #SBATCH --cpus-per-task=4 + #SBATCH --gpus-per-task=0 + + # Identify which conda you are using, here is an example that conda is in /opt/conda + source /opt/conda/bin/activate + + conda create -n sllm python=3.10 -y + conda activate sllm + pip install serverless-llm + pip install serverless-llm-store + + conda deactivate sllm + + conda create -n sllm-worker python=3.10 -y + conda activate sllm-worker + pip install serverless-llm[worker] + pip install serverless-llm-store + ``` + +### Command for Querying GPU Resource Information +Run the following commands in the cluster to check GPU resource information. +```shell +sinfo -O partition,nodelist,gres +``` +**Expected Output** +```shell +PARTITION NODELIST GRES +Partition1 JobNode[01,03] gpu:gtx_1060:8 +Partition2 JobNode[04-17] gpu:a6000:2,gpu:gtx_ +``` + +### Identify an idle node +Use `sinfo -p ` to identify some idle nodes + +**Expected Output** +```shell +$ sinfo -p compute +PARTITION AVAIL NODES STATE TIMELIMIT NODELIST +compute up 10 idle infinite JobNode[01-10] +compute up 5 alloc infinite JobNode[11-15] +compute up 2 down infinite JobNode[16-17] +``` + +### Job Nodes Setup +**`srun` Node Selection** + +Only one JobNode is enough. + +**`sbatch` Node Selection** +Let's start a head on the main job node (`JobNode01`) and add the worker on other job node (`JobNode02`). The head and the worker should be on different job nodes to avoid resource contention. The `sllm-store` should be started on the job node that runs worker (`JobNode02`), for passing the model weights, and the `sllm start` should be started on the main job node (`JobNode01`), finally you can use `sllm` to manage the models on the login node. + + +Note: `JobNode02` requires GPU, but `JobNode01` does not. +- **Head**: JobNode01 +- **Worker**: JobNode02 +- **sllm-store**: JobNode02 +- **sllm-serve**: JobNode01 +- **sllm**: Login Node + +--- +## SRUN +If you are in development, we recommend using `srun` to start ServerlessLLM, as it is easier to debug than `sbatch` +### Step 1: Use `srun` enter the JobNode +To start an interactive session on the specified compute node (JobNode), use: +``` +srun --partition --nodelist --gres :1 --pty bash +``` +This command requests a session on the specified node and provides an interactive shell. `--gres :1` specifies the GPU device you will use, for example: `--gres gpu:gtx_1060:1` + +### Step 2: Install ServerlessLLM +Firstly, please make sure CUDA driver available on the node. Here are some commands to check it. +```shell +nvidia-smi + +which nvcc +``` +If `nvidia-smi` has listed GPU information, but `which nvcc` has no output. Then use the following commands to load `nvcc`. Here is an example that cuda is located at `/opt/cuda-12.2.0` +```shell +export PATH=/opt/cuda-12.2.0/bin:$PATH +export LD_LIBRARY_PATH=/opt/cuda-12.2.0/lib64:$LD_LIBRARY_PATH +``` +Then, following the [installation guide](./single_machine.md#installation) to install ServerlessLLM. +### Step 3: Prepare multiple windows with `tmux` +Since srun provides a single interactive shell, you can use tmux to create multiple windows. Start a tmux session: +```shell +tmux +``` +This creates a new tmux session + +**Create multiple windows** +- Use `Ctrl+B` → `C` to start a new window +- Repeat the shortcut 4 more times to create a total of 5 windows. + +**What if `Ctrl+B` does not work?** + +If `Ctrl + B` is unresponsive, reset tmux key bindings: +```shell +tmux unbind C-b +tmux set-option -g prefix C-b +tmux bind C-b send-prefix +``` + +**Command to switch windows** + +Once multiple windows are created, you can switch between them using: + +`Ctrl + B` → `N` (Next window) +`Ctrl + B` → `P` (Previous window) +`Ctrl + B` → `W` (List all windows and select) +`Ctrl + B` → [Number] (Switch to a specific window, e.g., Ctrl + B → 1) + +### Step 4: Run ServerlessLLM on the JobNode +First find ports that are already occupied. Then pick your favourite number from the remaining ports to replace the following placeholder ``. For example: `6379` + +It should also be said that certain slurm system is a bit slow, **so please be patient and wait for the system to output**. + +In the first window, start a local ray cluster with 1 head node and 1 worker node: +```shell +source /opt/conda/bin/activate +conda activate sllm +ray start --head --port= --num-cpus=4 --num-gpus=0 --resources='{"control_node": 1}' --block +``` +In the second window, start the worker node: +```shell +source /opt/conda/bin/activate +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 +ray start --address=0.0.0.0: --num-cpus=4 --num-gpus=1 --resources='{"worker_node": 1, "worker_id_0": 1}' --block +``` +In the third window, start ServerlessLLM Store server: +```shell +source /opt/conda/bin/activate +conda activate sllm-worker +export CUDA_VISIBLE_DEVICES=0 +sllm-store start +``` +In the 4th window, start ServerlessLLM Serve: +```shell +source /opt/conda/bin/activate +conda activate sllm +sllm-serve start +``` +Everything is set! + + +In the 5th window, let's deploy a model to the ServerlessLLM server. You can deploy a model by running the following command: +```shell +source /opt/conda/bin/activate +conda activate sllm +sllm deploy --model facebook/opt-1.3b --backend transformers +``` +This will download the model from HuggingFace transformers. After deploying, you can query the model by any OpenAI API client. For example, you can use the following Python code to query the model: +```shell +curl http://127.0.0.1:8343/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` +Expected output: +```shell +{"id":"chatcmpl-9f812a40-6b96-4ef9-8584-0b8149892cb9","object":"chat.completion","created":1720021153,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}} +``` + +### Step 5: Clean up +To delete a deployed model, use the following command: +```shell +sllm delete facebook/opt-1.3b +``` +This will remove the specified model from the ServerlessLLM server. + +In each window, use `Ctrl + c` to stop server and `exit` to exit current `tmux` session. + +--- +## SBATCH +### Step 1: Start the Head Node +Since the head node does not require a gpu, you can find a low-computing capacity node to deploy the head node. +1. **Activate the `sllm` environment and start the head node:** + + Here is the example script, named `start_head_node.sh`. + ```shell + #!/bin/bash + #SBATCH --partition=your-partition # Specify the partition + #SBATCH --nodelist=JobNode01 # Specify an idle node + #SBATCH --job-name=ray-head + #SBATCH --output=sllm_head.out + #SBATCH --error=sllm_head.err + #SBATCH --nodes=1 + #SBATCH --ntasks=1 + #SBATCH --cpus-per-task=12 + #SBATCH --gpus-per-task=0 + + cd /path/to/ServerlessLLM + + source /opt/conda/bin/activate # make sure conda will be loaded correctly + conda activate sllm + + ray start --head --port=6379 --num-cpus=12 --num-gpus=0 --resources='{"control_node": 1}' --block + ``` + - Replace `your-partition`, `JobNode01` and `/path/to/ServerlessLLM` + +2. **Submit the script** + + Use ```sbatch start_head_node.sh``` to submit the script to certain idle node. + +3. **Expected output** + + In `sllm_head.out`, you will see the following output: + + ```shell + Local node IP: + -------------------- + Ray runtime started. + -------------------- + ``` + **Remember the IP address**, denoted ``````, you will need it in following steps. + +4. **Find an available port for serve** + - Some HPCs have a firewall that blocks port 8343. You can use `nc -zv 8343` to check if the port is accessible. + - If it is not accessible, find an available port and replace `available_port` in the following script. + - Here is an example script, named `find_port.sh` + + ```shell + #!/bin/bash + #SBATCH --partition=your-partition + #SBATCH --nodelist=JobNode01 + #SBATCH --job-name=find_port + #SBATCH --output=find_port.log + #SBATCH --time=00:05:00 + #SBATCH --mem=1G + + echo "Finding available port on $(hostname)" + + python -c " + import socket + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) + print(f'Available port: {s.getsockname()[1]}') + " + ``` + Use `sbatch find_port.sh` to submit the script to JobNode01, and in `find_port.log`, you will see the following output: + ``` + Finding available port on JobNode01 + Available port: + ``` + Remember this ``, you will use it in Step 4 + +### Step 2: Start the Worker Node & Store +We will start the worker node and store in the same script. Because the server loads the model weights onto the GPU and uses shared GPU memory to pass the pointer to the client. If you submit another script with ```#SBATCH --gpres=gpu:1```, it will be possibly set to use a different GPU, as specified by different ```CUDA_VISIBLE_DEVICES``` settings. Thus, they cannot pass the model weights. +1. **Activate the ```sllm-worker``` environment and start the worker node.** + + Here is the example script, named```start_worker_node.sh```. + ```shell + #!/bin/sh + #SBATCH --partition=your_partition + #SBATCH --nodelist=JobNode02 + #SBATCH --gres=gpu:a6000:1 # Specify device on JobNode02 + #SBATCH --job-name=sllm-worker-store + #SBATCH --output=sllm_worker.out + #SBATCH --error=sllm_worker.err + #SBATCH --gres=gpu:1 # Request 1 GPU + #SBATCH --cpus-per-task=4 # Request 4 CPU cores + #SBATCH --mem=16G # Request 16GB of RAM + + cd /path/to/ServerlessLLM + + conda activate sllm-worker + + HEAD_NODE_IP= + + export CUDA_HOME=/opt/cuda-12.5.0 # replace with your CUDA path + export PATH=$CUDA_HOME/bin:$PATH + export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH + + ray start --address=$HEAD_NODE_IP:6379 --num-cpus=4 --num-gpus=1 \ + --resources='{"worker_node": 1, "worker_id_0": 1}' --block & + + sllm-store start & + + wait + ``` + - Read the HPC's documentation to find out which partition you can use. Replace ```your_partition``` in the script with that partition name. + - Replace ```/path/to/ServerlessLLM``` with the path to the ServerlessLLM installation directory. + - Replace `````` with the IP address of the head node. + - Replace ```/opt/cuda-12.5.0``` with the path to your CUDA path. + +2. **Find the CUDA path** + - Some slurm-based HPCs have a module system, you can use ```module avail cuda``` to find the CUDA module. + - If it does not work, read the HPC's documentation carefully to find the CUDA path. For example, my doc said CUDA is in ```\opt```. Then you can use ```srun``` command to start an interactive session on the node, such as ```srun --pty -t 00:30:00 -p your_partition --gres=gpu:1 /bin/bash```. A pseudo-terminal will be started for you to find the path. + - Find it and replace ```/opt/cuda-12.5.0``` with the path to your CUDA path. +3. **Submit the script on the other node** + + Use ```sbatch start_worker_node.sh``` to submit the script to certain idle node (here we assume it is ```JobNode02```). In addition, We recommend that you place the head and worker on different nodes so that the Serve can start smoothly later, rather than queuing up for resource allocation. +4. **Expected output** + + In ```sllm_worker.out```, you will see the following output: + + - The worker node expected output: + ```shell + Local node IP: xxx.xxx.xx.xx + -------------------- + Ray runtime started. + -------------------- + ``` + - The store expected output: + ```shell + I20241030 11:52:54.719007 1321560 checkpoint_store.cpp:41] Number of GPUs: 1 + I20241030 11:52:54.773468 1321560 checkpoint_store.cpp:43] I/O threads: 4, chunk size: 32MB + I20241030 11:52:54.773548 1321560 checkpoint_store.cpp:45] Storage path: "./models/" + I20241030 11:52:55.060559 1321560 checkpoint_store.cpp:71] GPU 0 UUID: 52b01995-4fa9-c8c3-a2f2-a1fda7e46cb2 + I20241030 11:52:55.060798 1321560 pinned_memory_pool.cpp:29] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes + I20241030 11:52:57.258795 1321560 checkpoint_store.cpp:83] Memory pool created with 4GB + I20241030 11:52:57.262835 1321560 server.cpp:306] Server listening on 0.0.0.0:8073 + ``` +### Step 3: Start the Serve on the Head Node +1. **Activate the ```sllm``` environment and start the serve.** + + Here is the example script, named```start_serve.sh```. + ```shell + #!/bin/sh + #SBATCH --partition=your_partition + #SBATCH --nodelist=JobNode01 # This node should be the same as head + #SBATCH --output=serve.log + + cd /path/to/ServerlessLLM + + conda activate sllm + + sllm start --host + # sllm start --host --port # if you have changed the port + ``` + - Replace `your_partition` in the script as before. + - Replace `/path/to/ServerlessLLM` as before. + - Replace `` you have found in Step 1 (if port 8343 is not available). +2. **Submit the script on the head node** + + Use ```sbatch start_serve.sh``` to submit the script to the head node (```JobNode01```). + +3. **Expected output** + ```shell + -- Connecting to existing Ray cluster at address: xxx.xxx.xx.xx:6379... + -- Connected to Ray cluster. + INFO: Started server process [1339357] + INFO: Waiting for application startup. + INFO: Application startup complete. + INFO: Uvicorn running on http://xxx.xxx.xx.xx:8343 (Press CTRL+C to quit) + ``` +### Step 4: Use sllm to manage models +1. **You can do this step on login node, and set the ```LLM_SERVER_URL``` environment variable:** + ```shell + $ conda activate sllm + (sllm)$ export LLM_SERVER_URL=http://:8343 + ``` + - Replace `` with the actual IP address of the head node. + - Replace ```8343``` with the actual port number (`` in Step1) if you have changed it. +2. **Deploy a Model Using ```sllm```** + ```shell + (sllm)$ sllm deploy --model facebook/opt-1.3b + ``` +### Step 5: Query the Model Using OpenAI API Client + **You can use the following command to query the model:** + ```shell + curl $LLM_SERVER_URL/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' + ``` + - Replace `````` with the actual IP address of the head node. + - Replace ```8343``` with the actual port number (`` in Step 1) if you have changed it. +### Step 6: Stop Jobs +On the SLURM cluster, we usually use the ```scancel``` command to stop the job. Firstly, list all jobs you have submitted (replace ```your_username``` with your username): +```shell +$ squeue -u your_username +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 1234 compute sllm-head your_username R 0:01 1 JobNode01 + 1235 compute sllm-worker-store your_username R 0:01 1 JobNode02 + 1236 compute sllm-serve your_username R 0:01 1 JobNode01 +``` +Then, use ```scancel``` to stop the job (```1234```, ```1235``` and ```1236``` are JOBIDs): +```shell +$ scancel 1234 1235 1236 +``` diff --git a/versioned_docs/version-0.8.0/developer/_category_.json b/versioned_docs/version-0.8.0/developer/_category_.json new file mode 100644 index 0000000..89a7abc --- /dev/null +++ b/versioned_docs/version-0.8.0/developer/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Developer Guide", + "position": 6 +} diff --git a/versioned_docs/version-0.8.0/developer/supporting_a_new_hardware.md b/versioned_docs/version-0.8.0/developer/supporting_a_new_hardware.md new file mode 100644 index 0000000..2dc4f0a --- /dev/null +++ b/versioned_docs/version-0.8.0/developer/supporting_a_new_hardware.md @@ -0,0 +1,25 @@ +--- +sidebar_position: 0 +--- + +# Supporting a New Hardware + +ServerlessLLM actively expands support for new hardware configurations to meet diverse deployment needs. + +## Support Standards +Hardware is considered supported by ServerlessLLM if: +1. Any of the inference backends used (e.g., Transformers, vLLM) can run model inference on the hardware. +2. ServerlessLLM Store can successfully load model checkpoints on the hardware. + +## Steps to Support a New Hardware +1. **Check Inference Backend Compatibility**: Refer to the specific inference backend documentation (e.g., for vLLM, Transformers) for hardware support. +2. **ServerlessLLM Store Configuration**: + - If the hardware provides CUDA-compatible APIs (e.g., ROCm), adjust the build script (`CMakeLists.txt`) by adding necessary compiler flags. + - For non-CUDA-compatible APIs, implementing a custom checkpoint loading function might be required. + +## Verifying Hardware Support in ServerlessLLM Store +The hardware support is verified if it successfully completes the [Quick Start Guide](https://serverlessllm.github.io/docs/getting_started/) examples, showcasing checkpoint loading and inference functionality without errors. + +If the hardware is not publicly available (i.e., can't be tested by the ServerlessLLM team), a screenshot or output log of the successful execution of the Quick Start Guide examples is required to verify hardware support. + +If you encounter any issues or have questions, please reach out to the ServerlessLLM team by raising an issue on the [GitHub repository](https://github.com/ServerlessLLM/ServerlessLLM/issues). \ No newline at end of file diff --git a/versioned_docs/version-0.8.0/features/_category_.json b/versioned_docs/version-0.8.0/features/_category_.json new file mode 100644 index 0000000..56e0bf0 --- /dev/null +++ b/versioned_docs/version-0.8.0/features/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Features", + "position": 2 +} \ No newline at end of file diff --git a/versioned_docs/version-0.8.0/features/live_migration.md b/versioned_docs/version-0.8.0/features/live_migration.md new file mode 100644 index 0000000..2db4b1d --- /dev/null +++ b/versioned_docs/version-0.8.0/features/live_migration.md @@ -0,0 +1,211 @@ +--- +sidebar_position: 1 +--- + +# Live Migration of Inference Instances + +This example illustrates the live migration of inference instances in a ServerlessLLM cluster by constructing a scenario where two models are deployed to the cluster. Model `Qwen2.5-3B` is stored on both nodes, while model `Qwen2.5-1.5B` is only stored on node 0 (e.g., due to being less popular). This example will show a locality-contention scenario where `Qwen2.5-3B` is being served on node 0 but `Qwen2.5-1.5B` is requested to be served on the same node for optimal locality. We will find that: + +- **Without migration**, `Qwen2.5-1.5B` would have to wait for the completion of the ongoing inference instance of `Qwen2.5-3B` on node 0. +- **With live migration**, the ongoing inference instance of `Qwen2.5-3B` is migrated to node 1, and `Qwen2.5-1.5B` is allocated to node 0, thus can be served immediately. + +## Prerequisites + +To run this example, we will use Docker Compose to set up a ServerlessLLM cluster. Before proceeding, please ensure you have read the [Quickstart Guide](../getting_started.md). + +**Requirements:** + +- **Two GPUs** are required to illustrate the live migration of inference instances. +- **At least 20 GB of host memory** (this can be adjusted by using smaller models). +- **ServerlessLLM version 0.6**: Ensure you have `sllm==0.6` and `sllm-store==0.6` installed. + +## Usage + +Start a local Docker-based ray cluster using Docker Compose. + +### Clone the ServerlessLLM Repository + +If you haven't already, clone the ServerlessLLM repository: + +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/examples/live_migration +``` + +### Configure the Model Directory + +Create a directory on your host machine where models will be stored, and set the MODEL_FOLDER environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +The Docker Compose configuration is already located in the `examples/live_migration` directory. + +## Test ServerlessLLM Without Live Migration + +1. **Start the ServerlessLLM Services Using Docker Compose** + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +2. **Deploy Models with the Placement Spec Files** + +Activate the ServerlessLLM environment and set the server URL: +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` + +Deploy the models: +```bash +sllm deploy --config config-qwen-1.5b.json +sllm deploy --config config-qwen-3b.json +``` + +3. **Verify the Deployment** + +Start two inference requests in parallel. The first request is for `Qwen2.5-3B`, and the second request, sent shortly after, is for `Qwen2.5-1.5B`. The `sleep` command is used to introduce a short interval between the two requests: + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-3B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Could you share a story of the history of Computer Science?"} + ], + "max_tokens": 1024 + }' & + +sleep 3 + +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-1.5B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ], + "max_tokens": 64 + }' +``` + +Since `Qwen2.5-3B` is requested first, `Qwen2.5-1.5B` must wait for the ongoing inference instance of `Qwen2.5-3B` to complete on node 0 before it can start processing. + + +4. Clean up. + +```bash +docker compose down +``` + +## Test ServerlessLLM With Live Migration + +1. **Start the ServerlessLLM Services with Live Migration Enabled** + +Use the following command to start the ServerlessLLM services with live migration enabled. This configuration includes the `enable-migration.yml` file: + +```bash +docker compose -f docker-compose.yml -f enable-migration.yml up -d +``` + +This command will start the Ray head node and two worker nodes, enabling the live migration feature. + +2. **Deploy Models with the Placement Spec Files** + +Activate the ServerlessLLM environment and set the server URL: + +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` + +Deploy the models: + +```bash +sllm deploy --config config-qwen-1.5b.json +sllm deploy --config config-qwen-3b.json +``` + +3. **Verify the Deployment** + +Start two inference requests in parallel. The first request is for `Qwen2.5-3B`, and the second request, sent shortly after, is for `Qwen2.5-1.5B`. The `sleep` command is used to introduce a short interval between the two requests: + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-3B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Could you share a story of the history of Computer Science?"} + ], + "max_tokens": 1024 + }' & + +sleep 3 + +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "Qwen/Qwen2.5-1.5B-Instruct", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ], + "max_tokens": 64 + }' +``` + +According to the response, you should observe that `Qwen2.5-1.5B` completes ahead of `Qwen2.5-3B`. This is because the ongoing inference instance of `Qwen2.5-3B` is live-migrated from node 0 to node 1, allowing `Qwen2.5-1.5B` to be served immediately on node 0. + +As shown in the log message, the ongoing inference instance of the model `Qwen/Qwen2.5-3B-Instruct` is live-migrated from node 0 to node 1. And model `Qwen/Qwen2.5-1.5B-Instruct` is allocated to node 0. + +```bash +(MigrationRouter pid=1724) INFO 12-10 22:05:02 migration_router.py:106] Executing migration plan: MigrationPlan(target_node_id='1', source_instance=InstanceStatus(instance_id='Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab', node_id='0', num_gpu=1, concurrency=0, model_name='Qwen/Qwen2.5-3B-Instruct', num_current_tokens=0)) +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:164] Initialized backend for instance Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 for model Qwen/Qwen2.5-3B-Instruct +# Start multi-round live migration +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:178] Migration iteration 0 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:183] Number of tokens: 353, delta: 353 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:198] Migration iteration 0 completed +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:178] Migration iteration 1 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:183] Number of tokens: 14, delta: 14 +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:188] Migration completed: remained 14 tokens +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:201] Migrated instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab to Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 +# Finish multi-round live migration +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:215] Instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab removed +(MigrationRouter pid=1724) DEBUG 12-10 22:05:13 migration_router.py:77] Preempted request: ... +# Resume the instance on target node +(MigrationRouter pid=1724) INFO 12-10 22:05:13 migration_router.py:83] Resuming request on target instance: Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 +# Qwen/Qwen2.5-1.5B is allocated to node 0 +(StoreManager pid=1459) INFO 12-10 22:05:14 store_manager.py:344] Loading Qwen/Qwen2.5-1.5B-Instruct to node 0 +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:92] Deallocating model Qwen/Qwen2.5-3B-Instruct instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:103] Node 0 deallocated 1 GPUs +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 fcfs_scheduler.py:108] Model Qwen/Qwen2.5-3B-Instruct instance Qwen/Qwen2.5-3B-Instruct_dedb945f-74e5-403f-8677-35965453abab deallocated +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 storage_aware_scheduler.py:188] Migrated instance Qwen/Qwen2.5-3B-Instruct to node 1 instance Qwen/Qwen2.5-3B-Instruct_2c9ef57f-c432-45d6-a4a9-1bae9c792853 +(StorageAwareScheduler pid=1574) INFO 12-10 22:05:14 storage_aware_scheduler.py:195] Allocated node 0 for model Qwen/Qwen2.5-1.5B-Instruct +``` + +4. Clean up. + +```bash +docker compose down +``` + + diff --git a/versioned_docs/version-0.8.0/features/peft_lora_fine_tuning.md b/versioned_docs/version-0.8.0/features/peft_lora_fine_tuning.md new file mode 100644 index 0000000..af07fdb --- /dev/null +++ b/versioned_docs/version-0.8.0/features/peft_lora_fine_tuning.md @@ -0,0 +1,328 @@ +--- +sidebar_position: 4 +--- +# PEFT LoRA Fine-tuning + +This feature introduces a dedicated fine-tuning backend (`ft_backend`) for handling LoRA (Low-Rank Adaptation) fine-tuning jobs in ServerlessLLM. This implementation provides isolated fine-tuning instances with specialized resource management and lifecycle control. + +## Prerequisites + +Before using the fine-tuning feature, ensure you have: + +1. **Base Model**: A base model must be saved using the transformers backend +2. **Docker Setup**: ServerlessLLM cluster running via Docker Compose +3. **Storage**: Adequate storage space for fine-tuned adapters + +## Usage + +### Step 1. **Start the ServerlessLLM Services Using Docker Compose** + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +### Step 2: Submit Fine-tuning Job + +Submit a fine-tuning job using the REST API: + +```bash +curl -X POST $LLM_SERVER_URL/v1/fine-tuning/jobs \ + -H "Content-Type: application/json" \ + -d @examples/fine_tuning/fine_tuning_config.json +``` + +#### Fine-tuning Configuration + +Create a configuration file (`fine_tuning_config.json`) with the following structure: + +```json +{ + "model": "facebook/opt-125m", + "ft_backend": "peft_lora", + "num_gpus": 1, + "num_cpus": 1, + "timeout": 3600, + "backend_config": { + "output_dir": "facebook/adapters/opt-125m_adapter_test", + "dataset_config": { + "dataset_source": "hf_hub", + "hf_dataset_name": "fka/awesome-chatgpt-prompts", + "tokenization_field": "prompt", + "split": "train", + "data_files": "", + "extension_type": "" + }, + "lora_config": { + "r": 4, + "lora_alpha": 32, + "lora_dropout": 0.05, + "bias": "none", + "task_type": "CAUSAL_LM" + }, + "training_config": { + "auto_find_batch_size": true, + "save_strategy": "no", + "num_train_epochs": 2, + "learning_rate": 0.0001, + "use_cpu": false + } + } +} +``` + +#### Configuration Parameters + +**Job Configuration:** +- `model`: Base model name +- `ft_backend`: Fine-tuning backend type (currently supports "peft_lora") +- `num_cpus`: Number of CPU cores required +- `num_gpus`: Number of GPUs required +- `timeout`: Maximum execution time in seconds + +**Dataset Configuration:** +- `dataset_source`: Source type ("hf_hub" or "local") +- `hf_dataset_name`: HuggingFace dataset name (for hf_hub) +- `data_files`: Local file paths (for local) +- `extension_type`: File extension type (for local) +- `tokenization_field`: Field name for tokenization +- `split`: Dataset split to use +- More dataset config parameters could be found in [huggingface datasets documentation](https://huggingface.co/docs/datasets/en/loading#load) + +**LoRA Configuration:** +- `r`: LoRA rank +- `lora_alpha`: LoRA alpha parameter +- `target_modules`: Target modules for LoRA adaptation +- `lora_dropout`: Dropout rate +- `bias`: Bias handling strategy +- `task_type`: Task type for PEFT +- More LoraConfig parameters could be found in [huggingface documentation](https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraConfig) + +**Training Configuration:** +- `num_train_epochs`: Number of training epochs +- `per_device_train_batch_size`: Batch size per device +- `gradient_accumulation_steps`: Gradient accumulation steps +- `learning_rate`: Learning rate +- `warmup_steps`: Number of warmup steps +- `logging_steps`: Logging frequency +- `save_steps`: Model saving frequency +- `eval_steps`: Evaluation frequency +- More training arguments could be found in [huggingface documentation](https://huggingface.co/docs/transformers/v4.53.3/en/main_classes/trainer#transformers.TrainingArguments) + +### Step 3: Expected Response + +Upon successful job submission, you'll receive a response with the job ID: + +```json +{ + "job_id": "job-123" +} +``` + +### Step 4: Monitor Job Status + +Check the status of your fine-tuning job: + +```bash +curl -X GET "$LLM_SERVER_URL/v1/fine_tuning/jobs/job-123" +``` + +#### Status Response + +```json +{ + "id": "job-123", + "object": "fine_tuning.job", + "status": { + "config": { + "model": "facebook/opt-125m", + "ft_backend": "peft_lora", + "num_gpus": 1, + "num_cpus": 1, + "timeout": 3600, + "backend_config": { + "output_dir": "facebook/adapters/opt-125m_adapter_test", + "dataset_config": { + "dataset_source": "hf_hub", + "hf_dataset_name": "fka/awesome-chatgpt-prompts", + "tokenization_field": "prompt", + "split": "train", + "data_files": "", + "extension_type": "" + }, + "lora_config": { + "r": 4, + "lora_alpha": 32, + "lora_dropout": 0.05, + "bias": "none", + "task_type": "CAUSAL_LM" + }, + "training_config": { + "auto_find_batch_size": true, + "save_strategy": "no", + "num_train_epochs": 2, + "learning_rate": 0.0001, + "use_cpu": false + } + } + }, + "status": "running", + "created_time": "2025-08-26T04:18:11.155785", + "updated_time": "2025-08-26T04:18:11.155791", + "priority": 0 + } +} +``` + +**Possible Status Values:** +- `pending`: Job is waiting for resources +- `running`: Job is currently executing +- `completed`: Job completed successfully +- `failed`: Job failed with an error +- `cancelled`: Job was cancelled by user + +### Step 5: Cancel Job (Optional) + +If needed, you can cancel a running job: + +```bash +curl -X POST "$LLM_SERVER_URL/v1/fine_tuning/jobs/job-123/cancel" +``` + +## Job Management + +### Resource Allocation + +Fine-tuning jobs are allocated resources based on the specified requirements: + +- **CPU**: Number of CPU cores specified in `num_cpus` +- **GPU**: Number of GPUs specified in `num_gpus` +- **Memory**: Automatically managed based on model size and batch size + +### Priority System + +Jobs are processed based on priority and creation time: + +1. **Higher Priority**: Jobs with higher priority values are processed first +2. **FIFO**: Jobs with the same priority are processed in order of creation +3. **Resource Availability**: Jobs wait until sufficient resources are available + +### Timeout Handling + +Jobs have configurable timeout limits: + +- **Default Timeout**: 3600 seconds (1 hour) +- **Configurable**: Set via `timeout` parameter in job configuration +- **Automatic Cleanup**: Jobs are automatically marked as failed if they exceed the timeout + +## Output and Storage + +### LoRA Adapter Storage + +Fine-tuned LoRA adapters are automatically saved to the `output_dir` path you config in the `fine_tuning_config.json`, like: + +``` +{STORAGE_PATH}/transformers/facebook/adapters/opt-125m_adapter_test +``` + +### Adapter Contents + +The saved adapter includes: + +- **LoRA Weights**: Fine-tuned LoRA parameters +- **Configuration**: LoRA configuration file +- **Metadata**: Training metadata and statistics + +## Integration with Serving + +### Using Fine-tuned Adapters + +After successful fine-tuning, the LoRA adapter can be used for inference: + +```bash +# Deploy model with fine-tuned adapter +sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "my_adapter=ft_facebook/opt-125m_adapter" + +# Use the adapter for inference +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "user", "content": "Hello, how are you?"} + ], + "lora_adapter_name": "my_adapter" +}' +``` + +For more details about PEFT LoRA Serving, please see the [documentation](./peft_lora_serving.md) +## Troubleshooting + +### Common Issues + +1. **Job Stuck in Pending**: Check resource availability and job priority +2. **Dataset Loading Failures**: Verify dataset configuration and accessibility +3. **Training Failures**: Check GPU memory and batch size settings +4. **Timeout Errors**: Increase timeout or optimize training configuration + +## API Reference + +### Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/v1/fine-tuning/jobs` | POST | Submit a fine-tuning job | +| `/v1/fine_tuning/jobs/{fine_tuning_job_id}` | GET | Get job status | +| `/v1/fine_tuning/jobs/{fine_tuning_job_id}/cancel` | POST | Cancel a running job | + +### Response Codes + +| Code | Description | +|------|-------------| +| 200 | Success | +| 400 | Bad Request | +| 404 | Job not found | +| 500 | Internal Server Error | + +## Examples + +### Complete Fine-tuning Workflow + +```bash +# 1. Save base model +sllm-store save --model facebook/opt-125m --backend transformers + +# 2. Start the ServerlessLLM cluster with docker compose +cd examples/docker +docker compose up -d --build + +# 3. Submit fine-tuning job +cd .. && cd .. +curl -X POST $LLM_SERVER_URL/v1/fine-tuning/jobs \ + -H "Content-Type: application/json" \ + -d @examples/fine_tuning/fine_tuning_config.json + +# 4. Monitor job status +curl -X GET "$LLM_SERVER_URL/v1/fine_tuning/jobs/job-123" + +# 5. Deploy base model with fine-tuned adapter +sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "my_adapter=ft_facebook/opt-125m_adapter" + +# 5. Use for inference +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [{"role": "user", "content": "Hello"}], + "lora_adapter_name": "my_adapter" +}' +``` diff --git a/versioned_docs/version-0.8.0/features/peft_lora_serving.md b/versioned_docs/version-0.8.0/features/peft_lora_serving.md new file mode 100644 index 0000000..4c7d3a6 --- /dev/null +++ b/versioned_docs/version-0.8.0/features/peft_lora_serving.md @@ -0,0 +1,116 @@ +--- +sidebar_position: 2 +--- +# PEFT LoRA Serving + +This example illustrates the process of deploying and serving a base large language model enhanced with LoRA (Low-Rank Adaptation) adapters in a ServerlessLLM cluster. It demonstrates how to start the cluster, deploy a base model with multiple LoRA adapters, perform inference using different adapters, and update or remove the adapters dynamically. + +## Pre-requisites + +To run this example, we will use Docker Compose to set up a ServerlessLLM cluster. Before proceeding, please ensure you have read the [Quickstart Guide](../getting_started.md). + +We will use the following example base model & LoRA adapters +- Base model: `facebook/opt-125m` +- LoRA adapters: + - `peft-internal-testing/opt-125m-dummy-lora` + - `monsterapi/opt125M_alpaca` + - `edbeeching/opt-125m-lora` + - `Hagatiana/opt-125m-lora` + +## Usage + +Start a local Docker-based ray cluster using Docker Compose. + +### Step 1: Download the Docker Compose File + +Download the `docker-compose.yml` file from the ServerlessLLM repository: +```bash +# Create a directory for the ServerlessLLM Docker setup +mkdir serverless-llm-docker && cd serverless-llm-docker + +# Download the docker-compose.yml file +curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml + +# Alternatively, you can use wget: +# wget https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml +``` + +### Step 2: Configuration + +Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +### Step 3: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +### Step 4: Deploy Models with LoRA Adapters +1. Configuration +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` +2. Deploy models with specified lora adapters. +```bash +sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "demo_lora1=peft-internal-testing/opt-125m-dummy-lora demo_lora2=monsterapi/opt125M_alpaca" +``` +3. Verify the deployment. +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ], + "lora_adapter_name": "demo_lora1" + }' +``` +If no lora adapters specified, the system will use the base model to do inference +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` +### Step 5: Update LoRA Adapters +If you wish to switch to a different set of LoRA adapters, you can still use `sllm deploy` command with updated adapter configurations. ServerlessLLM will automatically reload the new adapters without restarting the backend. +```bash +sllm deploy --model facebook/opt-125m --backend transformers --enable-lora --lora-adapters "demo-lora1=edbeeching/opt-125m-lora demo-lora2=Hagatiana/opt-125m-lora" +``` + +### Step 6: Clean Up + +Delete the lora adapters by running the following command (this command will only delete lora adapters, the base model won't be deleted): +```bash +sllm delete facebook/opt-125m --lora-adapters "demo-lora1 demo-lora2" +``` +If you need to stop and remove the containers, you can use the following commands: +```bash +docker compose down +``` \ No newline at end of file diff --git a/versioned_docs/version-0.8.0/features/quantized_models.md b/versioned_docs/version-0.8.0/features/quantized_models.md new file mode 100644 index 0000000..df09751 --- /dev/null +++ b/versioned_docs/version-0.8.0/features/quantized_models.md @@ -0,0 +1,175 @@ +--- +sidebar_position: 3 +--- + +# Quantization + +This example demonstrates the use of quantization within the ServerlessLLM framework to optimize model serving. Quantization is a technique used to reduce the memory footprint and computational requirements of a large language model by representing its weights with lower-precision data types, such as 8-bit integers (int8). This example will showcase how to deploy and serve a quantized model in a ServerlessLLM cluster. + +## Pre-requisites + +We will use Docker Compose to run a ServerlessLLM cluster in this example. Therefore, please make sure you have read the Quickstart Guide before proceeding. + +## Usage +Start a local Docker-based ray cluster using Docker Compose. + +## Step 1: Set up the Environment + +Create a directory for this example and download the `docker-compose.yml` file. + +```bash +mkdir sllm-quantization-example && cd sllm-quantization-example +curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml + +## Step 2: Configuration + +Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +## Step 3: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +## Step 4: Create Quantization and Deployment Configurations + +First, we'll generate a standard Hugging Face BitsAndBytesConfig and save it to a JSON file. Then, we'll create a deployment configuration file with these quantization settings embedded in it. + +1. Generate the Quantization Config + +Create a Python script named `get_config.py` in the current directory with the following content: +```python +# get_config.py +from transformers import BitsAndBytesConfig + +quantization_config = BitsAndBytesConfig(load_in_4bit=True) +quantization_config.to_json_file("quantization_config.json") + +``` + +Run the script to generate `quantization_config.json`: +```bash +python get_config.py +``` + + +2. Create the Deployment Config + +Now, create a file named `quantized_deploy_config.json`. This file tells ServerlessLLM which model to deploy and instructs the backend to use the quantization settings. You should copy the contents of `quantization_config.json` into the `quantization_config` field below. A template can be found in `sllm/cli/default_config.json`. + +```json +{ + "model": "facebook/opt-1.3b", + "backend": "transformers", + "num_gpus": 1, + "auto_scaling_config": { + "metric": "concurrency", + "target": 1, + "min_instances": 0, + "max_instances": 10, + "keep_alive": 0 + }, + "backend_config": { + "pretrained_model_name_or_path": "", + "device_map": "auto", + "torch_dtype": "float16", + "hf_model_class": "AutoModelForCausalLM", + "quantization_config": { + "_load_in_4bit": true, + "_load_in_8bit": false, + "bnb_4bit_compute_dtype": "float32", + "bnb_4bit_quant_storage": "uint8", + "bnb_4bit_quant_type": "fp4", + "bnb_4bit_use_double_quant": false, + "llm_int8_enable_fp32_cpu_offload": false, + "llm_int8_has_fp16_weight": false, + "llm_int8_skip_modules": null, + "llm_int8_threshold": 6.0, + "load_in_4bit": true, + "load_in_8bit": false, + "quant_method": "bitsandbytes" + } + } +} + +``` + +> Note: Quantization currently only supports the "transformers" backend. Support for other backends will come soon. + +## Step 5: Deploy the Quantized Model +With the configuration files in place, deploy the model using the `sllm-cli`. + +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 + +sllm-cli deploy --config quantized_deploy_config.json +``` + +## Step 6: Verify the deployment. +Send an inference to the server to query the model: + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +To verify the model is being loaded in the desired precision, check the logs (`docker logs sllm_head`). You should see that the model is indeed being loaded in `fp4`. + + +```log +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:321] load config takes 0.0030286312103271484 seconds +(RoundRobinRouter pid=481) INFO 07-02 20:01:49 roundrobin_router.py:272] [] +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:331] load model takes 0.2806234359741211 seconds +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:338] device_map: OrderedDict([('', 0)]) +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:345] compute_device_placement takes 0.18753838539123535 seconds +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:376] allocate_cuda_memory takes 0.0020012855529785156 seconds +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 client.py:72] load_into_gpu: transformers/facebook/opt-1.3b, 70b42a05-4faa-4eaf-bb73-512c6453e7fa +(TransformersBackend pid=352, ip=172.18.0.2) INFO 07-02 20:01:49 client.py:113] Model loaded: transformers/facebook/opt-1.3b, 70b42a05-4faa-4eaf-bb73-512c6453e7fa +(TransformersBackend pid=352, ip=172.18.0.2) INFO 07-02 20:01:49 transformers.py:398] restore state_dict takes 0.0007319450378417969 seconds +(TransformersBackend pid=352, ip=172.18.0.2) DEBUG 07-02 20:01:49 transformers.py:411] using precision: fp4 +(TransformersBackend pid=352, ip=172.18.0.2) INFO 07-02 20:01:50 client.py:117] confirm_model_loaded: transformers/facebook/opt-1.3b, 70b42a05-4faa-4eaf-bb73-512c6453e7fa +``` + +You should receive a successful JSON response from the model. + +## Step 7: Clean Up + +Delete the model deployment by running the following command: + +```bash +sllm-cli delete facebook/opt-1.3b +``` + +If you need to stop and remove the containers, you can use the following commands: + +```bash +docker compose down +``` + + diff --git a/versioned_docs/version-0.8.0/features/storage_aware_scheduling.md b/versioned_docs/version-0.8.0/features/storage_aware_scheduling.md new file mode 100644 index 0000000..81723f9 --- /dev/null +++ b/versioned_docs/version-0.8.0/features/storage_aware_scheduling.md @@ -0,0 +1,123 @@ +--- +sidebar_position: 0 +--- + +# Storage Aware Scheduling with Docker Compose + +## Pre-requisites + +We will use Docker Compose to run a ServerlessLLM cluster in this example. Therefore, please make sure you have read the [Quickstart Guide](../getting_started.md) before proceeding. + +## Usage + +Start a local Docker-based ray cluster using Docker Compose. + +### Step 1: Clone the ServerlessLLM Repository + +If you haven't already, clone the ServerlessLLM repository: + +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/examples/storage_aware_scheduling +``` + +### Step 2: Configuration + +Set the Model Directory. Create a directory on your host machine where models will be stored and set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you want to store the models. + +### Step 3: Enable Storage Aware Scheduling in Docker Compose + +The Docker Compose configuration is already located in the `examples/storage_aware_scheduling` directory. To activate storage-aware scheduling, ensure the `docker-compose.yml` file includes the necessary configurations(`sllm_head` service should include the `--enable-storage-aware` command). + +:::tip +Recommend to adjust the number of GPUs and `mem_pool_size` based on the resources available on your machine. +::: + + +### Step 4: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and two worker nodes defined in the `docker-compose.yml` file. + +:::tip +Use the following command to monitor the logs of the head node: + +```bash +docker logs -f sllm_head +``` +::: + +### Step 5: Deploy Models with Placement Spec + +In the `examples/storage_aware_scheduling` directory, the example configuration files (`config-opt-2.7b.json` and `config-opt-1.3b.json`) are already given. + +> Note: Storage aware scheduling currently only supports the "transformers" backend. Support for other backends will come soon. + +2. Deploy models with the placement spec files. + +```bash +conda activate sllm +export LLM_SERVER_URL=http://127.0.0.1:8343 + +sllm deploy --config config-opt-2.7b.json +sllm deploy --config config-opt-1.3b.json +``` + +3. Verify the deployment. + +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-2.7b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' + +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +As shown in the log message, the model "facebook/opt-2.7b" is scheduled on server 0, while the model "facebook/opt-1.3b" is scheduled on server 1. + +```log +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:27 storage_aware_scheduler.py:137] Sorted scheduling options: [('0', 4.583079601378258)] +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:27 storage_aware_scheduler.py:144] Allocated node 0 for model facebook/opt-2.7b +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:38 storage_aware_scheduler.py:137] Sorted scheduling options: [('1', 2.266678696047572)] +(StorageAwareScheduler pid=1543) INFO 11-12 23:48:38 storage_aware_scheduler.py:144] Allocated node 1 for model facebook/opt-1.3b +``` + +### Step 6: Clean Up + +Delete the model deployment by running the following command: + +```bash +sllm delete facebook/opt-1.3b facebook/opt-2.7b +``` + +If you need to stop and remove the containers, you can use the following commands: + +```bash +docker compose down +``` + diff --git a/versioned_docs/version-0.8.0/getting_started.md b/versioned_docs/version-0.8.0/getting_started.md new file mode 100644 index 0000000..f98364d --- /dev/null +++ b/versioned_docs/version-0.8.0/getting_started.md @@ -0,0 +1,136 @@ +--- +sidebar_position: 1 +--- + +# Getting Started + +This guide demonstrates how to quickly set up a local ServerlessLLM cluster using Docker Compose on a single machine. We will initialize a minimal cluster, consisting of a head node and a single worker node. Then, we'll deploy a model using the `sllm` and query the deployment through an OpenAI-compatible API. + +:::note +We strongly recommend using Docker (Compose) to manage your ServerlessLLM cluster, whether you are using ServerlessLLM for testing or development. However, if Docker is not a viable option for you, please refer to the [deploy from scratch guide](./deployment/single_machine.md). +::: + +## Prerequisites + +Before you begin, ensure you have the following installed and configured: + +1. **Docker**: Installed on your system. You can download it from [here](https://docs.docker.com/get-docker/). +2. **ServerlessLLM CLI**: Installed on your system. Install it using `pip install serverless-llm`. +3. **GPUs**: At least one NVIDIA GPU is required. If you have multiple GPUs, you can adjust the `docker-compose.yml` file accordingly. +4. **NVIDIA Docker Toolkit**: This enables Docker to utilize NVIDIA GPUs. Follow the installation guide [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). + +## Start the ServerlessLLM Cluster + +We will use Docker Compose to simplify the ServerlessLLM setup process. + +### Step 1: Download the Docker Compose File + +Download the `docker-compose.yml` file from the ServerlessLLM repository: + +```bash +# Create a directory for the ServerlessLLM Docker setup +mkdir serverless-llm-docker && cd serverless-llm-docker + +# Download the docker-compose.yml file +curl -O https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml + +# Alternatively, you can use wget: +# wget https://raw.githubusercontent.com/ServerlessLLM/ServerlessLLM/main/examples/docker/docker-compose.yml +``` + +### Step 2: Configuration + +Create a directory on your host machine to store models. Then, set the `MODEL_FOLDER` environment variable to point to this directory: + +```bash +export MODEL_FOLDER=/path/to/your/models +``` + +Replace `/path/to/your/models` with the actual path where you intend to store the models. This directory will be mounted into the Docker containers. + +### Step 3: Start the Services + +Start the ServerlessLLM services using Docker Compose: + +```bash +docker compose up -d +``` + +This command will start the Ray head node and a worker node as defined in the `docker-compose.yml` file. + +Verify that the services are ready: + +```bash +docker logs sllm_head +``` + +Ensure the services are ready before proceeding. You should see output similar to the following: + +```bash +... +(SllmController pid=1435) INFO 05-26 15:40:49 controller.py:68] Starting scheduler +INFO: Started server process [1] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:8343 (Press CTRL+C to quit) +(FcfsScheduler pid=1604) INFO 05-26 15:40:49 fcfs_scheduler.py:54] Starting FCFS scheduler +(FcfsScheduler pid=1604) INFO 05-26 15:40:49 fcfs_scheduler.py:111] Starting control loop +``` + +## Deploy a Model Using sllm + +Set the `LLM_SERVER_URL` environment variable: + +```bash +export LLM_SERVER_URL=http://127.0.0.1:8343 +``` + +Deploy a model to the ServerlessLLM cluster using the `sllm`: + +```bash +sllm deploy --model facebook/opt-1.3b +``` +> Note: This command will take some time to download the model from the Hugging Face Model Hub. +> You can use any model from the [Hugging Face Model Hub](https://huggingface.co/models) by specifying its name in the `--model` argument. + +Expected output: + +```plaintext +INFO 08-01 07:38:12 deploy.py:36] Deploying model facebook/opt-1.3b with default configuration. +INFO 08-01 07:39:00 deploy.py:49] Model registered successfully. +``` + +## Query the Model + +You can now query the model using any OpenAI API client. For example, use the following `curl` command: +```bash +curl $LLM_SERVER_URL/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "model": "facebook/opt-1.3b", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} + ] + }' +``` + +Expected output: + +```plaintext +{"id":"chatcmpl-8b4773e9-a98b-41db-8163-018ed3dc65e2","object":"chat.completion","created":1720183759,"model":"facebook/opt-1.3b","choices":[{"index":0,"message":{"role":"assistant","content":"system: You are a helpful assistant.\nuser: What is your name?\nsystem: I am a helpful assistant.\n"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":16,"completion_tokens":26,"total_tokens":42}}% +``` + +## Clean Up +To delete a deployed model, execute the following command: + +```bash +sllm delete facebook/opt-1.3b +``` + +This command removes the specified model from the ServerlessLLM server. + +To stop the ServerlessLLM services, use the following command: +```bash +docker compose down +``` \ No newline at end of file diff --git a/versioned_docs/version-0.8.0/images/favicon.ico b/versioned_docs/version-0.8.0/images/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..4ef51a7c4d8cb55f59100813fb8ffa9c88795d74 GIT binary patch literal 46169 zcmc$Gg;SQz_cz@g(j_I`_0S+CjocvJ4bt5u(j_S&As{J@G)Q+N-QC^rTfX!D53i2n zJ!7!fUc2Xf>Hq}=4Sv48!qY$HjkLYr;@6}%a^z|1K0{BhW zsmL4(Dga9Ey|}u^{9&eR2I>n*Txc763L#Hs`ZFEUStE_||2sIw98$7B62 z^C+HnWkmn~f9yRAcBw)fi>+W)_<4a7CE98&(wSKxla`2*R5(+(yJ5${8lHWHrHb0%y)EK$4p>Up9s=F7pZCe+g| z@kVPn?RXdtFL7K5ykh(0IJ_s6(r#_@Q+^p z#uYAx$F&!IK;RCHM)BZ{!|Z^O;=rQ{m#D!PVqEQy-#{lDWe5@b+V_GT5He0_5@&`7 zi6);x@j=5{ydCP#`to9R$rMn*J~VkEO-O_EPP!c?KNr~pBQER*mN?aze!HC|Q$r{} z48~dLBBeY{sv?aJZlA3DTQ;uf+(onx=Ufc^qnpFZeCKnGdW5D#hPb~sOh=VP@A2C; zaSA3pq0-)#E=!~uS$1Z|nl~N;K9F{uDp>&m}n=O1nUNHuOi$ z-X2^hfw!A-Z@Bh;$!(hEtEDEEmuIq6LA-T1sSUIEo=jfH|k7}BNtwzC2S8{5sUJIRbFaXet!m`=F+gWLN) z#7#3!cE(MCJ@6$GvW&aaQv)gysZLOgq}GDBbk5)<#+^%{ud@%W_QSsXZ;fLa`@sQR zP*DhJi$mN_oNDj-lx8)~gzR|O2IT1JxuP%enB>BdQ=`7L8YqiV5_ayLjoZ3Y4|4vM zcMA8q!G!SsHoovP)q+t83>Et!3cEEW)&_HHz0PbHkXw|&<4WWI($50c5@)N}ZYtC$ zLyi{e==oT~FEqs?SX6AEQ(BjHo=(ST{r#6!)#ShRb?^F&(b!hE_~)kXgBumdN<=9` zh|{Wbz*x`oX7)$uCzMsDkgpeP&<${L(FkcJ+ut~i4fQMhC{P$0eEqaD$oIczs!!Le z=r1T}DdHx%vP^N;Ojx>axICQR1&83#R+Wt&j-+#{iDvf_7_Ob$h2Zm8KeG|w zn5ng$oO}}p%ilMGW0%AaN_bS*Rv;m}Fnd?JdIoRl>Sbl=sIp{OSXvN%J`dKcQ)E>$ zIkw(~jeq_-h*4ckB0(&HCckfmT1gnHGh*PfQj(A>U**Mir-PCASR_cC>m|5dLhi?9 z;d9o1;8qworONM6K^JeI%EPh3o~uLHn_S=|BdV@UWi$ z1=7(Tga#h_r~izp`^8x?nU@FlCm7`EbgHq~kMLms@j{K}GvEExVK2+Jg+5m$f{&7s z-54Kwl&C2caqeAVEFL&b5HeRJEcVyXn(f>nxiMxZ&Sg4x3})oEF;Ox4UR&w6W8_9w zE!6v)q;K%U zQGugg-1O2^!9#l0{tZCo8 z1!GGIJXVh4lPeTG30OW&C->E78YyVG4_N8Q{^?^99v{b_?e-1?Lb3j-mFoKRcsx~$ zu@41pEg*$-I^ue%Q(+Y+qra2KK?qK!`Ic@>JDL2!Q6IqsVXMkhnJ9AhB>c-|KvAtK zo{63M6mPZ5U;TN={ZE)S1V1QJxJu$r#*CTus$$Xa+e}VPa}1z;I@ptzV?v;56eJ*s z$a4STHV`WAUaS)GZdg8?$x(ku<*s`FVr!tqVpB&J<+WjA$V=2;MVpwC))VlkP*q*k zH!$#4R4@}whnS&XLzPh*nI`o%+7cN!2yhnwl)wNVP3_<Agt>AEH@)F`)O(C;% z5!k{6K^+nb=9T%Mwc{A(h8~-v!|md$A6I=%_L|YvWZlByLQlUudj;Z#`@uT|Vatth z1QJqXY37FRN#pi<&f>yRss}Q{;TUof$yU-OB*qT}_^@=l%Q|ArW{N^yqI(HeCu^Hj z)C0%I^gXD$t=9$w5{cBl!HQKgg)9)nyi4rk=$FPB#-S*i7cZH_#eL9(bZn~9Di8~Z zz1WV%6Ki~)Q>O3~8HyJmLSJO)kXG1VNMQ7}%;KydRSAAd3#Nwy+XF}GI3Xm#DH~+E z>at3=>O)yyv7<%}!zLg1Emh1fIXVZ{DUH23)$wm6p>C?As?vXZR&t+)dX!2Y>zCGOFZa~-U7@nAu!AM>)V=)$Jdw4r@gn~73zf2Igd>0L+F^1>`so^8%rb?VywEtwj z2@^EmLJ(`;OZ*aVCJ)_Cp;qbL$-!OuCQ(5;BCXrzx@t#OkvqRIS``VU+7482(;=(T ztp5@)vu~ldY87o44EpDf!u~asL`g!>`Iki0so48)T7JNAK}iNa1TF+fwfif4+%6Me zCHMl@d60b{ly#^{0CtqWUFAPaSW5MYP-a&6?VQjPFS5GH5PkVoBJGU7QD>6;HUqK+ zF+}fGKmUi^g5@!+W{F#RZ%$897NOjnXvxuwRPmRjheFNejsV71PC6>Va4q8KNYX3* z48>fDaA0gbtCT4z51C!E4U^tH|kD)@@X|?CIQvR^pCa zrHGF+F3Q&3+#R!rCZz?3MeHkF+=#RT1$&r&&k+P71tqZeYbb*~glSu0`;Z~;U0aclnfq!uP0Z zfd_#XVqPR36WpH57-Hyl$Mp#JL4NRKGO2WsfyE zP!Lr)c9Jyis*vxjSsjaWxP);jL^oS*4b8U8IwR|@O(Xn-1b;$3M0m*8n{E9{dE#uC1JnMbp zhSTNVqEh?tGZrK&7k^1j7{(AWNUei$ZufgfeLAil%(X`%lT;o@La9G{j%8%^v!nZV zh<7JpHg_IZO2j%DEmycj?Ha4th9av+v4odVeyBePz@3r|kPC7S%+5Qy@K#t>#lx2m z3MWeZa#1^|w6Q+fo0ya}vDW~aY`NsLPmMJqMa6P|OmGtaq6n5{l2<~A?bjAv^>%ft zzG2y|POXj06Y%CDHI6NnyK(?3n^1)%ic3Kbp#QOcUd#0YpHD+9S4sG($`5lIy(}|C zlu0-gV0B_`J$_w%n2xyQ@048kk~AljglhaM`lGE?+WHTcWUv?2{LTYNtb)#O z!GqFLv&f11S50@`arHXQe2cf8(-}cU)!_2qGv7}qTVf>T{<6{E&eg8F8Vd-6Ak=sT zQloSm%q~K~gy8rxlPXQe&A5i2j&j>a2!7&tUzDR{!@)k+pUa`}{K8LyWV8SF*Ds>I zJs+y0nOB;#znIfYAZ^N0M_+$MX@+7pr%tQGL%=uZFqfq&R3?6dJG$cBkUZ=?uwL*? zjLvDHOcpwqFMO||{55VudYoD=F3q@;0O@>f;3)IkS1gAch{bv4_mRBv{*kp}>DRi~ z+hKi8`YWfZ9KLB#{*x+qH~ybk%l^vTBGRHF3jA z1;zz>@|&lutTa{EUHc34*qgIxo6&7h%;L=Ck6ni1i=~%!8q@ehm-=F0 zlWt*h8Y<5$zqL~izFs3IL-LP3XvX&qRG?$NH;ReIEBQHjLhk2A)#kOInVKZ%)tV8d z{zmu?wg;1qt`x~lqp{L^o*AlI)qLWIx*Pesor#BMiNvL!*#=1B(%PJ;&)0ed(PX5~ zpQTi-T)p^pm%3(rw@T;SzJeWnS7Oz9gbvk(uPdb*Zx#;NA*+xbDW4v9Fld^t=aC_u zCPfRxMr28(%zjK{BN-LY6cIzv(hyDp0fhSj; zs+(HL;O5GUOt^ijra1LiZXwaRD=gA=>G4>;oowM45nVI!B5o zOidddl3ngPODSV%Z5-!<=h7C$i{Y;3~4Y3YEexVZ8pLm>Xaz9|j>*Z;Z)Z&GG2 z9v|_W@AeP|Y|^3sYpEG;^-QzsS0Lm18mUAv3Z9kIR!pg0POO;G!P9q&d;V9EozdQx z7NNU)ZVjHEYs6Vzz8r{5Bwpv30n#7Ng-*SHXy7n*43B09)b9?4 z`oUv~cMaOZm5b}Jd~tL4smU8@Fp^sIUkKF|uy87zF3zn>BM|*GERU+;SX7Mt72ijE z%D~XjLyjCQ&=8Ok1ycPe9u7a8oMM#pBUTJ`-rr{S0y;O4-w!J2_^!kMGW8WtH;{s` zf@<)XPF}8>hSJ%;8GbU82_;D}7<|SJn6@=MUsczzo&O6F^?<488|ycXEuWIk3(W6= z12BY0J+=FG4fY+GUS~s&6k6ZluG6Cg60we6>7rABdy1_w*xo4~0VwQ5Bs>V5Re!iH zyAAV=zCQ+6FJ}0j9%X+ z`)sNOFDvPM>6=ZyeqUUi@Xd1VOJu>t#sb?!$zJa7rk3WXJ?Yo3BZ#4H#qmH`Lg|EI z0|Uy*_7~Iw=Z;cGyzPJMDE91l1%N!e3Sv|{4f^pOGUhecAr~&h<}%90C+jbj1w`a> z+N`U*c|7t4T#WmV#NjU|+{$Nc7!)pC!eomrnb@`t6E?polfjB2oTz(|A#>`vJ%%V^ z_4Zc~X>#t+gUR%kN%&`?s6-Q6=u%jTr^OlFvR+lVubFFRoz(cBfe*@SjTZ&y!v{O; z)+*vr#^>-s%}pL${*R+!hyQ&d3zu*ayrgbt6K{rAyY<&XepF_HGTFj-nk%ld*)B|E3%zP zz!lV^$J5}~o$dp4_*H_kU4%&6C1x7F)7G0U`R6gz;P2E+`}o{i0{zcm<7YvwU1*9~ zxMO!e+#@;b{fr3!-g6}MI`l>HmZ3COR4r6e0sxGTfKJ8fIz(Az%a9rJ5lP(47xg0L z^I984xzRx@iiFY6zl&B8=CY2gE6vxopPO9Xq*+yB%)sJb2|XSnHlOt)JO-*Lw?flv zkF;+DJorP9A4eGj+`@$u1KNHhWxF9mkQ5MLQ?dl&4g5FG37@?K2JYWv`Pgy1c>2>u zvy>~96y_G?(`fO#_YfX8I`IP7)8(4aQ`h*Z)r#Umbft5E<>6PcJuT_kw?SX5+iR@& zw|RUNe^M?iV!A2R=Jby{Nz{2R48)43`|`yZh720hbo`#}gK``eFW#rZi2t}j-aT)$ zM8VAY2^CBFZ8Wqn+T{2UIf_Dqln=J5b*|rAp6_DOo9SO;g+wgQ?kLqdWv1SLL9eC5 zLWzh7x8U+VxVhPZ$KT^K)fUgNuoef^s?=(V7@5S|nmd1{;8TA!XrR9ssX#ibd{|F4 zx!*>x<`gl4a^>?fgW7P@D8Rww_OA4Fe4k9{@)FT@-WTpGK8h4Xj{SS8uqXdY<1^_< znn+{fs*K6T3G-anBklJ&*9L?NQ4a-)6~{;1FPMhoiK3KzTuELsalX@8soRUN)wm1; z+|D;s$#!BKOj?Z&)->cnSQ=G67aENva|+{sLhr`gG`UCRA! zxPG+C;PQ1TRACU)s}7zO)gR?LVKzC5*M@Uhe>+t5(_uEL`p-#^v&p-DxZj^2sQm7_ z1eq6VQY=3vvQm!CC;3x^!%D;>+25_G!n`=6P$o>%%WGr2n24#m_sS?fulPO`6K462 z=}+_;+=p1K=qZ&9PWGF^6Y`aQwO{dI)f{H%o}nUoho|wcsy-uypd4t43q_?iF;jyS zdi2ZXSQ_fbEuS%z9}klolKkm{yKWQkB*aBApPfa@l-P*sXgJksMeC+$CO($t|D;K< zt&L00kd6HNx2)}!?_ps7UV)3;^^}$_HI?5Oicgn66Obqc)z8xO;=*c6eUm55JKgXl zMRqmr5g%j|x~y=+&ixJAGO9nw^hEtE7$1>TW?Kz~FF8e&6lR#@X|Ip__cny`Q8a<= zRP-lQ6yQUK-?(#Q!p8T9HY9$u6zwRG3}?6cli_};|M`SU^{M&@>*Z#uDIUMzZ*fJ@ zrfGrG&X$G4D368e1Qk6cc;w#)NTaPy8JL=%3v=dv@$K?svlF2=-;N3E`YJJG1$16J z@pZ_a#bRl1ucWTi_#R3J=k!9J`ezB8U#KhgE{$YRr63#woYaT}4uiB@f1ZP^ywaT> zOn8qOH!4#^%-169Zecr8zXs+OZJPf)YxzF9b1<8MZ<2EdBcA2wP1NRp4V$j5C!9oj z0(A^uiDUXqU^y5Y9(X+%g0$1BnsAG7DpT{JssT&FkiBowsbAswe&ahK?~myIK?=09 z$Hyd}lhKObM`C6)O^S)hyZgS!Q5-_gc_e?HswU->?FOBJMgoM3$KyTt$L;J3t=cQ7 z*1o|-f-rrTs1G3nO@fi)5cpu!>f19i0u%YliCwhuUE6~ZYB`k*D=Hq>r<_sXbmT|7CGO*16I;3L5Ui_;m%%E#(YU0>|DGaVb zga{kIOApenzDqqmQSW^Tf%$ql$_0G`-In*AJ)4nx^72`SZL@x*G2+mzTM(UvmHc+c z`|<2gD7T=oqMwv%|DeJKy}Rzv$w|`})nHU(jmwC6b8_!H6Tj>F`cI8UqpHZQ#nM%9 z(VU)IR)9HwI~d89{-pY=)F#}URSI9%<=*4-cR|#Eb2`!10AiD8i^6nd2ACAKs7aE7 zY21STp9~S;&TD(e{pTH^?H%AH^VR*0{q|98to`@4B9$2)!B5SfF$eihdaXIQir953 zblAT~DUfmmfS%E_gIQJ49Ao&dlz;*`hlfV5uKb8s>Io&56S`}wFc`tKZ(^sbw6ND=xe%><}p#ZSa0 zP8Zv9$pN`y@zp-VhoFw*n-@CwA0j{Q+7O&qH*GtCoBhhx4=?eeq+uz_7kjEtSm~FQ zbn%0NY)M30E^w+QJ{DCYTDpY^3i#E8Wc95q_?Cy>z4aeEzu`Ol*Q*nmY;)K_e>&Hn z@SeZ4YR3URQsb!o2pzzbz*ZJ^Vsr&U{UM@68I^_IRYNo3Ts^s}!HpfjD1oJ;KqRb! zDPTUAAW-cB2XQSs6wyUhONsgIV)!QLNGvVKfzx_eQyJOE3_swP3wc5u4b5kng^NO0 z*V1AXC$5j4m(L^(X2;A+HJYKTEkAW`Xa2HP0wJ)pzoHm#$Ncovy?hH?C>r6~;lC%> z2p-ohTRtzKq&OV}Dw$^LQE8ulH{@0O&kO6dCz3yI;}11C@#0FOwI>@-vtasDb_EU~ zko(`^MpxFOUc8htQ2%)-o(oXW%ERoMgz} zg=+8L6?mR*qW@HDnsYpgy|mKQ><1Lw(fiCWAf*-3h{|V&H4kx*WDC3r?q~-!>=ssh z=@2s%f9A93d)MI1Jrl%49INDn^o+j432g2f{(p`?ikqlRp}6Ay!Ko9W(&hPdGY3}W z!vIM>oRK76r9xW%y#H|j69m#fSap^{vD9@%pfG80_&fK!GYIl{IzC(U+46|yak2+( z`&ahLz?Ow_=Ra0aU(ksCE^iMWpQcAG>umgjvffJ}9cBEI3(pWVL2GN=;q>#I{h7gM zf+j%p>%LaF6>8x|^L$_C;PwJF(&Mr__Fr8^mArU@X?TT!H*cXLskAJ12;OhnG+AZd z1mk%2L8BlMJ_Kw-1vv!xJ!HIm`Zyb|4dfz^#}7>k;%Q_L7#I6Nv>}~qB7!Ve-taiX zbO>+E*UoQ|q%B`>wjcocn_z386}Q(x~4OwRkrG%h-H3kPvtoomS0`sHqNI#k_B zg{i$3*<=qo9rgM*#SaRrQT>0dtX(VZ{LS1`MkT0;wa~eJronDyz_2U+!HtUx;U~F8 z)IKO{P=ScMNi)X2oJyAq`=>wWhIXR7_&77P-99YT)Hq*<%U&=1tfoJoV*6|Y>cbPC zk<`|cnaQVDVj>8!gG& zyuF}b@81dA!oc&n*J7*K?9Q8@s41FOzhNza30kRn4u#c6=2@DwYJ8~e{hGg|ykXu6 zy&L>ZN7%4iQ<)mYDIFZrTcI+Jcff?igu*uZ5~YVYaNoJS;5mYy^a|F0uB`k`r3&R< zSOU(Z3rzemr96(le_WCd1}I7R)QI?T6mn3UeVis?_#9?Klt5(3snR#$Oc=xKdp`f~ zb^iuZJ2p)%1;tCchk(F7IktZ&hrF3b8|jO|9|fP-%9-4*$%y$w@AHet>d(HH4T)%^ zKI)s5_Og_rDR==wl#8yC$h)JqosDz&%M>-Fz;VOX7^acc4tsrUTrvmG$H%AIQIE%; zu~vIyA|*`kia!_!r2!_pxqPtW0Q4#jqM*HCClHEC6J>7Ey(E9;$;JyOTG7*n3VtdP4(pS>|5r;#$*Kd0CJ z+n#>8TUqR$-bscTo7!JrqWyk%BRB}os>BKY*v2CRBd~xyPnVhgI4M~vB&~wJ4K4u* z^QSs|u{KO82oPA2UT(X~6bP4RMoRmSw-NpQk~BrN7baU;0(;=+e1vQeaSE5j&9k!a z)8E5Nd=N9?`5*^(do3dMj3-Z0e(u__K4C6FM%x$!>@Bylxk82{Vj+C}Fp zt$S1g6sE_cBxTQ2gOiP`UX(W4j-i{+BC!-JjvNd#yw@iKjc@#}HU*oX&+hfps#X2l z^W$k3-8cT|y3ORLN@uYsVP=;u_sdh1nqSog^S?c8kZIb{fM7eixGIG~)RX%2KHQ#$ zQ|zJ)Z%xWd1g-shdJ%no5V2>`c2g*6%Po5K1Dax2W*L~O015?K;n`u+pn%%azY=45 zDM%%)&fW3w&RL7jNUW9W%-H&7igN4UNK}oScg23JV0$>duwI_0j}!>UlNTYgo2xpQ zZC#H)bs9g4F>3GMjzas}!*Lo-=phBaMK;9^de6P4B#C=yTeBFfTQf2y5I@RE-J1`? z${q9fh+YEP$BZAEx-&aSV{oPH^t+<8pUZ<3D8qHnzyC%Ih_2`k*uRi50e_E%p6)cV ziERsX>*!z43w0Gi4;0>+qkM-_`=N1+#UK34iQV;F>D&!`=|TnU-b=crq=XLg#6CA> zhEY%3Iw_O6d1s{@YkY*ZP$t4R8J-dhg6p&+_tQ##S5j=T)k7xJaxP=UN!+m3xz~6K zdZ26#WJfO_z(gg0oqM)muP3Ei&RYoTqtN-& zft-BZ`P4z=l@D|1j&aJUhKd z$^u24R*WaW6ai;^S(GFl$^!Ne<6BHBnNT6z;mTALnw+b6_K7EnW{M*Ig76ar17r%Z zCWn=qam(h$W;VHm+wEG53$Qhlp<4JsJ3AGd1pfXcDYF#Rj|_5eKvzq8JvlTrsOWyJ zj%+KpOQ@v_o((cG*~Qjrp#(#ug9iRiT=lw~i8ZE*NI*}E)Uey>1#(4V1>2emBzT0v z0jBftzmNb^8yL~=?#RK^J_uO>5?Xl%g-DS@gcqVZ-TR77F>=uFiU)4jx`j@MnPlav zC_`2;M_D$vLQ5D%h5?nt*P5{$V@>5r2B?)^ZCX|_QP2Ogp75w;pKEU*BV8ysH(J1zU@Vpfp72&eV9(C*%IsA)`8sL_u{wmh*?PQ1@yK1&Y3O^7`GsR>oU;4byz z1%HazXw~(rh*2{`I8D_5wbq0^V_)QuBpR9F6Wa@jL?9$%6s!o<;ppDep%9oqm8X;k_7oKBwi&}{re z+0`vDJwT|25FJ4popm)x%S2T@NjS{Ln|V8)Cj8a;S@8reZp6ERY0ef!X;@$ah$oxP z9PtAHw)+PL7@j~Z-x*?{mW^)41Sd0!G_@D#H&!fz>jqpzsqjNkJutyz3LYfs;r6+x z9W&=W??Xr=exQ`moAUjfetGkI@BJCR&mMjNl|N29A$ROcKzYnZ^(25wE;dI$Pa9wa z&>q}4tWf9|)Qyj`HYvxMa$FqP>yEU{X6K={F~Wg1mUZ&HhtGJR(}@3U$uT~3Ch56u zQSUG*#pNn)W9@p@Rg!&{*K1(0H3Qp1TPIYGCE;+esHN+*{T`|rI+Pl{HFeq`nN~!* zHL%-;mm=}Bq}cG2BWv5!HN=bX*udU*1ZXHG?0x(#ej0Yd?N!kavPxC#HT;N)<6_zo zNdlA7uMXEzn2n{St6K@2sa$SjPSW%O2B)=(NH9vRHd|gt`=w(rba<^G26oIX_zUZX z5ZKyijFI&JKKP5kqE_{NP&V4T7lv#6@#(-zzqXKqR(c#x*SV3{l&GXz!OI#4*h*1^ z-$rXHg9ZGz2}|E_0kA?=u_Z>$@=HWD?W=(`g;>|6GAw!i_jEr9{wo>{JJBB;a|;y~ z#MHR>+p7;0#6NmP$`{y=x7U$(8iH#C5~l;eo*Lh%unD19!e4zq;&{=`bvG@X!a>O< zNzn^bsk@~~)K7I*2D%!nY%mgH)@pBeh6(fhXI9bHF>7lP6KTTVG(~oo@Z~XDBesfI zDxG zBnBSHc?tc>tBX)nJpWUL7>(0=(E86#J=5GwUxbq zzQ2lSEF&B^AiDm~+)wwfisCYLZRWRZ@#M2R$>z%r>sJM3$9;Hf-?<>pke3s9XUZ@4 zO5~nK*vc)HaIt)2Th8o#oHE5Ee{OEr&NfHww{pMY$SSa3R15B`U*F#KewQ~{j`weC ziCS+)YxSPI>#G9U!wcY~nE=9E!Kxi5o(!n@ry)AE1m^rgy7_&Ht_=Lv?s zy7=#D(r#Xyb$Cqu8mP%|A7ekXcCQ)r1{QmWWM_(9g!>S*enX*jT+a-tA-&=nC-q-i z4C^9G$EAh^ zud!~6C$v-c9}1}bZNY5m>(P_BJv$;uOi7Y9*T%v0yFmGZw{zMqx;%+*B}8bN22yax z-nVAYcY_Q3e8xvW+qu2sEC12Gw^kcFvuE}9+3>5V&<#~EO7+#71K>S8ZZ9(t3Fc$z zSgXBn|Jbh}4?>F(JRH#SL3^qAo}PBNzz{hbsm4~x-Q=W)G57$F#VaeTJ#?vQxt7$R zqMkHKkFn8PZ;sL4)_ zHTm-h&$k%juXCoR%lH;5sU{oGG6og~T6XmxR%I2bZ=TNDgf0#Q`R#JwV1r=l`r3)E zykunJ=*-bfwtV7JH(Qp+ck$4<|DcC}*5b!&f6OuL`APK@v9qCBM5vV#NFp?;3-9e` zbBGT|lrUs4wW~jNepeuO+KS?H4bKt+}6-xQdIh=KrHmGk%9I}+9>zIsi ze^2ke==rod+S9{~URXo}s1XPSTz&m#L-atyBAEe4%Zms${rc~(LjUt=rcDEI$NQ>= zqza?ysd>U5o=**Uj~EU#kPq2PHU#}2*oF%NrFX6BDDB*yI4`@Ob49;NFAUC z#n-(hmsyK-Iz!E=gqE|!_f3Yw1y{hduuvuAdj}w~7#cI~YA^|KJ4zJMEZra2OVv)0 zT(qAi6Qcf=N_Q4h;=O^BV0?}XVk=rlmA-}UWHgfc$PYx+6wy!ki4D{tKwjAb4io!# zZkhcFbnj11Pb%5f$mmr=fR464HxU$BTW*yzjnZ_K@?iQNcL?4ASqIo|&*e!0;^x30 zYJKS|5qfCP{7-5DAkN3LrdiIDz?~xd@waFQTH71bf_H=dqHBTn`r+f0xJX~l?qUDw z>8BrkCf3%S;+^AxwzvGr`gU>H4O6KeGgcHtgQZo~)v;YmKOjNOZUcmD8SJPmW7*nc zo|4Xtzj()4@_%MiBDhs!2!0e$T*6+s9N zJXvuPNg2s1@;7()yz9ob2MZ6(3iZDy%(}lTLt_{j_kUOw-MOf!7oo?)pm6nI+c%q$ zkWC*p;UxU6jVr5Mn~;(w4Pu(0^niTe-}X*s5NHE2WE>4h`JWfLYt1R6-7&Yp=XvEyAWUwFl@S=o(h{m6%cOpNtJQt)_yc+87j!VNAZf>2Xk;;+^MlN*LK#4x#Kr?n#k` zCZu1pa%4V$TEV+BE)!4I#Ea+<3d>NSNyKu{XvIfoBcs_5~cJ`TN5C@r#8g#t0I#aNzAir)Z ztBMgEQ3X;ta(-^az(E5kuP#M=ec%H&VM5%``HG5X7VH;)zwM0pu}nUdr~)Nix^v0M zk92Sai=fqM`;CK$1|~j9G(zlMSMKQS+NH;WhJTIIg3eA>f#63?d%=HJkv~QQd9maF zjERKThsM()XN&exyj)OW#Kt1FI>Li{6-AiZE9&P+V(@Z=3bfn``JZ5&_WpJ8APWTG zXarFB8TZrY2*PFIR4@0%n+6BlmhXk$t|lnOlRuH~R=?#+LyI~ZQ#Q8O%G}nCX_r^Wz4Q!Y@bqQMN9LVjS+^&10pOerf3sZO8xMApcRY!Ayw>@Y z(!o6hmK?RdZ!`?ulXNtps+g-SR*Yus7D$@znHYinZa%_o_ke#DR>NPTAfF>$n%*Zw zlm7pPBj`k#qN8ePG;EiZCFOUDB5CA8{-V5i_#y`yrmo0j)cut|_t+<>cN`5Kt@c{z zU=95Ufmd#A4e@Wi639{cWWmcxxQUS*7t-PINLnX>G?7q-MnP!P@OeoykPYkbwsb$R zrT1ukX;nT4N`W}DKBQ~OQp?{FyX>-YC%yWTfD=#S6)yhX5Q7XobAv3eh5NAC)GBXd zK-mMM$7%M-nu~YSVj;+r_XXZP9?|o0{dc~gQJ9kbyfw_bcEaC}0DGrpwP~>cqH!Io zxwZ%dTPg|Jaz;x3Y3(uO(`gwi{4K{q`r_#6!#%$QqK-~Q#evK^D4IiQQOt5R|FyjO z2r;Ba=@rykm|;X{oUqj_lYokSL8)>l;nek9U zKV9B=ksklU*@df)*V!JKI3TM#WU-hl@C9ubglg&dz{E+j3&N(Enp&KkW4ZaX>71IZ zqEeSJ@A+aA(=Fg78)xwAw;VKhkW-)+#RY`5d832Z8%6wW^wp4Yf4tYKquBj#f-+bz5%hAb|{oC#FT_xGm7i1QB}m8s1|y`{ZmL zvBae_E~brvb!fE>(?1iN*7Fm~Ho2k<1nDu5(ycJ|{tbQ7)$lh69|*a`Q$wQFc`Uuv z!WP0tDb`0t8cSAnEAP^oRllAjtjU?Ej9H$vv~*|0KvbR`@AFQ3!leWr!+J9( zAzB8P9e4kc2j*cG*0CF&QI12Z)ZmQC$ryIOW!#^=T%t|W8Z8WPb1#wPgg`oWmLh+8 zPdtG(!if{OsUWrl4|=!f^%B0LKfT<%{mb*d1>WXAZi5&aNAo=-^2QVE)~i$be|pP_ zS>Dj;5Pox6BQ(eUwPFNoVjV$kej`Kb1j?GTE_eF>e&Poh4UoyEd%6%im}&M(4KLWQ zI4Ged-S~@6$@0t#7BVB7gSQ;YF)X5CFW84$;akA7c35zDNiv%?Y?qQ%U~qAF$K?6= zQ`yD2s$@lAQKasug0jOjzgbCV7*mtaA-_02qVWyLUV^*o@f1kdL#}0wtE80>Czx7d zB=w3d(ISyFBPVfZAMO2xBI=4=%tDc_#j4PQ+Ig0dN=#+ z0QfcgF3Ux63rqUMI=cVMpV<_E0N44vc`=FhFT0w=_~fpSZ4J>NV=7mt?Q!~DQ)OWj z%m|e^vLMJMoFW80Uay>dOwAm@cdlV>wHe8j08>Vo&lkcH5nc*IeB)mgxG(1kv~S)G z#k*`@|MUBSl0rY`xMqpqM)udiJLK0TtKD`8e=WXP$R;&(Sx@7jl1MRQ!+D09QmpAJ z^dVEp)`~q#5 z6OV}og$9UBJ_q*#-~p#|MOzk8y7+Bqk%7wOWc>COMO4|Lu?`f>?QJAN7Lb`foMa`9 z-7BK8^+(^?4rV#wK{_nIG7JdAsbBARVW$=@Vo1Q4=8chVB3r`I%31y#w6gp;z)K1H zo(_qMX0QLFHuQUT)aXxCufS)tj^kut&d#i-9H`V9z~(mvO~f2O!Hubz6&+9_C)hk?Mja;(RV_i`o5N!@aC5VKgU*wpx9N)VOa_Gr7;$Rsj&JdS~_}W!xc1_M`akZ_Nx&90gXv22%uX_tN zzu)QEPgja9Ob#+7bXmAc(#BiKu#cN;_OIR;?O^gNe;|KYi(Y*8A9eVFN)audETB^E zh}L}D)Wk3{NI7S{HoMXg1Oi;8%U-`Vjhb>ZT!7QZQAGLO2-b;6E#q%{Q0do~w`tU_ z7|&q-w(u zk_}fzpU9EkNy5Sbh8L^HWvh4shiYift807$#HHrYSclBT{ODlMISH})t!*4VJ~jG{ zat!~0N+?CnSQ;r@MSsWLG%qXBxyBxuTv0f^7Q=!q?XZ=h=38MfX}MD?Wur=MTI?Ih zEZuL6p3ZE6~a9}+PBuY5M#*~Z4!qaK>k5Yq5jjOm*5o{1&yxfb8=n$S{zh(K-y zq|E_20)g926*I}^2g2~TPg0gHn|n2mumwLDpizNnEi<^YCQdz!o>7HD8N|>orRVz# zMO-{NdVx()kDv!Q%$I_LMW0jrD~f}_fKkIiG1pZJZ|S?ojc8iX3Ql^?5c%}^Gi}&X zi3MT@a4T50K?EztI_an(J@}2kvI2YjJS&Bvp?GpKrb?U?U_L;w{->7kTYE_FGhf|t zh2dztw&@<>(2Fe3Mu;23xMDe0pzN-4ve`9Rx#3mMC$Ha@h`9hUl-uKqd}>X~3@eVE zC&q?(O@YG;@cMZyND-oQf-qoUi>&YYEYX{x) zvXA^3$sti1+z8=m|G+Pm8L&Me2u+2W|9y0HPLN&={8RLcjqg&fBAvTd(_5 z2!i9S&vOIAh#pUgS4-tcCSUN*|5oVbJ0tovy74JD*ERGqxBWX5iH84Y%AH`UNC`xS z=2C8e%{Yrx*)F>6Spk#!D8*{6LsaC(=zW0kD6O1!c^5SsbFPazG8gr)z7Ca7$_qzG zkRn~*W4v-<_YaFFB!}esI?a(0UWf?6i(8(JK{Qt3#)@w4rp@%pCzXgHftpbCMTG@0 zUmYee5Lu7T33IN_o3;|P?)|h?Yd?!z9bIo1GZG3n1c29?^D=XQc^=D`Jhr$W^ALoR z6ECy4!m154lARojH^APV&=10p5`aRO3!v?g1&TPp(0Jwx@5VJRY!NnX=6u1>G7W9I zw=P*I7MO$~Wu-u8Q7R_ZW+<^*2dQ?mX+`y!dT^_{j?dSx5Mu=v%{WK|y!YSf{i*fp z^mv;TG{0{!G7v3j@+})HNI{2_&p_y7_Mr02vGK-nRf980q9}!G6ez6#H26G2k!Kr- zw$aG(FX_Jq88^)f96<^m$m+*!R}$7FM?yEtDC?>x{Z;S5fNYzu7j%Wtqbr8`lsnOu zDj2o`Yyo%ymr9aHM(z)uCVQVTH25e!Dh2A(J7nbOBz1A(5j7rDRN~$uQP%9;eN=|F z#P<*AIsZ5@?Fi&BuxK!fR^(r|A%>4CvO?^yxR{2MS$OP8ZKeC8bf*#&3mb!*yHU(MJL?4~@Am_Z?qmscffw(A>YgenJg( zA+mGmm?>^Y0B!*20HDgMstL?P!v*DhC0Y;>efnp>3TzI1(_OmXxz;&tmVUvx?|hnV zVTau;hW0Cpf}9IuXK?d$^+nlOK1H)(?QibZq$<8nO^DY;KO#jc8LKfe^#98LN#~f~ zoDr7o$4BSJHpn?0UG{idIoqey_8pko+y&d-75U};9i8|KO_zwwdTn8Ct>+A2u1rmJ zVM2}ySe~el&&U ziN59?ZDSg$>+zb%?NGL-V{;x4)48~lMJKQxYbqIlqqWcNajz`F#PY=)z&sUj2|EnqEB(%gKceL z(06IzTLnOqmfwu8W55RO<)Ntlg(ko4^@fwQlaoQvGQU|P_B-7z5qeqGLt`9z{ayny zeh$K}!%DnItdS*foy7xrAgsQn@gAR1GxH;{q~;8H(;)F&OHy(gBwu*$K@qS^y z%?DPoEC%?G5D2iNGq~t%?g)S#07fOqH2d9mUrTjX|;&*})5(l}YnA#qEDHJPy*fVbZCP&yv3|)B} z6O1+98qgA7d2JNmsyUdoCm7NJem1rFo>Q&N)FK~@5r5D(XZ!=ig8bKUBr5P)y;LhEiZP?en?E9*F`g1p7C~?433Jg&i;RXF4O=lUFMcZ{@ zK#)>OKm?>qy1S*jZc>o$lrHJ+ly2#gZt3psRJuC^zKiEQzW@AS=AOA`@3q%DH_e3p z4?E%^jF&n0a82E}HBlO8+e&S9Q7qriLx*y!_dl-Tg0|9!k!bIM-*P-wHOahn?x1KtQG zB7rzyp!>$cCNu^@<{2p>Qn3LbS3+z2PRwtk;Q;~gAZquek5(*TbVOA}A27C*rJzL& z4XOAPJh2Ry8cu6|-J8%yWrCau3BEPUl|MD7*LG4;>k6=o+Zsz5LJodo z?}RV}?wuIsLCp&Ak#g2nPksUB#bD49W|XR5#zqV~)>@Abv_qhmBLv3tuQeAJe{s6Tv^L+t zIW{lf@>;#{kOLv!-;aNcDWYgY%m|-FvuFMqJ?&^M##PY2=4N}(X@n_Ab_TDDqKW(y zSl>E6#09bBXiL~UmUme$JBoa*Wm|Pep34w`eARB%)ka#@Y`QiQ-)jQy(!QqJ=P$vxUVN!(w1qHyK+yJq@yA zpn*5FgLm;7c>s+62?IcFr0S*zR%mmiVS15W)B$2SOnNEUI8yS z{D94=(iZ&hZhhoTj-=XexU2(AL!BK>f5UPO@;G0bNmECZQ=AF4OQvTdrgATMI+tu=Z) z;#L{yD`~d2{UnJ|M$aJN@`nwk1C3Zz6d{|Xy&L;)noERrw2}5i?N0g_IQ^y)ML=0H z5P~@WNIk}|*a@F$L-Hio;Eq5H#%Yr`XCue#u?i5#ox@)-+!UVDrx<=OkiZ%rJKN8x zU72*SxS4MBF?Ux~Qi-r+RayjAuvahF-*DiwQ%l?CWWbpsCkj(WwewJ~gBHhw@hAro zEp?m^NVC9JpPK{H3DZop_$y-A>`GstVQnF=+dcqo$IR4mV*D_#yfMPGrf5dCWGJlb zZ_m-&^)m}c?4yhxO&0g zk3F?DydlD3D1&pKy@b6xi0tyJ636DO4r*X<4U_R0(&WA;zV-I}>)6ulVBgYmjniL+ zUqDqL8>>Q0q7}ss=e`PJ01j&qj~n?4)3V8^BGKf*pViWhe`Ax=+wAW;sX~eP8b{Wq zYAb#}u99P8tY5(={dQnsp?ougT+2P`%81{4fV_Er|EsyBMg${ZKlS4@J2xnt0c`~U zVx~4vfy3MOfwgKvfwQzAQhZIZwGHNyz8{^+G(;fMg_NHh)7x%PTQPw<)!OAL0PV%` z2~j$tE2taj*ulI0R{7Z{nAWWE50LPGS$!qo`J}!1M=i2GyZOw`56J?hZgL%oB>3Cj zM>w)1ZSJ(`JdqS_;x92NUJxvgrf;NIvQFGeo~VRLeUP0Rh)Ubl!0SPGqP1XE1}DmP z;j9OO#jQYOlNT?wk5^kk{oOH5js8!|?`}IidFlqPAB01*n#cvkzLi$i$(M#awiFB` zx>!F+;ad~F#G!zHc01U1j)SKb($XUtw-T{Z1Q8Pp5&ImPbHTscNZ^K{m`ZNG`-p!C ziw*6+b&8yUfh0IdF&$p_e1cIWWc)Yr@r%oeK@I1MizhUU5PA8H%3k*I<4`z={dE!Z zB#g8XXvH}P6;}KiR*Gn@SVb36gnjKNq5=6RJ!c(Of?mPp9J=aK4Rk9G{*^4NA8sz6 z+>3MPzL9g&KmjAZ7Dw7=S%aNZT^sOqyImdGVKFn@S+0654pqP*V+;=`$RXE$tl~{( zhFxBeiqPtvMY`CY6=j4kS)eG_V!PV zm2P$#EPz@VLj278PDK%Ji8>9x)VVKKoj}w)lv*VarQhh^DarD8xMp$WM?}c!!l3Ll zTaW5kkuW0>gWmyzsdSpF&7Se~$D$)Fc%d)ISzVQWdgya!#v>#Ke1)W6Fv&bTFT5U3d$lQx{5etX} z0NPS%cr@E!@3^25cR6kjhVp9IWu(hPTr3j5umua!Yw|QW%-_@4u<~*?JGfM2#uXz; zAu-=Ar2mLsjC}mCDDxaW*&`+I!DaLH*)G(;Cv<*oMP0rL2H}&^w5tO*Wjjeo?=TjG z93Sf&INRvHfX{3<7+2&+;SED}WHx-KC^Di|miVKzp5fMD-B0x<;oc# zSYKQgIXzLWoLyh0VZI%<6_2#78WD%U_-F+)8&*riM136Y}QI_0$%e-orWiq05e2NkI_}EKe+S8Wpe5U{tO7+Os*! z))w`cf@PrwQ+AqLp1_mr?b9A@z$fVrY?gTwukN;P%5#a90>St_bVk^B0rMun%q~dX zt06!&Dv{?qbp!S$o!}Y2GNg8_B$oPxfk8pT7r{59!?*XhdIlh?Opj~s3koRgfa}if zsVDZzUA=w_?HgfQX)Bj1x^EVV#Qp{+9k0vVwZ|zhyCzRNykD3*MyJ)T_6ALSpG)@$ zA0g8P(^Fr>cXI2iSNJWI-=9i|3|cY+wF;nN$fgo8(QWa7{DM?9^=bH5Olpy%kCe0| z*+8uKf+0Xn)g70tTl{tl1QFh?)-nF|-$Zc!zaC;TsF8GeaSImgz;H0SvlQ{OyJf@1 zX`p&zMKXn2G(+BOHRbMDwX=aDX6;xPd+M9vWOgJQgfCBwuO)Me%VWq(>VXXb3t~Gv z9Wxv}UgUbk@72LS!;E9pC|*#8hW6&l+RJMzU<%0PeQ!|#o#1B0g?_59$9M}dYm}gU zX7ut08Df2(w3S~}X*~P=dAYKWJ3DmF(Wp|~J4ty-lBxWJ$Q`46vwz-=n)+%WIsg=4 z%YhK}ckcjrr1jiJ79^pNfqCS`M`gBs;9~@};kcq4mZ<_aPXFkPLD^1-9+Kna9==3q zeD#FA|Ir6wqVH1{6v^(90;cF*99$NXkfIYe01D)f5(q+s67>dBMpWiF9MvC*7gE~G z>`h{lve1o77u~##Qk$&17qDu|glQEG`V^6&eQX)hLtiI@K?Vw&tP{w(A}VI9U?w0N zN9>H!jP73IffMvI{M1(%evQVEkk8dP9Ap&_VPZLCD|~nn3ccMeG$!`IwFk!Sq-9Ys zt9UMtb~GPzLj;_J#`1nrt{?{YYH}YZjbXs2Xv|&RtY&qlyedPibvl6#?$X#XG zCH?zkKyGz?J;;w<3;}a@u{}j@aY|PyBS(7XNGe{JE*YC&)_r>XnppMM`W%&LL@j#) zd>K3b>QMQUOVN_zJ>-6dM2U->0qo`mc#BWZu%EokDS6P8G@L6FswixN>huZ%PY?Xj z?b>#4&G1|$PIOuO)3}Vtj=d=`0e@sKg-&IL&B-kr149M?dt~_h>&)c203B&zXNEB6 zEvNf!dLAF1D0=Jh(wr*au3^viD=a)J#aF+^JFq_j}y z6Vy~*a|e!?l97g~$_fP;24Gz@^zpC~5GnVhLo12>Su2V$P#RbLC&K#EmUgU? zs+El%WyYPmHPW0ApO+F-oAa6{QVvEq+qQ!FOwirco>?iG*NqXx-l}L7m*$5}N;sUQ z+JX7Fi1f9cP##uFPI1fm1}<N}je+Gz$I3QsYrJ==lj*K2gzfe8 zboF9$1dfAnxv`mzw^QrkY>&X;19UX?7Gn#U=Gx=lIXtRR&_@0G(N}!ym*QQ|%?B

V|+jeLV99KIhW7UgfC*;HeiH;j=Z=S3yByMyS1jyz03(uU1Av=+J zsMcTfSP({Qb@aN+H@Mn*hLT#3H0C@NPhx&P09+&F$SB-koVnIGSlw2N&`Lkj|v+fOLc}zIqA!#=m=Qr;oX1%TS&ko|Kk#1oAoE zFLJ*lley@1Q1_yMIcTy0BaR_2I&|S!64knh*wsES`%zQ5=WhSr6$#;Ufc< z$hN4@SgmVb8S5YRNRW@_N2#STSSa35X5mDW327@MXO=IPUZ9A8sQ6YOmi-;58OC&; zO|_%25>|w&q4Zgxn{l@i`To9S?0L_ov^J}*di}^G)BV39fcRf+Z$48d@-~A`*XXOi z04BmcTDyDB%{O8P{{NeQ*8<`A@lgwgc?-wAKu0|T_IhFePAM5}!T6hBd}AVn7BHGV zY5`=^3h&K+pXsuiv3;cRA%qENeKI0M%WT3ilmskFKyksY4l%Tdr=$xhMS#|SQ9hE* zyl3ytCkBsTyIQ3uLlVV%303|@%_#T7>>e??J)|HISN_!ez@cBMbMf2ZNR`^0+qp1? zJo#WOv-qZ3Kz*ToDU5?)>JjG+pBvProm6bM1!0-NeM84f3`g8kwwYsk-~h;s%_#9_ zs|6N|R_RSU)VnlflAT9a)d2R_nx$aPu-^BZUiVXFc7*^l30Zh?%O^{P47jHX27$s3c)N8$$ZRHpsN9FLKV!2*ilYh=QD~|L zvR)NyV>BlW+1WoM+Q~GX@OaA3?4jK)G_HSH-l&84|G%@FtDxTTZIdjA45$pzt<6ld z6F~?R`#i z7SdJ1j*PKsUaFPjXPOGH78J1HijO5e?h393x(DGu077Q_Ri`ZaZSq^i?q0i zCVN3FFh7IW`9qOp-cNPOItpE;)GL*qcW_^c{1@eNorA0gD4TA1HK`n^iONYrZ(@ou zq8f6x#!5pF!#{Fil7$53tMY53n_Ruu#DB~WGu}1Pse+QQ*u;f;J*UE=!^-|7RQ!^y%}18V_CT7bFQNOn%#K=pSs3cWsj z6^w-&vWvZrIS!?5m>SCYtm-BNPo|Qv`49i~`}aitx_3=h_(jDBtWm-Uqe-l3pz8zN z5+L6U1M~uADOGFn-3dx7mJTGu^L26Y%V>oWusG6#DtTZ(Q_qb2$`=q+hqzqi{>3jY zJo@)^nT@?}3K3jNY_wW>lcLXz`v%xmG*P`_mHl52uch92pq@fIXkV;-1Y9FoZ2b1G z-XBcu+}~7nYpadqJ6yKsHtnqfEzCy&CsQDJYO8DGeDd&ed_Jc~l;%M5)?-Si?x^JO zieH6W3VGcy6>37Uk5Ej!Sum@ZufPLIcR5%H%mk*!4R^M%@Dw4naSC7nhn0F`x4}W! z13Uv)a8Y5W;$thR?rPIil!Eamaybhd+Z|pbCYS5JNn#A?$1M8y;_kS0zpwL;9yDr7 zEyRs?ZbZ8|O5Hv^PcQ?P9TTo01}#PGyLXj8)dysz)YFmer(Qz3M3m`3!LFK8maL+v zd@=3b?Rc;sa!*b@Jk^mYy4J8_gHFk6wiF665y1sH2_Cn@xX-+}jii+7_LUwlzd*Uu z+4#M&I9``o7$BUc8aMaa?t^a`yf3VbXw zebR^wF5eqJOWmG8@H6hG^b|Brpww{nMvf)Q?K^HLs88&UFA4DY(OP78?C`JK2sCO` zNbe#vQ;HY_`4iIBw~q{+FmxQBcY6l0a+ecqH=zcz=W}_@;sm1h=Q66%@BvL~BGp%Y zI3LI9;2K9IIskV0cFJq>`gPMHiVCz1BJ{?HCumYXQ|KDfpov_-rNO^J2*3g`nq*|- zNb)j2l5q+At{4?jfk*@fSLKqhreIwT5eYOrt+X0WKEPINX@go73?hpwM=+CTo_m8ImZuI)r_GZ8 zv#Go9_vfl+k-pxJOHK?Q+B(_CCq(K#&gFK6O;_@P^JN?@qnONqBVg>Wx?G(cf&Hxa zLQw-6VPLR^&fH<%!0}{N5je~D-RUygv-UPf57MraMgtoc$QiSejXGq}eDiftfnGHfcT{ zwbNk4PO6Q0xjQI9ZOD#^e)Jv74S=Jh2_1Z)$aH**AmuN;sJjx>_V8GceF!gPnk}@0TPKd0)_V#hY2lNq_dFz+O2ATd z$fq>)d5m_q{C($+spE5>|8b}f^+X-aGwkqdjoH>8ebsX-A>9-H(DIJmc+OO%4nw-B z3DyRel^BUO^P}14DGWNdA?i@AE!OfeIDSIgacs>ln^!&m_K&3zKXw0CFjkE$X;eAU z#7^TK1zXSAOnAX2|ip-Tt!{3?lBFXZGB%Tzt$(k2*ngU@jMy{0S9C`3ERyy57*y#3Lg@<0u_^4eh*h@hp2!AHd86Uo)j9#P^T zi6b>F((V(iUPZKHjMH8l2+d)8vtv`9s_9~DI<7P2dRBcdxYU#EB$7^&5yN;V0~*ar za!Qbo$wBI9tK!y7%h2H|ilo45cpS-;agdb!xT9Q@ipPv&ND=jMH1OyF5Cc79{lDVn zopI`D#ge5QS_7Pj$M=iYr2!fFFhzf1gl(^;l$9WAH_x&3y0F&a?r;HqKqX28Hr8mP zPrb!^^L)7Xe~`l{o@(%;To09?MY?=VKLy%A-CR0UHDGz>l!?|2n6?`-#~L!l0X9yC zIT-cD@~6c}IRz$)mEI`G|B8)`mjH%xn2x5SpoZs(xp~LJ{2w8Lo?hSGrh=69f~Mez zC3aAoD9ZaQe<~!U4qVuH%5ZvAsXmvdd!Nw|xSQO8v`(4(kc-)qew*iz5e|vNF_^PG zv~>-CZK<2%MLaWy_TMc-ij2Cp_Y5DkhK8N^_UH_aoOJ>U&2c(n32=r9OQW_yQSF&K z2qP$!V!t%=7HSheUL3Ea5iO-EvX!K@(74hzFd;FE-m3_gjV!&VM|fj@+k(Cx$0Q;) zjlmucc3?yF3;h2%U$|PDS^R@85oRpSi$33X10ED!*Ag$-RxIDP<5mInrVOUf(wxuX zO8!uQIO(zCDf8;-oBTN?ymvQNI z?PHNDAk?L_X zsnkC16F{p`9l)k_b4h0wUdLC@fM)zO=!su!cBJ3zDy$Pb=d%Va*ZvAtI73ifijk!5 zaYy6}>TUpx=?HMp8H~X}QV~dmN2;&n+@?AR?=&j}495s$>0^q{b-=ar>O!xleI6y=% zMwvY>Hy+ro>gOUqYQqc=r1BplY~Q(1)=>5$$!Dg8y`<1|<(;5`LtDO-nMQI!ZNN^# zdz6C$G!u(hOEBNTKLv{0S(zbMKg#s4W@QYznVS3*Yl9xrXrm? ztHkV~%2&zniwuv}KbOb?TR^G4IeYh0Vq>3<;pAJKOYyeDNH`l%XJP69lJi zRM+31J$p0BatEZM0ugkY_my2oJsa0Or+GaTKHamgFKZe9ntrQv-}>^e6KxKDJR4gf zm_E#hw{7)ZclUK+n12ab#LOqA2^rv@1IBjXbOVq*V1^f!Y@7Yc#4se5NS6$T;Mk?h zBTXO0xvi!)t)QFq`H*P%)Hs|1axuOWp}o22M<&F06L5C-2Q$#r-s+)mzizVihr5Cy0JA!;e-H*DAFKjNt$0{j|jV zYXl-uLXzJBasaF?$d64js~f!fDiRMa#*r6y?art|OPf-5rD(S8cle z&Gi#gkbz%0K&nuiP1l6DK$~H^Vy5XSc(rLtUAJ#I5oOqe&%nKI^3Oz7gtYlAa%R1oHmzjk4ViFZ&i zw*=R(1W$24>nf4x7hA|pD<|KsoK58^96R3c!QvicG6E<6K4 z$R90FbHw8z&|X5CRVfXGc2sczes(gJj!biyQsfgdF+v~&APnSepqhuq5??v8F1BjR zrHTcVMUdmDezLio+C=H$lAFAJE4Rs_5Ws0{K}0?v44QOM=$}_vmaaNQH&M8&oftxu zwcH^0ON179%Xbf*Z#P9HAgBsUrYr zZ>!sX**dLwo};hO(x^~vGAHna(fnwc4Aj|4$M$)QoNTfRVZ0c_``&kNAdVkiPX0Na z_%oi?jN%LxB@irtGuw15^4Z3&_hSSu70e*+BPZ_$dx5Uus$Vvm+>apDf`hzxLquXA zAR$#T1DEo(y79*wd>}gv8{Dk*H?oKU8&l^2s_jN=l+x7vB9?5!4~8%}D73|Mk7=Z& zC~WMOmL*AanCb-yJtuCBIzzy^cB{BdbGmze(Xl^;mU^m+CrebEU#Rvt@!T zL801K{6ir$?AOv)#Y>{)VT`&D@?uzIEVL){=M4Vd(K+kL{=1;^s{%f31?|d0{&UqgALSozN*h6o zSYt|l%Y`6kpcwWF$fVB0x4FW-AI6m~p7HI`=P%~p11^IXD&O<)S1|3I**f`Y9SfD? zsPS<^E6S}RKiEl+O*@MeD@u&iI1c(N(X``)Rpx~=?FgTOs0;6yR_ zcWo8)6qWsnKw+ShBSeWJtmg<8C=@YWBzbRhGX9f8=j_&^KBo}mDa8G;d{*{)_dsWA z@w4)1>6>({KUwCut=4kmhc9VEW52)Cu~3G$>qa*ocTUaM9H8-gRm$_ALlBs!Vh1EF zl!qhe-8C z<+nJFFxRio9Fjoq-Dc zmIoE1QOfeup(1MtmrgLfg23+`O}SK73{gcz%mj^h;((=vFvM}xi6%J)!uf&BA8LAN zFpD;9Y{(IyK_Esc4&!PJQ0Lv1$gWJ6yczJtZJ?n-Y4sFDaa@oZ&~PUAU|uKx`WzNH zqjS8B@{E`Zj4f(l%l`TkY0ul+Ny6BM*F1~z^N9sI{{Dn9A`H+Ec z%|O!BLdgxJxpX;~S02@%_|)ptp#TR&frHmi?>t?X@lXu%{FYZH=u5t>Iv68Sm-E?} z*YAWH*WA4#=52$Li;7A_=&}Wk&i-p2n)*W%FpzJJM0J!#T9hU5E-vl|%$hhw3r`TA zPhc|vXa%cWPLA}=7Mb&^$^M92dT{?GWf-rxYi4zCDl2k|4$dZAKfVpb8Z5@8s;J#M zy|r_znc>YcQK;)NynL|tpoawe3+wrIpy^~1R*VHLZS`z?1BJwyizncg z2@{V9jAt+(4?zI~B41uoqlv89gA9U>Mnf9qQ!`o=L)P4@G$td~n-vel+#t+8PKJu# zr>v{b94=lgiq|+7nD=MI9tC zUOYY^lfY%Ojf1vs>uzS}0uI3mkTfkH5VDHXK6pF6KfBw-rx{wwfJTC^TH~po$IyO; z_^MYJN=$OIWiOZ0%7OnsLSO#DAF?__8es9K$USQiDY#Z;o!CFW64aW#p7h5`kmn#U zo@nxe{VbJgd!xrT7r=W?NX$x%5Yz<=BlOP)ilBnxk9E}hf+fGXgTD6&7;nVfXG^%B z1;sHU_-m@Ujcj}4nLsrls+l)Xy405G_4^Kyz)Mig@cmfEaMW>2?6M3VfN9^-D#ey9 z^WMmS%ywpm=X;IW-IiS22N?X)!V~LfLKqb2Th)Iz$iJTrRH2}Rm16wA1Yu*RLG9uZ zNyZjheLGDTXp;f+Y>f5O+q1h@Kyyle)y7At89zu9$p4S(I%I`TKx`EdgP6h=UtgAkY;>e+>5_AND<-1ECmf^h8W?*pchp->%d=caab@mNa?D_VJz^E{IpLmSf8W#*_%WV4v zQ_v-s0NaC59rnx2le@-?p9{a#6pEWWP3{ZwUxHj%Qy6{r{&*BA0|&i9rql|}*zj4z z0j!YB`X-MW&RT&oYXOw{OydIi_mk!F`HwaWv){1G`2-nD_dDlwh}fpIL=j71g@s^V zemTPWrQDMdJ{ne@6gd--@82rEx9Gtks#06AJyYuxo>WYW zobTbZL(^=v=fM4FTUeF2RKXdf5hu3)@Bn?c04s({*0Y%Sz8;v!0f4+YE#A+RRTo^1 zEVMs6f0)@|I#dbTBu%@s+^nqVgf@->maZ%#v%L$qpM5O1#zrVLhnq{?n}fj#|^ofq-5rN zIvQm~2uQ)s{X^D%-+2V%OR+V8_zGVD84AJ514Vg{7w#6`7M<~sY&kCS7{f=2R!r;< zWlW}d{=!Dj9-^-EgINSpip)Tl%k4`PNi^( zWRxwL!(WdF7-?1afI@jnZnX*X?(4fVFtf(_5ucom3 zi~1y|;W&l(ivSFZY1of8(vsg@qNS5b0vX7m1eSGU&AlOX?eXF*`w2gO$eVt{{DSc| zy+|}_gGDwN06+x52}kF|L>S9g2vn4mB5q%>1bGhOp5Xq4Fr;l}z19BYO%W8-g~fI_ ze=x_q?m-+;PlREmV9;sSo!?9VM;wwt`tGcWMk+~xN+xUteM|8;_?quojyz7g`N#Y} z1VqH#3Gk^6sre%UByte)(pAu72P?o{qA8pVai9kQ{?BMLd3%lcR5^fP0W2FXbX>Va zvc%G*(X_oSPUl(JX$cJVR2z|!^`)u@M%2fHw@!qUNeMFuEBCG}l)L;%w(7)~Zp~mde|SEQcYN>y=4*JWrS^x|1LzH*OC5-lA4>*S zoOuBi2>3ruWbwpf*@Wy(5N+LBXmzU7z1p6buxj7-$IS3|82-7Y*=Dr+qpJd$X-1Qz znqPDr*ZFbsUTUnY4`c+9$zuYln z`(Y$};`VU6{=3J96`e1NW6gbGYlP!B`{vzM!S6MB7SOV)*+jd3z1AS#7_@gSshXbwq_)-Cg-o@nsT{98*#pYQp`7e=D z0Jg(G2$b99eF>^~vi$zO36bh6H96lUo>z(kl4_M+A9URjO&8&ty-pix+3iT0v5iFO zv2NXHQawTSYDb25fyM_0Zv#6)+z(0=3#VB}k$p&SAv;A*2W}rx5$dQ3$rFlJqyz`k z8v)c{#n|-e+J5BW$1|2)$hq)DWql!7fq*4kqB$<5Ou{N`3|L#2zt8nTjUd#JPE*@)9)H3eaZdW z>&(7Ng;u}bK>WQ^ZVQ9u{e6qv8=AFX2)Ku5A0YpQ=w%N}2?C#zndI zn(95l=>s|e03qv1Oz~&yST-4J$|-bt6w(BX6;tq4X4&i;nqs;j5%Dvu%2V@F$|ZOO z;?<$?rsr=&q&#Z5cr`?Y6vc|Bq{ZzIewrw98&g59WDj{2gp^CFwJbg-lB1|jpKpZ9 zF1yZf0YQ5I-k8AkiEq|wt*^ekS<8z76Y>|VcSH2=Tv5zEfgO||6shS#RDrs`qZIaU zqvndS>(ZxrRZmz*R$AD)9ezS*2y8@wkEJURSy=-bP)lZhl@>yQC8crg9wyaC8H^2| zAR?8--HX94;cvvUR20qA_$XYkU#B^eS3nk%njmIp{Q`)=`F}vn)WE0&G1~x8p7t+N zaQ|CW37{?@CjP0yY*PW`ii%SU3#W#!RfFJx%va8HL3XFTYTe&1RZm*1SuFXYTh?6Ql@tVSh-H>eD&rNmm7W(k0n#JS_qOW zoe5N?WEjJPml$DLrOv!qP3C+52O$dFnM;i;pew zfW8HII3rm(_#g!to0$%{Ud-UM_EgO(M!S8LQqN~zM_6vXDFGsVZ2v|2(1@$R=1lYq z*omUrTH`vxwWlabm?~O(aHSp(+tLz@sWp>dB}qO_bk6I&Qm=KDuNVbJ^bEqMI{{$2 z+p=|ydBK|J6ZPd2)`OiNgXT>m+O|+b(4DCn0!0_!z-1yTz@`jv1Ee6(PacN?Ro@%U zL{Q2!a2szq`lIzBHWXGTe74lvXF$6|0$m>9KWs4&aPpOCBWiWu6O zn|HlTyD{~RnYL=KV8A%~9P7Ct1>B*aB4s*Bxu{H%jd^Sq@r+A%7SQUWEUDWaQ?;n2 za8S)LZLThq7gU5e5tdGurGXsT3$ExZ4W_DkgFc>*Zq)X3c^qG+?Iu<`4$<41A7+#N zu{iL`8jf+QRCXBG(=c>54@(?hJFniSqc~8&%1TRaE+Wers7vyr@?UX3ps3wQA$IqU zhUB&2N=Qw+++9>nf;~4yu1f$0uO2N5mL$Nj4&Npa#GeAGsskea$C%xBn)smnq>39Fc;9< z8vjmKgER#IIe|CJOXDr-RB_>!xTf}f31$C0qY+LSfjcaJ(v*_Pdk+&q7g>%MiQv9#Al8W#7-5~N)wU^imX8#_0`L>J5xV7zYmtKZJ-8{ zzygKhI_|(fESY8z3rZC8C)h9?Vu-N~xQS^6R<42AJIsYtsQ@zu@=g+xQr#QpFIdf; z-^g~)T-fg{9k;2YY(yHDsC6Q?0Gb&dD5u)20${`$Fyubr-;HNEkNpiuI&Tg@tB=t zvJOs?xFoRdGB|cQY0HVAR7^4I_p3@Z4r@l+>l!;i?cJ3DJ&&r@YTxnagyAH{G^WpN z0Y9mz0kC_k30Z!tNbP;kZk6WU|BU+oJ>+8{DI;>1xK;WTKG zdKIlH`tp@V(ZoM?`ei?Y_?~U)>iwk(;}WU;6qBZ&ETf+E3~ng)j(M^__%<^)P^J2& zIeeQ`1{|A1X0qhCm`=Fr-t&9$gQk-mD&666Xz)ZhE}mR1-BepYfMvj9ijZ$CmdhS{ z$zR+%XeX89cw)Z)F)6rM(b@sd7nkd!u}iDYU?0~n4vQ|24aPyKcbPUqSUJrYH?l-^ znTITKtj*j_)Uxn|}= zPjFwqyj5?rBismK?IHj6{)ip(U*MF!zMqNj#ytpjQ40=(#Yd zhG{%kWiLp&S}9lfv)z9quF?6Xh)0TBSFK40qS!Cv@VP&Ejql?(iqX<>1w;tbXa}ZQ z>rUsrR!wyEP)J{60#_~e7HOUaH{sfnO?fAgJc>QtU_ZlKl6yfvU|8v_H>d&fnWj- z>)5hHel2oP1x{_{7R`JNQDOa7Smh(zJfvRzs4&L=ff!W!O&Z0;)F$zf+Z;D5!2^bd zv)wUZ15B->zwpxQI!dUJdxt8PNlvSCs=5<~Tq+K~gHaoc(7&jGE=OCAE*>4n$GhGcc4En=GVhB=}ZKxD8BcvvLa$l8Qthwl^Ii1H!Ud= z4fN}kf0lRw3+$m6%rEd{kog3Q9~kJ>M*t(#II5_pLpRtj!Erc)`}H`cu(dI6bwOly zX%m4Tt+L`=f2UC@4qF4xKo+8r-FMYxTo{c+mxHOdn1%~Uyv6>D6wStUYHUi1b230g z#D1v;vSwb3d>UU&J}L-(hq4*xkQDy9RCOdq189e20%62G3vQl*dXcKI@eY{2FDtO! z{;`Uc63a@m+cq#Wd(ow^{kaK3C^4@feJ9-hbdJmSNgP3 zLP6_7nURQmwo(;C{S?jxARy+vPeh0mx!W5Nr#mKK@Rgb%CFH~nbmenE$OFFo?zAG+ z*mrd57&6*zx?t^Y0@(KhLfxV2ZP)$kIYrIIXr=QtsTQM|qE$0FJg4PJED`dm*4b?f zk+7w!O+VySZGIM)hqQ_pFEriQF7LBS#+ueOy8_A-;DRFrk&+SHrP)@|X13t&(Pt## zek{pDbW_SnWd9PKKHC0I1^`AG^EiWq;1&Sct6xT%WFTeSBuJkKQ}V-XdQt?C^?*W@ zo8b;tpq{~ZqT7GLizA4QciUMSXW=pmk!?n)deDW76GNRn{{Wkw0}mLT$LEY*XWTlP zmEpzr6pN&4%TQ1(Pm&@+Dk_lZ9|@y;a~V;S<%&|)#?fwf=T zT2TbZ6?@=0cmWj8sWD1Uc1i#$oVw{7KS3Yv1(=`y7BxLBZTF9YAp%JL1h=yhfZ;c(>N!z2q6A5ZXhu07*2=6hAJXXz^%Y8F`alj=mn_oq zL(g1|fL)P3pK{^%=a!m&gH793MgPw>w+GJn#=YI>V11#X6w)_P=z^%V!+rcnF*i-BU`^RCS)MpNlZU)0JR1o&?~axO!# z{p8Dx#AoR_*JH8V;i!{_q?2VA`T@yA{!cn1F&{dQ&lTtL$Ul56wp>>*^3#Exp56y! z#din++~gl>t=uCs-vPYcmiU`ZADb0HNg;N#0acwlT`r5dOQh7BcxTO?05roq zAe5yIDe4FE|E_iIf5^hUn06#NWCSNk&w8Q9*s-_s9DjG=lp@Kos@iVeh!EE9f(U4nt z`>-VG?LbROf6dRVV$Fqj)<#n`U+39Ar+)Em{)(M?&t*0rUH`6@&!Zbwxw&QYXx z6*o$BxAhQjT7M$8Pcew3p$33kjRuSh_kI8{3)BW&&9rHDu9q4(9~i_aUba2~Rw=$U zgU;7zL8|5QcuY?l(1Jg^7<@93{UGU9JR5%&$`$8&m0Nqrso*p9mwcf3tmQ5Bpuucj z3@}9z_zCs?J^QtO@m402@ypykSh)CMOZ@cM#KL`)^)n0PZIwso0F9gP3!(bM%R0#T zKNhW;CUQ7~D{7eo_1~YB3bx24LH+r%?Osf@;n3rWhz^Y{LU2a zJOBKyMFp$Khza|!TIX)ck)-78_%Ja z)xa1RHkQ^ovok_MNm@(i+W%MYXqL}oC$RdmOdG7MyIdM7G&^*zh;C2B9w3wq`o!cC1ozmQHR`$%#zuC>6F8?|t=$jTL+n4OmdQwd_ z*ekycLBCj5c`7(xK>^I($;vnA8~#g@W>ht1&t+T8>{wlPyCox&YR|7+;>G*W9iTb| zYj?qc;fQ;*OpAwIjR-bYT9HJ-esS(F3&&L&R<5826!|*<<3*JHkSzNP#+3cTm8rBi zfXaM-x~T^=E)I&^{|XY^ntP~v20_L=B9R@eN0k8qDB0VBNW9*Plh%xm9@nPRC2bu7 zFFpPfI-U3S9xa@nH9G5RYb}w!X>SBWZxLCkujjDKb^o*<;+R|)m%+Q(4CH6j$u|4c z3mJ^^&)AaGK{d83jMrFb$E7cMUA^&92o_|VbPTLRQ?EHQbOz}ySYhI)vtEqRRHneq z;I6afz1&9KS@qRB2fBCSD4m9kpm)g)#01EG?Xh>ykI|nK9AyXv(g4>YDRG|P%eyki z32fY;=6tRRS$3c50q>Y{+K-;hO!pt@@PiLBF^;@MLl`qxWxO0-Z~`qyB)~fjU~6Mx zCl!pa51X{f2)Qe|rOXVxI09DA0V4VI5U1;S|1Y5R1tdhkI#sRZ$N>CXrL`+x4> zd6lEQ$86eScxnWhe4sN4DkoNvp}z@ob?smVEV3=iug%H=vjA7!Z4qX+zk0#o{hnMT zz)h;hbJ6xoW@)xAbGypAbvtZ|1}>I|=ig|fqvs=hp%p>6f2N0>+M_x7mT51m9;+Qh z{`^gNL5+EBGxVh-^#==yHbuYRF|wPT{*}gK&%2Y!{)s|TGIsrWiRXloX0ZC%D}&3* zox##aTX)a%N3l1FM0cPzYn?Y!NH1B+Jq1(BpT`~?ww|Ps5C?_kG^q0^4bNsE1W0IoZQEP1l?HW5a0+$vcgi4~;`N57&u&hr={Dr>iZ+QnTF? zyr=1S65(N!ZbLymO!nudWghi)c0?0v3|Pg~3+@f!Weay6r+dUt=Yzc+g+(4Gq7!RF zXsZKe(2xYMorxE9wbi#K9@g`(&C%{Q>R8#|BRcn*tvoVr-uQ}i0l(f^l*7??GhuM! z2M%`Mw_%mu8K5KVr!2Yb&%7k(`|UV!|?H*kO0-gl&@94LOmNI-2tCYVyC|2|>tn zm4fy@du>)>7|~SX z7H5W%Ph`ofL6Ur4Lv^xjwr^6~Q%wMU921@EB<|D2(j2$-K~oSD*Q=PYz(C&-qd5vX z>ZXe;K|9u-HY}eB27Ol28H6xvLKAttd0ro%E0fZRN=Qwl)#lx*zFM!|p)@Xqe#apv z9*WmC`ZJ@rYsST>M3k(cBZngxq3Ghl_st2tjJoO}eu+7+`^GFCHhkI!JQ-?? zs0a(c)f@X_rc69Dr&=d5>(sm zKD-um<X!ZN3@J<6{O$<%x5Z#>T&z;B5xd@Lt+ah|TPf^2 ztJse*x=dOshOpdH!}onULe4cXf>aK!w3c10PnXOt0*0D%jUPX1Nt$}IVzM58_7g7* zpx!u$-n&@I;8hXqFw}-7&#f+valhVw%fAr1A^e{3gG}{+qe59zK~Daw<m zCC0u^oQ<_pO2!JSp?O@|Ua5UJquP89$0xbILONu!*RC6Xk*6yX5jo{&ElZu{5r9$n!sNR=0}6AYP?k@8w|w=FJO9@MDk};|^mEatTR^bHJX! zPmjlsbu5y&dc~%H2jd7OKI#~aS2f9H)eJiDae6o~`C=uj>CS}=ZV$`caleK4Z!81m znae*NWz4-Lpdjg_&@@`=eLoJ9chjc_CrN17Be!C|1OC&OT4z!5Vc>Z8`nIw5J^yQ% zK^uhEa$V|EFeX+qb|L4Ys%k)?Abt8VRh}t@M(^&q=tTp*nP!2ZZ89WT(32E zMs9j*D!Ofg@3@rYV|b8fA@07C{ki#cjn46x3#2U1ev{Kf`MRbbSD6yncf;Mp<{;2t zTy)OT-$r#!-IkeBpOiop@k4ouB)dw)(sub;q&URyB>K*z_)Y~SWN+8YtF?g{Uf@{d zQ=>e@%!e3`SZB7c04&D+(43xJ0c_jc7s!vqK7Q4P5<{4h8-dC*Wu;^1z8L2P88JoK zupjA{bs|mZTZdXMCf|c4)~22F3{hjn>i9q5NGE-z3{Sex31Ar_f&Hd3i@Qz(o~{(BnGzC@M=NRayA@& zcA0_Z>e< z>n_bJxLT5PgJWogxZ2`zzlF^^@HBdTsw+gRWJ0acyW71cyd1S>pKWl1{T?hRnh8VB zXu3>CocCLjDH-mme03(b=d;4rTAo3CLYSumo!Xu}Ji1yDgX$`R7zH)n$Oe1gwO}EK z&>-y8k2V=lux0WZRo>?C2!$J^WEegwPM{gW=baOUT|m4H`$K+3!eMY$kQ@HxugAk9 zE^aRUpkT&Q3y#mrtIq;dXvaoSc}qjQc9l8QP;G^#m&D8Fjl_E;a{fegYh|#gV78Ml z`>10U3_n?be7L`ZM~-UVZsS6__U_(r?!Rau|9$t@C=_$wv4o@>JrwGnwl;Z%Ey2Wi zGPne|WzNXJdZyvYOfIY_dON;fYja{`uG9^I<~*S_%MQ)MyddSjvsL{xn*vq>VYQ)a zl}42v@k#4|2{>q#+wwYsdjbSXO|1Daf%x~$@WfO+yrcrwJMAPKa&drg6Z)ey+v^V4 z;|YG$Bslw2^j9m=*SR@!CwdQI!gtA+T%^H=e7tf+%FF=&y~f*Xn@>s>M78ws8z0pW zfs`MPNzgw}L0-WO<94mu;FJ#?6U*#VAIiJYw4wPci=%cn5`CdBTxVk>a*XmSUpKm1 zwK*13d4cU_pW(uSah@F#^&ve(F?*q|tq9S`Lan;2h z8Jp=SGJy4?gq&=m@*S%;{^9@4aj|terHH|%CcJ~2FuN9T<bsnE5`qqWW7bFc5#~ldizL1;;>CyZ{VT5IlZLXTuQ0l9Xq`1w%`3L zC^z|gB2`AR>pmk(x@dR|t~|g$OwCWZY7xUNl-wQM-Tqy0q^F(l8yO^qyIQgxL_HGx zD5vPVmn)N--w-+U0ImVYfoWKD0{g`Im~9zRA1t)+!`qRgfdRPK_b;Q-X%O>Y0a=L$ z;FJ564gmw9f3ObBChS3{ydd+n=-1=^$21oi5)jHf8kt17h={FY5KhQgwhEU>Mzv&ZL*5ovOEG>10F9lh^_}_m-s9wd?o&tV)J}&Zo}Q)=iIX-8u;9Q z4Z>+$e6a;Z(5Kzfxl(5m9Re-=NwenHIOfP$tU2ToV=hBn66&HEJFBAs{9R>ARefvh zKgqA6sQc__hEe+Y=V(aCrnPz>p#+U{q!Sq6qQbV~V8*Q}4>mDIY9X2!s`7C1$BXj^ z0XWs0-^2DKrKQf)_UcF*dE#*6y1qFfE67oIhr_3z#xV&$&EzXRKh4HboCF?0#TGq2 zR}EIa36vNr!Ij9C|J^}p{RAJ2CnZZBtT}XIoeFFjL&eCJ4y(2c`ZT8YAMTi_AtGAx`c?T;(j{Nh zdl1sO^w3@E&RSncgRhk|&>5?^SKGOB>?w0%n!h`}D?= zHx?iI7mI{}K;om;$_GPpX5Az%zqWYTUImKSDD!$e`>(CB@?E#b5#Y6Y=!@#yb z(Wgg5L$v{vcf=A`nUP79V=nNyv&TK(zoUi-=%}{9n z({iftuom(f0JsG1Gol&i9|hd%LVQW$f_G?)C_l0`4XeAkN&oGqLL zE?bwMO^Fn(Zk;A;=ih96UoFw&Dqg_S8(9^;-j@Zc*NzG|d1jzpCGrAhB3YTRnp+dx z4q4?x(yB_N|F2KabiFYjOiJN0AR-C^r6K|45v@Ci=H|QFrdWxbmJ4#Vfvt` zx`Mtne^P(pz0IGx#q&tw_X9);QsTOp733?SLxHyCfh>@==b@mPj^>J_e&f#>i^W=5 zlwF5VPR0<$U2DLy=$j4a*R012MwZH%aAt5E#n|{s+ZSoK;;O4BVgtWb9umDp#> z(U8BG+)(sN$Z%vlGZ$Gm94AIedHbUp9rSvXo{8sv$|ns^)(mhY_;IosB^38vv)a{~ zqR|kT1-t4KX|%;&w7o`ov`ZiOf_=Oa+MwDRv<5fcmDYTE8eQ8B+??t)AKqgP!dY?4 z`3HUQJ#Ur;0`Zkhe$n@xC>C+5=EebtP)tWM#){&)vTW1>b$F&xnPM=P>lKRUG5ZAa z$%k*9Q2|=70OWZ;KOgt&Y4|h#u|((A6}U_#ck^xgwN#gZmV=yI@g>24D-a3b!~kGY zvJmi=9j?>b4M6!ITTGkCNrOsV7xQIL047;kcHqA0ANX(hzJwuF8SF1Tr|rQQ8HKU6 zJN)kLI})<>@H$WT>i7IPglmi4m-hVCruuA?xOp*3gpbuZ)$UEen(z? zN9enub_3v{jJD|_uloy)uJ`fVfVK;}O>>%s1Yi94(^I>{f15A*DHoxY>@)O` z&WwytO{eE`Hg?A(XwUa{>-<5NN;ywwJHw%YmM+$iXu3m{4?psG&7Y#FEs#WvlIe|? z0I3G@NlRiC4l9_$>%b=a@tWzsNYr6XdtkOb!p7lvou0YU>PC~*aQnJ!I|Vwuy3jMI z;Hk+dul#iOobF#EGN#RAxNxfV~&^vM1bj zd2VFD7O$cM^xCBI1<-QB%0#38jLctc2R#ys*)_~KI{|_yo87jhVB2$O$^|7;GJ6N-r#n;`mMM{6O|| zn-Rs+%_kZO-Z_uO`Hf!zHgu4OE&H~|!GJh??ej~>YcxRC3Z@teL<4G}`$NUU{Syxi zl4PR>jwH3xNI2#K86DiddcyIo&C6mSM~hk7vb0cmgKJ6+=Z~D>d0YZ(R(_=j?pr^+ zxtioB`|zNj3@A0MZ61*MnBOOE67h;8WE;`9V-dbZ0XTuJWNmKOE0>C1U7IqYXEp2K z;xoMKvDqM@FGeN6=gT$$H{!EM#vcg<2@I!3ZR`eA>Yu-zi}Q-Intq}fdeHL-{6ml| zV?etvnrVAF(K_kyZx)ju1&3!;>J0pbN38R@c6q#rZn8Tk%kP%-Pz8(7ejR4dl$W<; z#a{M6oMTNZ$^p)JZ0y4F>%2V>oneOexMI6NK0;EDmeay&hd`juwc78wwDWvsuE9wc z-0S0aJZMNEIB(+nhu2=(QGF|;*8*-yHvJrP3nJ*x;*D3kBwuxgqn%H48>(a1L~Fma zJ#pQTu@o1zUlxY;=iGK&G|j9ThcHcO^;Y3jy!|M=N5owN1et|RQLlbg+~0t%a5A>W zA7VWn^n~;{nhI1r1I3ED-hex0Mcu2PeY8_sq3TojcGAd=*QWTo$sEjRIEu{K0U@M4 zBFe^QpGr_nPkT)e;zk4TThO)m^0K}%d~%9nQbESN<_bJUsBym+)vV*<=+g*lCIoPx zr%p}886H1v!OEEF-saTX%buH?IzIPVxZ5}NJcSGy2i|x9;nK@Iks8#<+nEsZd>IIw zNwT4B$QwUG2K>&_?dCh@tzU&d&$q=Q%+w42-px}Y_#gq>Em%8Qr11m~_8O5=>f|sD zQH+UQpM2Cl>umgiM~*0Z_F(YOeU}#aeAj+ez;ru}*Am06B=!<$0ZY1jc5DW+ z%lkM&vALGlWMwE6IFnc<(N3+QH)9m(`oeG8vDen1hHDZ=0i`EHDY2oPN&*^?4;2C!}G_DckT8mzCTQPFj+i-jc)8A)O-rBYt># zpB6>n1}mR`de5yhjKSohb(sQ~82;|RfOKt}^6xP_@=5YQ`xF`&JyRi znGWP5yMH6Bc3q}!1kQj;ElT()%Fwejix z!~lJkF)II+8aQV)uE$f}4+oVfj)^ai&*VT%blgJok&du#&_KtiAF{S85pS5p<|SY# zkjdtgaw_iA)Ulo&IKoWS0!(FbvV2h>~`)wP0TC8nJeVn z(_jG5Ntim`))&dgzgL=~ljm4aKljO9VU3ImUw+;{j!zSGMRV5i-2U*KKThqw*&f z9B-9H@FY-Lj(p5FdI1D&lCmH65kXK^0f?UISOtxSnScq_Mgkd%gci62 z6lJ7j$G)8>F4f$XK}HdWg&}1|(4Cp<91gEVfp$^-Y4(ple>z%RXOIqx8jN^3>y2b- zv(h$U&-nT)B4dlz#}Tfb6T`ZubL%^AtMt~OD^4QT8`}}J?&uSQFq;azJNPqSp6)(z zQ+0fR@N0Sww)VDtE@|0h|1cZIWAt>KwH$aj_Z0;bD^(!ir#cr6V_f;W@ySM1fSz9z z`s}t<((>r%CnB#PFAo(3l8(hF(N3lv)tzghOnL`61R9g;+vseDD2V`F%tJpj;)nVP z1|Z+%J<|gxjitwQ5{9_)9`D(BUDcxp*CvZp;Hu4^#{6}KvPA-uf4Z9LA zi}(9)kpsim5BT}byafC{IZ&*wwq=|?cUb@YZNI*ce6rrk&xV!y6rpI>beXs3?Q~2p zR-Hnv5)f|?JstkYlJ(^B!xt2ACjy6jhN4H^&n|%4-b4wO&I9Iw`3Ane4JWu>_8L|< z^yQ!ihper-VFHB1W?+tY27EX)cqRxIyfR;g*c;-Yg##3|;ede7zKid)+9cF&7y5Lt9%EKkv@v0@+Q~Mfi~Zi#|;n` zSad4hg;Z!c@Noli7l3^oij&9ZHLQ;)DaIV-^TI+8Kd#1I>m?LK{ptDMlAPEWe$QR4m6ISqj{!m;3htjo`$B<0gDkA7#?$zeu(!4)Cy&|D zyhkF+1~_UpS-HLgBz=36gsXe?n63LY_RLB7C}>0W&8HiCsr>$A+U#k=U$;NcIxUCy zZ%xwYepfd>ue-{~ub#x9o-Qm)1fEFeyUruGJjoCT1K)FzZb=^OKZOiOtHuMe!j}(O zI_g6pYE7WUFe!UP13LgYiH0LhDOFhy_nS7f`Sf;NZEbnA(xN3F1%Lt|D(FqEqct>Y zo!j5spMz`zupC2@JunXQX0;tm&yO!tFnix~e?C+m8H={}r#_?lRP=|sP{TT-iWZpZVJHB536XQ>t^)6?_!3*iMhXXz#-r$a>ZL1$6S-q6mf${$L z1bX+saopW+&sOOL9mYk02!o|)CO8dVIY#rKR74sV4*uQ(AS!$-x59j{Q+BfX{oAsm z4r^{NE?V`Nes>H~6^Z0=O_=fFE=7nYWdW_Rq7I=KoT6Vxw3aZA!T4z;ezYwaC4er>ln!j~7wO8R!Ta;K;r+U}yV!Di|_ z?41r%v1=dBe4q_`WL+l$yLt1!-mzthhErt_NUIHqX^ybJUc7xoA9{o44y5}0#tq+$ zSNz{c!eppE5WX0Qc+s#!o>(c`U;BavPe1wU)N*E!20VEg`3DkGR0?SZ{80^B3d3L0 zq#pmbn>`-oHUxjxll4gU;}^7y=8qn909e8A_|2bn($a`l+^`A!zk^v@Nus&$Q;CORL=%`(V48~(pIW4VP znZ!i7v+&^~e%HpPfPezoHFk6n7rq98buc) zH)T<#6xF?4T978>DM0D(liJY%i#&de0+LfWTQ+GmhO(bq$k>o!OTrP0{d;aX<(NNK zwSZnC2w-peIk7tlKez2}dP{|U`KKFptzQcicU-6BH`dzYb-HI_ zTBIe2Qb^Z-eyzAyb5VvEqlCI#Etu)Fy4n&CX*&~pL}$9*gCNXLUNKN2>5bo*tbGu$ zZ-Ikg|6TM?)!<5)pyA;)V?I-c+R>2c?{S=RRSs2J#_h)VAhpN5uc5{vo~5ss#nm28KE z{)(YYi<+?$SK|q2lhv*)s&DuGc8L0JG`sswjj?5KVsT+K9UD?qTf(1vS=sRv_Be#1V>GkofP&bz)*UxMz-h5AwUB zAmd1q4j3DNvA5%0~Ag)T2Abyv+*@8GYD$Z>wFh zB~A-@V$YWj;L)Zu-hM^H{$o{dQmcp3po?-*kKlDPzmVcF(di20*-rbDTkzjZVJcqm$NS@l-ofjt0@+u3oUU*mOQ1oos5 zt7h{rYFK66W1X6|D2gNNTJR-reZ(~NDavG2Z-)>~UR=XR7d6cPhyC5{@cy9da+?EN zqoJ!9aSxt~iz+!A|08rrT$;x&~{5E(XA;J$YEli5Li)2ND2Cu84H?kHgy4jPHb4D^okujF%rAG!cpxw z=M=t(a&1}3f!ehrRiIsYkL4S^Xau_fUVI{L9uGfq(dR?JGD2{2Lc%wlNOA=~1c3;Z!u{{a06R;_*1Q^ZP z8#XdF{wSI%UDl&Tz|KiyVfcu2TfO?gQIWjR3V`b^S`WEtdhn-3=9%^ zu%avgqkx7|_&$WEEWZdvCfma!@nohAYH=k}{t#yt9LQVNJeZ2jV__exGxzDAmy zAE7Q+x$|9G$6$VSx(x>uKR_SQN`%6~4}H0@$xlpn`u`60XL>pPvSrp@K6iZkVr*eC za0{mMs?_O=QY2h=Z*lQ%YZ3*z;rUfk&?BZ}dv@~YHo8w-Hr!hPSB}k!(N@=%Wuf4& z2jzBk3oNvKVcf2oXnV@zCNEdjNs1ba#rhD+jLxj2t&Q0+pJ zGLkcNcU1_g8G@aaCH}Z!Lt`cx5Ysa#GLk`ekJge`D$nI;%`bQLAx*YsKWD33ntjb*wN(^9yz~_55t0 z7F0pr-wk=`GD@daFhcbcTB-BNhl{ijUw^eAt-seyU$H9B@~t3&Q8Jwis68M*nY%VP zJPyLx_f9UZQ*c!0$&F~@W)8G&B{60TPo*paWQsnYmgRll0D3fEZFazt8ef|am>8(4 zCM~*C&-$0QaeCFVr2nLM=VtxfgZzOw;1PBMgU@yT-EmrT$d25gH-fOlA{7>nLrE@U zXNCcgQ5YahY8JlsG32|85fIXppoM z$3E|$I+eal3;(2&d>MdCgtYF~$>p8gK>aN&ABw#iZG|J|r8jd*pfdta+$u=rwWW== zQZ`wwn*^Q@m5BRC3Or4ynODode>KxntIFo~^syCyUe9A{c~u!eNJCMqSd71==cqpI zWhV_8N@&0}4OIKqshELtD)j{skB~sdz+C{9^FPfA@FP^*rBF<%#Y1lpf@UcUmi~?b zM@npr&Eyd#5yeCT6e5vmxyWf+f^UO5Xcodd&6^%`Fu%avy{Q_zLz=R9!?+c6804J) z2O0L!^N+E(H6LTIbU08Vj%?PD<@UW`09zM+Bcqg-t2%aT7l9@V-RAy=1GW50#gVXo zy)JqZ!v;v?(c=|h5x%(kQAXtD;9R@q6IR$9p4zCr7@2PdiUH3G zzcaHLt(wXVWHj$)*1p`oI ztu!E=*twZl*vbVekefqDSjm4~a;u`AYJ>ot!~TKmIEyDTb>brnU zaWA)-4$I)VTCZ#b9sz}fO;s{g26TAqaXJk zA<1l6KMLe7KwFE1q#`na>SRuhD#V_CLl@||copC+L=(J;aQr5>*t*sKFb0M&Z}q!- z2d*}f$b?a^apMirB>kPp#GOrK9(nwIQ>U zAiwr^>UVk;PGP8%IK8AqyZNvC@1P8}VA5;hTX5*bcfDPITX!VRvo|Juo}YYpBI9=1 zNX01ICCT1|P0IYD9qeXVmGz@)%pjZuqg#cD1FH$76J~{xDDb#gPyT_fif|s|2w^A; zN2RhqntyzLDR71Zqto^c-65Mq5qU(_!bAhAZl{RJI}Q@Za+hKc=avZ+t&7RO9$s&d zzx_oI-ZL;=70ogw3Yx~>{gn&@wW7#!qi6&XcUn-vSBMH`PDd;(z`N&`9^k`y!*UAb z>_UoK=crklVltm?5qm(H)PTBlDoheTq;MCr@QwLIj5tzyf}B+J`>^~OicGusuF%~+ zFSrUD+ZHq)@mL9-sB1ggV=#WB_D`h4F<@NyFg3oshlP3j}v_fUE_3 zy&FvqQUBO`ptp60bs{jRqe-JMhJ6qZk@^{gHg(Hx1WhjEK;7s3bcTdM#9`7I1PA|r edcooT6j(dJdZqKP4N!c6K=RTmQokikLjDJx#3W<@ literal 0 HcmV?d00001 diff --git a/versioned_docs/version-0.8.0/images/logo.svg b/versioned_docs/version-0.8.0/images/logo.svg new file mode 100644 index 0000000..0e459ae --- /dev/null +++ b/versioned_docs/version-0.8.0/images/logo.svg @@ -0,0 +1,302 @@ + + + +image/svg+xml diff --git a/versioned_docs/version-0.8.0/images/serverlessllm.jpg b/versioned_docs/version-0.8.0/images/serverlessllm.jpg new file mode 100644 index 0000000000000000000000000000000000000000..81d88516352589e7742c7a1bdce0639194d4b967 GIT binary patch literal 102673 zcmdRVcR*9kwr`{=2na}#7Nn{)LFpwQAVm?QB1CCXibw}RiL|XEy@?1Y0#Q*>TBLU( zT`AI}OGyF(LLi}pKnidBo%8N_@2_|6eeeBo2X?YDnarLwvu3R|zqR)M_&yHOZHDr^ z4S`r&Lry^;kb@A80|F3MkU9YVK@NyP*#D*>5W557|3Nz+IPuRmED#8b;J@0(dP3O# z*&gik?_&AS&;NMmdg`*@bW&YPMdFJFb5pTB4ZIRK8I<+SlR6El`05Qwi|@ZGBxCeqh$ z*h_N`fP8X81R-Z2`nOzz0*o(TzWApHKYu^>Kkxs0p^W~S2?%6ZDbQ_uusAKlI&eS53kCc97=v_y_&t5AE^~ zdj1c67i0o#v-PLF$KN!|!9TRiKWK?Rbg(<>&wU}L{s9l}dV1Ummex3Z`n0sMo4<>@ zw9SK{V0Ygj>GOWB{&xfX@7@Y_cLUr0V|;&p1>yfk!#^{2LS0u^=j7kD{|{gOtiwOL z+r0Ak{2mbQ{a4+;ZsdCJ?)`s$I>2%U0{K6E{#PG4q7VrDJt%7Qf7P8$hCp5`Lm+$^ z|Eg0^hd_>~K_JaJ`%{o7pxFE=EZ{#Y3kwS?8!K4Y*xCLRb`JKx3&%eT=ilYvpK|Cw z%HO|cVPOOR4{@+_{73!2P3%vDQhsXx3xxk5%X60JtOsNuEc^#p`48;3A9xEcJNw@Q z{97ddObj?qaA2GV4{>oHfv_B4Wo2PwWe3L#eiQL$W3lnG3mjKB<`BGci&G}>sK%qT zHwR@+%36f34wB@~xZHhwh)Y;R^q82ug5rsjr_O3>Y3u0fnVvHHHlB{eEr8V|1bRT zgZ!|tv9Ypo{^92UOUNH)`PtZyt8)k#U*WtJC@7=x=-^S4v^Qlfhh)!OB?-CQ9pn<0 zJByPi|6%EGj{ZG{9{-m(`VWTwgP;9L$YIt4;DWL8Ltqdl^TgW($bW}xf8f8-SO@D%z31YC?0JaxOcQ~X8NQtK7$7N?YHo;yvpkG zNbt1G2xi;i1@R`lNv`-~)bHyLPrloeaeVf3xbGO|rGXReHnVFV!q=mi(tHYKYVQ8K zrhYoQj{feYb$APYovK_z7ubiih-0fG&nyZ0tWY>J_aPfJ2^$YmX}#`!1%O?9M-(n@ zuKx{Mpr5r-fFN|WA*reWT`-@Z)(Phg=cel7lP%gqcjDJ^!^O|0ESqE0v~G7Ude`2Z zj62}r6HxaBkG*C5^;5@O;eeN>>usmQW_k%*(_L50gnE6BUNM9GS1(}y$N#&TznGG7 zb@TG);o^lKnunepR2ldt_v}bvT)zAZ(ASWJ3wb2BEhFQwqyYaeQxcqcD3QR;t=pJv z+F7IJEM6^5smuCaC#gSatVikeu25-dv_{<`GCbz{tgsqljzD`6ZO-fAZF&;X4L9@njuVB$Y)}_CW z{nig+0=Wu*5jOTAD}l($tE4PmY`iV~**@fPxa1{`BH%vfo-WuCuQ6D2gX$kb(T?9v zJ>pt0e%8CPf0!8W%TQXtiU2Jmurlk1cj(9FV8?zZ>50Fnq*?Y4Bz}9fX^d=1hWmO4 zh6PII=J+a?=YEXSJ_G%W(|&6ih0HJ|H0q@pr>`pL^BbAl7 z4+$XxclrrPwLeqsu@7OtME)FDOm`c3^HU?Tg(o9cYSy(TR%p+kw8P}FW0Y^_KBJ!} zFFpwton2RX*@S&*L9?QSxlJ=yUtsE)Qs9;;KptGZuib%85^F9#QM;LR%=v z+}Tv3+VSb&vc*r(^MQdO9f_d=J3ui{h0(AN@x@5SrgkDl4Lm<}GF)oMFf+1SN&TNA zS(^RF49?@#jygxwwAbwDB@yW%8jbz?kO*D5u{|le!9FC)8<=|$$$e-aa)>t#&nHY= zVH6eh2(M!(cPAnxLEUIkYh$xWGoY z-sQK<`IHv{&8P0)Q%Gap$iAIk=q(aoaEG$t^wVm`C7Qc}ag@b66q?NBpnFl>aWpmI za0^mo4UVB?llHc}@kb>s(dxnUQGI2B?zj}&8&>R!kfgD_1*$1-v27i41MonxnhbC0 zU)hH!7pu5b!)d?duk0a6$Mzwd%FGdNas_2MYR)ezKz^`U*mK3D#o=!5C8|mvfyGep z>!Vm(R9wH;mQF}Y!3%=0RL0TFb!T+ox9oZK)&WX-ZtpBrD{Afr6%8)oQFwevnxF@1 z$E5j3+Yv5CAIR$kTzhY`4jsZ4CT%9r+u99)(KEUZcis!GP(AooNoDtc+Mjsad}60H z;lyi}|Ki2BKk|RCt4TG#-Qk8&CF zTi3R0t|_<9Z(g1j5xO7%a0zBREQLCUg!c?fAjt0EDmc^c(h!M}N11*0;qMqJqnqtn zgZJ2e!p2vppsNqGVUH+_JBxT2U3iRn$d^=?XBbA{NQ;mmCrjysV!@%YJ4sOP?24Y{s;r@7k;k{(I?M76XORKmE z=fGGyn&j2LTEs}4gGw>Sna69UjOz5L-sX2tw~h|`?0j#kw$hE}&=VC!tO0Z(HOdaC zpB@KAc4q;SuEV&|dqqVwUp33&QaN>tz#v^!+Dqe`Wm8?sIlLF5b*9s%JLne4McEN> zwO0-*`Z*KAh}eQB6l&Z)q%Bzb&)C=^!S39`)yJE~s4BRIWOVTFYGM&TxR-@%F|~9d zQaTRN#toE(Ym&Za;Fij2ZohJ8e@onIKaqnVIgq)TypO>SVo>i4qtAwkQ`Iy}P;6hxWzK}BXCU6wFitj^s(|JB*RfckuKNg{fc+Ax8 zL-5dF+xw82Q}QbceeUh!Cjeg9BPJTHppWRs^25B|BBE~>#kks?ZYxI5z z+Pb{Ez^E15?T?2N6=~<_j)#quP&@6BiUJYqM5n83J20e?MgOw%$wyO~ZQYzT=<(hZ7IiQ0N-yZjTe^ib1Xk*V<^wT&3w8Go z)L&yPx?wS<6~}waP>mK?wR@q^GGryR6Uog41(&tafjrh}A%=WX`Ksiq`}bk#>bS2m zNXq<8e$M;ST>B6?bUgYg6vpd4I-H% zC#@(Y)DV)2tLJgDIrf%oU`mmv+&4-rq1;508I>^B=E%}I=jB?sO@FI;vi-sb867Ww zSB5e;iO&HREG`!|mpgl?S=5Rz?SA%Qv1B~|f!t@Uvgef5q-JU#S+UJPjQpfE8cJOVjJHpK-_k}y+;kvQmYR7OGj4FE8qa=@s6ylSItm<#U%0pSxFJB-PgYRP!-NMbo7b$G zA-~?;wI;RD8KHwYOU)-2;5b{LyxNIOb!z?vQfcPPzR{u(o*Z3P9lkRs6bCI7KV#;Q zluw|x$N>D-lZioZ!v(BixZ8=Uwy=r;=id%|9!+&luT$4e{rPt=Eva#{N1GMGZ;}mW zG^rJMNvrm~M!Mlz(LI&vHr`r|UZ-lzUuUR@WnDz{^}rA$ADPRasJ50QOirAQ~3`Xul5K9)g4pvco^NuQMMX7CBvzHD^ z&-LA6xS)&T`(60JJ|qcCvV&7kEOvd!h?J%y$y`%?xucU#T@>pwyn6UI!S6%X)j{0&k8wid(G`x zD1;N^MdQ+u(1OlAM#`m%y$?;F{^=1owY_ zru^iTOsbs=M5s|lRsLhwW&SHJN1LOK+7_UKknR~?mc_*a zrv^-}Q8uqf0QE?5RDT=5OBXLf?Q|lpEbo~|yBjjLd$Gy^Qx4`sJ+e~Waqm+S#J zZzDBPS(Fs&I;eV9zmbF|prY*}#L&N)K8oeA{ ziFXRg1@3+YZsn@yH?#nBY%=JB??Y<*+u%`R7~&x^6t}Pw+X#KzbvP?DCSSAU;}wI% z!?$EI-LuU-B`?d}=}S*Sv{W%lfR-U>Wd`=KkpNwv5#J~uShgJxkJGe_3ZHc;=a~7f z#ad{6<^scLi!00pn_$0DmIV zz{YA9&nS(QtV*3bc6v&FpQ8I({8#xk}RupP)FZ$5M1W5fOkal?U``mLo?LDIY;13Dv!Gp>}gija`H4VAay? zGn_lR(5WM1Rg)XV%jL7GvUw&n~ zR){8o!w8{VJMHDM@!`imI(TK@8;X7W`}?)LLr1qx%DENs1q31qNi6rEDeCl&ewge@}2Mh z)Ef9aGb8IKP&%{yPW3JJ339@Q$(zE+r*n+XKpDl2O1B7ohFV3xL;a@D*swM1Wjjam(=Dyk5eILde^M6aU$Zb${$X zYmHwv8Jaz()ggfpG0QgZ#}`+7?lKN93UT(^Yw@}S(Y zt-}j%pS$kJ&KF#Lf5RUBVu3^da)2HgIpnb#0iLV+nd5-?v#AbPWy%|GQibVrPTJXS zO&({nPK&*>SCua>GPC=Bsgcil?*WYGz+5zzbZK_ts=m|J{+1PjZ{%UYLR9|k*R)s^ zF`?w$b-!!H;$yV-O{8QFDmDw(%-OG|(!BYzC16wm+gY5txyi_Z-9%0)QK~c5X$EwI zQ3KOEeHFyzgEgNI!=f>MhwTa@lHOv&0FP9vravW-9DMorKBN+pTA9l6Wy*14S>34Q zLCWt9L0>js=8!=vwoYEmIsU_wBr^|RU(qjLQgjGO_a)eEp${>=e3SZ2Zfo#(HqCgM zG#Qc@DLdg^U#!b{iS+B9=MLYTNAFDljWE?#InZy{c>2~dJbVdqw+u05!noruLbe9hTK zFcllPxB8MZfvsW$p{6sHs>8?RiY3R~aIGTu8u687!IttQ%N~!|z^3!u^Ojth4Wof` zbecGGQ%Ox1$mmigsI~1wxRmCB_eU!xCtHUvu82wZ2>b;JrEx`B|JdsD>ol}5uS-~) zyc}Nl`X;%WcR1#%aSH+CgV@zW`FFsoV8jV>Lo4WKggX$d!#XCr5;eV@G=8jzC68?6 zPha@d82>Jytm|2q=;cJZ4rbeydY;tPI?K)!L51K%9!t}Vl5xpeQvtV;f#Ty0VT z>bZp)VcEMWia21k-{aVitLYnt7tz|N_xE_OrM97|Iy6d64Fiq6*^~N2jAt_hak8KdAj#DM-gQhHQn{9H*+vb%r_Gj()qo1D(j4 z%d4VUG6$LuQMq4pIHZbr+lAiVI;JkSQFvaPR%UK*x+IJpT{IsufRT1uDA~NpToayp zD%r+tmPXf0h=hZ2`Gfp_Nws1ht5LO}lmO^az|YWu@colN7EblfKd-Fe`skxqL&;4_ z*93DEwPdo@0o+AG8_zS6=VFFs^l$%oVWK|4*}D&UAr6SqciJ7TI&NPhU7plpHWQX= zZ$Ha_k`eNQnvWd$HMZSQ;(&gWi)S8 z0zndnDvNjC+$4Vg-S55&ZH=W14O8CjLt;Re^YO!Q6KZ~eV?=>zMYEnlYUNrg4R&!K zVv#&N=y5MBDKP2`;#UW2DZp!Z8n#D$-_jWds-}#{kw-0sM`ca2C**ga-0>wu#Z#%D zm~UoD7VwD@z&(|!h-gcVts!;sVm^)UL;StoU*q&{vTYuzzG}4f^c=YbZTWDCJz|B7 zoEvM!q|ET7&O+y_;1dOSj&%I^5n*Ay zUikZI*K)ZyKk1;gPJU-HNrcJURC7Pdno-a!bk;Rh|1zm#W}Sxb+>ta~TiMa41cB=S zI{q!%FwZECTpTQpqR0D56*Amtma7mjo#wpg+gL(8-0y7D87S`M8mMZK$PvziB9`gLx-Lq1=@8BeW06 zvtU`Bq~e%&PKC=~~&bJ%?W|6-mdKGe!G=S2a>U z@bJHY{bg^z5raz$bW`WYGfVprcg^6NTzNdI+sn(yU#>J(pv4!& zL1)KpSLyQpAO_ItKj#qfG2F1^!iKUsAFixk|3l>$+%>-^_te#YVQb;zB1|qE?KmhA z#%ISX`(x_s1{!act<{k_i|xc0#x;l_M5w%$@7(a##Wui6)&PP3Y1$&%+caHzCj>bf zJG1bTaTs1(ndQ2z@${+QkiR`|7^3s%j-aI3Q5(@KW=A3a=5^pd6BEDq=zOi~*~7di zKbT3}ep;%!jY)@j15*x^07fiR{2N)H5=%a8Z&rHDS4D8AW^Cvqw7#lF^a1keA>5H8 z-@6;f7S!lMrcCY@l0OCN0Ll!SwHb`CC0L3bJI>c&Jp^T<^E3{pt9f4^Ki#q^rK$}o zzu8JPA{2~46j5J2AOnP*IKM^|f{k1Id28G_L|9M#%8 zwW;@^=r^0XSOJ?hTIIIK3YLB}mB~FuyU3i}K+-j-)^Z@^8Yy92uTWO`BH8qE9GRJW zPl_*-pkIPJDaG66`H%z45tITtOWW;K6TjNH ze&r{}K^>x#|04rdLTZAPaybdw79q9iJX&?k;FSZ95oc}fOLFcfiJN7FJkCp@2;wYV zFpGImoOYQx`M3=a@5p$C>uUQ{ZCO)8LKC@{EIAn*eIyf|<-n8ICa3ZR9#vy&vEv|q zNu()8wlD?3z=%I3gw(60oo+7EG^$!=RPf+C00g()A9}#A@bwy_-JZH8LuNhnxJi#Y zFCw$&SJ+E9j~!;+YC4|&j@gPN`5-z{<6v&&MPgrZP1M;;7GoSyp782g23)7Mc9&RE z@l*QcS6pK?W(7@gX2iqXvE%)qKS4@hem6S&nKY^Qq_#fZpl9l}F7!@AdhkZlhEPu7 z)@Am)uxdW~QALUsBii5$qlspUD(BJLxZax?YFM*pUJx>V&{t5cGA&iT`W)X(AhJuF z*aGTtW#$j41kd7QtUI?5)fHE#ZLapsD);W4{vNb8hq!gm_v3bHeA6$+evt}tJ%)bK zpuzA0f7UGX9yeY?uYOyJ%rTYKm%TMR(!m?bZ}=`418JJ=K z*7{b4r6Ya`vRDg6x3PXt z$TJ_>n(mqQErIsar547fN+TrvMHX~}4KoxlldxZR2fhW4Cx<=&qsDahJK4N-cqcna z&bDfs*_oe5KL5?(aYGgvmg}?w$O0G}FqeQN*&tJv*%zakLSFV}NsWvGq`D_p6m7<7 z#X=XlWM@^ z1S`I-ovex1i8Bw8^*x;Ruf{H5mV}p1w+zepv=vAvvVI_J!Zul%tVr{4Es|n1?Djt7;iO2LKhKA0*_Ke1(pNrwY#pvF ztuMRCr;X7;NDk*2_{W*Pk51Lg9zn7-Me03~RrHl-efUkT+GXDe7{+w#XK+M}*wHHu zb67s@gm7EeH`3kxr`EgNoUb#vly*ThH|i7M#WoWDfV2;xXp&Y5tJypisC2t{cn{@O z?&#==zm`a?=|->yy=>-3n8&mle5hm_85jC9-rxI{`0-2=7V|=bZ#ifjSN^k5@L;Hc znW~{YZ)RYy-u+PKAmZXLJ^`ku1Zq+Gr{sGnz0N<_1s#m%Z*;e}y z1&`F1*C;xdY#V3i!n2S=oOA(k=0VT7eF*hL>?#2d1@SiFMvEeL_-`H1+LlD zKf!c#xZjY_nh^)xc2IM8^JuJn8QMyAs<+0Q)HUm}IEg*ctOfcY+}ODSEbFL4i4`ul zKinf(0+g+KI zqJ0QZw5^T9b|Rku$0~y`bZPRXH2f=G*wEQm<7zI)ma`t^nnl{zE*i8nxV+gZ0@AiI zu^W4{tr@8sQe;F_qtl%AUH`+V*$WMWRZ6b$zG|<-9uSO@B@&9Uai#tP5YCHy1-LyI z?ARtA^JO*)Cb0zt(T)Djb`EPm^*fsH^3NjtHD(1|zC_^rzDFb7tUL%`}w(70Sdnp2v^ z7**j1~z#tc+8PIp0i3Vwby-oBlJ111uyBJW3zph=)1B5sqo^pnGI=Y)|*w=6@n zh(~wD9(U^b5C1w|!R%{Huc?aUrPj1cDUcxJL9PJnl%q)-)h)pm zVY>R4=9cE&-gNEQvkx3L2J_qkQLEyW7*~cf%)NgKO}WUZIIBj)PIPtdg~Vm+w5N~f zzwjqM0=-&FkWb$6JdLkM(e;)k@W~*G8si1b2M+Fp*5fowls25KS%31nVm3^5W6IFF z>4$^IlYPjT_T|tcWbVKjUqI}gj)MPgr}n#R`a-ea{i!Be-0#J@y?>a$F$TN(c!}W zxN612j8Z3kCf^<{HNQ2`jH@SIxMWvqnsyIdj?aLg!Qq-b14i_A8-7}o6-JlMZs_88 zFZO81cMVJ#V()r=p6zhaKph^N3BuCzch`OXAsrSL_i8Zz zsS>ZA#Lbi(#NOM;E*R?aF?I zSBwqZ42=Wiu-xfiy@X)3Ai*3=_&&DURahM2eWEVU)v zfW?jXN@PM$Bdq2JuQ!XmcQzX1R*JOPH4I*{*E7cyp#Zz;u}Y#N35sPk9sqC=!_3ASJZm5JpG7gvmD*9W?C-Y-YnS${v^DMc>hx_ z`R${ZbKP@(X)?}AyUZ>xMxsjJ>G8=6A0$6n>`g=Y;#+OssCn4HSLzrJKnv)F%+1fn zA-U&=nf%O|qCh-`Pxb39vUUC1h%aI-wYnWf*w7U1O^m0??Y2KcPGVYtBq(>YE|m{X zeM1U!DkqD?ZdRzal$usrTOUU-xzuUaK#P%) zG@oer;gN4cYts(b2K=?pfB4PS^ez@A_X>3*UrA=``%rfzYeKm8d#5lTz6RB4zIWZI zyx`gLk}_s7ZFn>{Eop24OqT-l0IF!(K#O^!m^_Jnj3Sb>dW)|R<+5i+b80>RWEEV8+J67B7B8X^UPJDVW@9+x8>4$hMg=-K%%LUaVs;sng$auFTr zUOVd-Z?(a{8`Py_|c2}(ZjSPdtX^qq2D9ECw*oExjX|`)-(y%7 zz+-nU97cLJY5ie>Hjg>(90j7&YiE*D#wFP!aV>e! z6DRP2-RtkdLeU3u$+AFwTltOKnRt)dT9*x7`P_KrX+6e+{_T&o%fkgLLn*naeTWWb z7)w={r3+vvAP(_3g-V*B$_T!uDp|h(En)VWjBg2b_st_CWThifI_}!TIv&~A(qq3- zz4s*K74`Zvl}JWL(T7IR@|yRC-FttSGN_(>|5OCzDLkicoZrwD^%(Qm>;AHy^mO?6=7lVR5Z?O5x4N-u!sK+g}tV zR?S#W?2Z?uBh`VH3&7lGScL~&u*f*wWK0n$6mDD>8<85L?2bI^hs<=C&=vlir~cF} zkXa(?r$qG&>Ac=JRP$qF;m$6o#$?wwR#Relv#Q!BL3eTQ3K@>aw!t17v0Xb|`E~tx(tY+9Nfi=o#+&M`4==F;5?C;zN7c=0XCBX_;lL5B4J_*ceMj?D ze!TAck^jNRXI}iK&E)wJG<2g|3o^EeqxpjvE7psTdV(`GHmTXjyF?|qsKT>eKJ(V_ zS<+9Jg9oJRph`W*Jrm~69AQa~n|qA^jUuXCd7ph%NY*&hr3-RGA}5bS={MEC&FicH z=0(^LaZt+KvqP|$k-;O&@ClyLB6lS*+qY|Ds@9^lLJioN@3N$z==S~;qiy>zdJ} zrkD2MQ{(FUC{3UGndO^TYBPV`@86lvn3y7a&3~j|22UR|7w40@sCl+Yz{o=|vFHkI zvoqzd8^O!L* zy<1ydacs5qL8hBjni%~!W*x!g31H;#we?SkfZ;W}690u6y6LDMP+cYI2=>`oUYnXCw=9QotMU-1D74$#f-GGNx9 z$ z+*kb4Gs+Erzn>3=p4JGjJUV*iR8Y(83Nm{}3>1}%%~JLRrXV!gK6wW-zGdkPQ`rd- z#W+Nv@>N=HAxBTy-wg?cZRiFSqlKKSUN`+HPQ|B62X;vCHb!PJ)~NnWo*RsqRY-b= zloq*(rgq0Kc*Tw0+;$wyIWKz5_>=pW2=12i*l|V5RWRE@oZuvGKZg9POr~}A@~DDw z$j6K4>;hbdSLb72rbimBhE2VlkB|#L-{W;H`jltQ(gOk9MU%vw)n~96UL8qn7L|E# z5QXSqDw9;}_~Pk8g-%O*8D$(wMTh2VULIr;I&HH^n%}P=0lk3i%8~xutAU}=a-gDy zE`nilxzY?peXuxP5P+My7zSMcSW=ZoY-&8<0f>7VDOa2EoVxx~3Cq8vwyMoNQ?R*SR8lp^u?_xP3%taDD+A?Mo6vxrkSi0COp68^RfiLY4>hj8nZ^xL&WR;%;%OUo>BRh~JTVT9`Laul`HrJH^hEoL^>SKQGC7>#^e(D6SWV&LcOULc_Okks{++!ni^b=^4lKNyd$#C@2x7J z)NxtId_wDVp}e2>uIm^bhC$ibH;~2>d_0CV#Se*bzq0z6eGHezdrU65wOk{Q>*+!1 zsDZJi(qqmMCVTqf*+xzb{{x1>$Vbd3^dY0ubQS96oFoh%+tL@=)!q#KF>u~D;?}7? z!IYJ!kqBh{(zyFgThM{0hJY-9Cxpib!dV#Q=~t*FB#1rTh?=gtn~P8NOaFA?0nEE5 zDe@P>(C@h8Gb_ty>0RWLG-06SFTitlTo%toU7IyB0uAGIl zARcHye<&X4AuC+D6GuvFI!xA&~#BbJ*JCx((lsgJG=#tCb7vd zs;mfIWb(ZH5a14->c9T8_iDc}IhsIFT(j^lv<%&oABK8j7vBSJ-_H8I8hpg&<~oFZ zb*3?;nWd(zxCv^WI-)`E%u|p6Bc8a#vscVei>QAi9(7><-*}GjVuHOAnyi=*Za6o~ zJ4z|h~rlv@Gdu^ObM_Q!6eO=AdWF)0Nf0mF&J zt^tF$cJm`#@J8&!ZV7(T#Z}VhT@J}FK~=DWN;}OQb9ZnAiQ6h+>+a+9#W+;dB-Y(* z>8O+*cw;a}V*BbGL5Cn22B4WWRls?GWTr-y(ZbMf{Cqb<{yL5>;ntB04wug z76HtO?Y~h)9U*^GZPX=Y4*73?5iPB%Ny5FZ=%0QfG^O(W^g;Emx;D{$$bbXRzZDyU zWsQ{bL~JkZYK2@8-t-L7tlrWof&UFjKILy@p#u2v=vEUy=tVBC>BkyL3a1ZpFnjXW2lmZe4Vc-`zD8erCWTw3*wscGwlBQc&Jnr4NTo)15NMa z<8&7Wwu)OV5hgfEXaN&;WYT#)39)Qm+nD0x&3VRG*J4)};A*bu_4Ri*%&EnldN}b= zzls5v@3mp1+APce!WM+oAEtf>#>XhajCWXnXnZ`WpAtSW!@y5a7$$@r`^a4GoI|Rm zT2FIWZ29fapEgmtIkqzb-XBZ?cq|*)3;isgi%zb3VbcW@IHL3ogKRZuIIrHlJi2&y z>e0Z3kc$s?M=rJOdD4u)Cv(^VcVvtfjDvZA8amdN2OaL}@%L%`rU3EuNGspGKox(N z_VPQ&B@RwPDqV<`$=y#nLxK95g8A-`{0X!112OSL*So>fs_?zY>@!&zz~ZDVHtN?| zM~{u(vG}sxK986$9Um$C5Uq$9k&|sVZxPFQR~ZS6w=hrO5wtwEttT}BT`3Y@Ojm#F zK!58{u=^|h7{4)BOy&d2WBl#K<7@;7feJoH^f(G==>d*0RnLc!W+}%jv%cg>>c8=g zT#{YUKDiUbMJL}lDS2>HVab+YfY>C`E-^>Zl~7_TS;jvBs6@p3HJW~Ewi9~3T3z*I zL%OQQ(Sq&f#udonyGZ3Aq|VqBX*Omn;G1n{*d}pccXc@7lwJ7qt@s)|OaK+zskbm2 zPj>(U3x9`fyj=5Jmw&LBXHM&vEA7brwy5}N<@)nL#wp7;nGft{*i_d{3O{}W6%1!d zWF|qh%SmJ|ylVIT8%fSb>0Jf;5PNJO%mqBZO7js3v>WdWz!1gQ%W0sKz8s9P4lqvpjfEP@PZvBRCP?;s=-stvw z?+I|7IH_+#brNHa8!6K7GP3p|Zcx^#pW!xS)kkwYT%$6XPaXT%lfne4DkYsmROY-T zyed+NIfx{Ii7u;Fq#^HQ#X0lnl5orB*eAJyG{s9U`VYn~_yHR>LS#+Uuy(KEPj;#6 zjD1K-+z#RCT)@n>=Wjq6O|t7BkDXJa3r-OtqiMR~C&>y0_>#M`mev;S+F3K_0yN_R#W{o`Z#ym;HYsiN^OF-kXryr2n}zl74eT*R^1qqf zBYq-1e`b)>fBc(4&W`Rkj6L-hS^N?;SanURrF2d8{)x6HT#0$QS05NKS%+%j$Ckx7 z+eEFqI`{=-7_}%gHBClMQZArC7c=SP(53r@OGBGU1R~JJxvEQsa3)Wr8jPPPF8iI z3wr%<(cQ}L@5fBc`uBA_Kpp0Fu)lQHwdT`A944Va=wwgOh$3~DECXzZoNh~9JBOy7 zHp-wU0xj*}V;np2Oa&^o4OuQHR^&8E4JfTEHgB*sBeuIM)yhqeag6dTViYr08Pjr z_a-V8>j_LuIzVG*tM2xb^;^iqMB?|BdsElD$jWOb(*#-;XBg0b4-6$#67$;Vsmv;& zmRD|->VuT)FOYj{!MD`6Ph8l_P2K(kW)u5DCn_it)Nrf-%mcKEE7ggWP7bUK&NsgP z+zqkokC*X#Euht`Evqp%)(em>r-FFH>U+?~;w8~*kVnpL4;XkJz2(zr^T9mvfmF+e zN-HVra;f@4Pj}L;?=fA>CY&lsUME0T$H)Rb_^vM)ek95j`59KYsH^nE?Sv(nTm$Cl!YzLRHZuB;~7c9w)i7L%MqO=&rq1f#URmMD*6lsYg}g zt*g9aCg=VKVP75(9&V zlf5h{`#xjenX!zQSvt@6cm6r&pL724mk;mzJkNVS_kG>hb=~)%Vy`CkiQ>GW63)@U zRa$Su4+kjVVPNb9*sp3DFAIY$oQedv@)g<>`g^j!G1Vt*^Mc}Xx_JK}*0^qiCuREx zRcDgpi(>oTI?i3vu~6J{T=+2}*GG0#BwaDK0Vz9;U`2-En9_xI?QF9h> zNpX9vZee&XhJvq{3FJlA*DoL5k+p$c+-N_zkjiEj5_bQH`_(7fI!w{roS5SvI$|KT z*>*y?)|I?p0l$5i(*8>zT{w!B)nsx0jQ8BCQ<9(p=Lw zul2HjpK>~TrCo{4wQFfIZ~nQB0ULuFYAtbPAZNPr0)D=K#Mao8ahwkvUq!Opnw>VN z@c5`oEuOdlX&+_l(QTE5kcO=?(Fayqr1ybK+f@&V287@0XLok!+J*Fpmo9D~Pdljv zbHhF3zn{7gQZyEEHvO1UlvSEz|Gd3)orahfoEWm?1pOxgo&bBzkp~^~LXK@yH2ZDb zjwoyY{yq)ymOpDvuvr+?mwoa}?o%2%OJPnofT!cR%Oj65Ll<=84TlDeE97BH-F8uzNtTaa z^aEbycAEz=^1&Mg;rFIxa~Ao)DnY4%y55zqy>gB?|EE-S{u#_S3D?z3-wMR6dvE1ZEt~-S>F} z7-?WRn+c@*8t}sqe35dNj`cbPi>B#6M@*j|kR@>CI+n-EW<8-+t2qwL2@o{|9L1h6 zc((lZ+SPZMfdV54RqpH4L>ua`waXht;W9`MZ>Kx$`pLVr{-81aV9pUi^zQFujRYbN z2zfW@ufYbwF-@{G(E2?p0?l0@pG4--6iKb6Ml&M1aJ`gIT7H3rFO_ex{03euo$}cH zXtmY-bUPz9J&VX*UAhI-eNpgUZhrp*>>W%KH-Z>53?ykRQRnC&9e5_K;Q~LkSzK1-@f@?H@Oo0_^X7Mbg9RtZCXs7Y5uqt$YAXN?BGwkVzXIs0tRqSf^(#&bAdBX7Q zbW)E1Gl#^{w*41= zYw6ip9sJqqg~Zfo+;Hmb4L{_%8P#<;&VjhcGx;oC;FG|@;Hhsu{DniBszNufw-_rJ zpZ}3eb*lePa%e%a82SWWND7)>Ki2PAF92%Qb(jA9-y<`Tmj?W#zOEg$3owK7NUKCl zOOMq|M_Z8%L5H*u>N>2JTrFFso@JxSQLcE-g!lB*b9YVyR2|gSO%DMifH{RBez6g& zf$otiivtu%3C$*Rj?}7$&wI`sl~duwH?wl`)m-3yZRNRcPm2r_Qkc;~X)*Zr-H6@RMQDWD_3UqGghsv-VeDY zIU>Tl?f~TOlIxbi`sHZri3CFlZ9G;Z*M4txd|X7+BlZXTL)cAv|HWyV+~LTKYd{v= z>#-lb+MAj;Aratbx~?5wy@Rua&XFW$7T3>_#gNc-lJeDhvcy|2<+n0x{hp89f3La6 zw_-cQqrZkEoi*P(0|N3NkWRd8JdXbTcljR^KgKz+oV*t^%XayS4odk(e;PGc4Yr&h zzn3FwfbdqjYvAk2kZgWZvQR+ZAPjXFTYtFUrm5@yy>}rQ$n?RVy9roWZhB?p52`3q z2y{3=**EHb=!b_lkXFatlB|lasG5pSx%%SPyD0qN%hW=*VM<(za%+j&hXSt8vR)<3 zSUcXIf#4ddT@|{|HMuS@yG(zs8*X5X5UL4l`P4%Yo0d7~yakJ>8j496|1CP)lW(&| zl&vgycQYhZN%iMDqGqu!h6htkpb5(YL>>fi7m`F1KPI%1Jxy`{fphE1ut_>=P2-(h zmB&Mx9s(=h`Zxq>)ng0$v?An076jN1rDi|o3;KriDO^-%Ow(h;W5S4yP!a!(e#Ed9 z`7Tq?bkFf8^ANf@Kp||rnig>7Bz&d%CIh$wIQ>VM(WeYT%NGcMsqe7lg64zJ4>P~V z^QV?)%q}*3sT}*4T`AL|0lI=rm!0?4w zrnAE6!%30U0(X=95!-#PC~jc=^vvKui1LbF2m+Sd=ymhK03E6(l)2cP8dmwNN-g=m zBiE-tES6f8@qg1*rV23LW}13_?4wW+)df95;Pc6A#Y8sB4Q0Ng{h3@luwcLH8s#qrWIrRvh>A?OJ(%laxpOV^@jcJx$r3>`of`Y9BvxzJMd4 z<292A{U`=JI;bis&#US22;0qYO!+^i52ZFs4{k`9j7FF^Ez!Kwfr)3Zo#Vbbk{@8YefZCgeGm6!L)#jL*a1p)fZH0eKUVi>FcEm_)UZ>~!Q@SOs-V|a?EqC+R*W`D#t2QF2%J~|kouPg$X{4?=$LdRFP_OZTxz~Zk zd}D?#->$@(HC53!i_$^=N;^YUhfpsJXRNQO7rXL#QmihO%MP|*K`$J6#iVV};Vz9m z6%ixo`fbMu6<;LfYI;ZiKN85X-{>}P1Pf@E`)`i*w3O zpu~D5Dbgo~`~9l&NLdtPI(A_o&!PfxG(h`Lmy6Dus{WlWG#^V~eD0u4Hqi@5EF^By zggvP)GmwTM4ji*H8u0ce{-dMX#sTIfhKHaOvAodhl^g|A<}#S5QS+_W8} zErv8NXcS9TAz0Ujcj6Dc45!^+J`Vp@p@1NM!I0eW>rPY{C}&#b-^Alv35UmzdDr)6 zh?$j#*VdH^eN5E1d?Z&R^yx-}InkSSMUerUMy(i9k7AncA?z{GCF3`xH1Q!!dB8nk zneWhmvd_|;FT55w!P986B|+nT&tO%gDxt>#V8g@U(`9MtjWFFv zSvl=tha9-5HG8M>D!=Ea_a3a4FMQNLFeOsXAl~woY#-#h2*K%%s5#%74WSqdsJxiu(x!-B-jN(JSJ#B0ttaQKM(#ZQUXgXfpK725q3tNf;Vt-b*qkOa4e5 zns!gfL!X#K$V3*k1=MMr0Sy#JFy6XlIP{s+hg7EwvS;31PNLod z?I!;Rm1M4Pm9IA_K0|ShlH9aBk--naI3cE=KMIdN(#6EJp#gx^bQwX+$BSZ!X?Pw0 zH)x#`b3(HYUc5g)L^EsJqL=c9)xY=zLx1@%>~0C{XyC?E= z_>6`by1jf4xr|^O2aV?=%zqiOx&`O~L`i$%3G56uebtgwRQJt$3~MG%k#5%YnarHN zxKc0vGZ{h;yd0;CnP6`Y7R^=GSHx$epSklbzD}e_`0a?)lJ66-vm__F6UH8}H-?EA z0NA)0L(`<=_pOZ#z8|_6T9SO<{E$w<(1!4P-oJD9!odnZEzejmE@GOVBZzNPS=NKH zQU86v!NE_$m;jn z9GmB2o(0-*(fwe(J~W;z#_|8mc}74hk;V%Rdw6~eI6vsH|L&>dj)px#Wia!_gVS2Q zu33y!J*}QDi32`gqc%7Pge6j8XMEuY!&ZMP3-8uc)Kx!89b1^P?<`a3EGtVNWWehd zLW_J+R`sOC2-%DMyCQIhs@lq$>%4z@PVxFqsUAuk3_5CX#C(Shtm-vz7}}p>O7^0O zbSzvOW#mOlwyxd3U~63=l#fk&`+=DexyXQ@GYLg}2WApwR4vb@<2natW?_&L2bLw_~t^=*={je@(ex=7$rOsQEMd%#svMhv5?XvuF9CpK48QD0Gugm_=*iaSi6LApE9 zHZjC%6v zIA8*w)Ha=F1U_%)>rIp{-N(87u}?3*ruM1Mpob=S!p~fPnu{O8PXA+y+#v}2O58~^ z9J=hEQoL5Yj_pdFnmH+|!4s}`-y~%HkNE^y?L=R9N#pOEeOAgpI?E*=7f9;6=26eM zJ$#z%w}aa@A|q(Rpn^X7hIz1R!B?ioN=LD z{&h5AujgY5BXKJ(8bO@KLKk46%tdjn(w|-LuB4)4^3~wIDh(#srb61Vo=5s?syn84 zC_S>zKuotwVgK3z^%3Ug?pNP?!Qo^nwPm+04U~ZG1)91Tl}wJKiO5qAKTFFXVM7BP ziwxkaMV}KJBp638Y=9IMy*iH&z%-3Eo~o9Gy@s<{PciS=52)0vm+r<&Yit7-z(|Jr zF5!^xV)K}FarKPL>3>YdWsNdb{S)Ix9A>=ZHF_HvR<85Er#|xi8ZHbaElMucHahUx zNVXFamoOO;w3BL#;}Yam2Ky7Lu5b7>rJQIDYi67#Ekuq*@15R$_;7-rkryfC>(kiU zB7Kvv0y1zu`WY{Q^MF_ZsUV4QwDTZxNVFp2%-`*MLglEaTW?C{#vK)0Ir9GW_?*YS zJTTui`!hTfV0E3#2zJHd$+IG^@4pJ}ri&^U>;e&?`ESvZ|D$Dl=>KQl`MBw<`2yG^ zH3mBmXktJT4r?tKvffV1tC{a+XzX;6Y%eVk#oL>*I=n_{IeBY>oeuhel&FV0DJeSKi<@+$Fc;$kH>SS*p0=)oRlzg3UwK<%{W0Wi|+==q2I_ex^eYz4$zl z{kUBx=yH)1o1Cj_w#ApQ3vyeUp9|g=usxOCdaX$w!spHkG)wB=Xq?^hp2#U)JHSxn z=aeU7A~cgIBj;V6;8kMy^<*wQ5ox zmX&2r9>$*L>K*z=-$33r-+TLBL~|&p zW8BPr=oYkMaM(anN5q-KVYh*|>K;{P#ou)yx@lJX{%CEin!uI2a@hghd32e4BF~8z zPwp|~72D@+l=Q20UaN~m+f^MR-kU_44tRaO8Hlp1Nm$t0xS!-W@3Lhm9&Pz~vhnFt z37L)rLltTUc)_b+$M2xf1dITF+9|-QA|B4`du19^A@qSU)No75@vr*gqoWVzgE~7Z z`H$V{Iw(HQ68duss8OEOGoA)`pRat_ax!866%5c|fnqH>KHzD6Kf;ADR%paIEp2#Y zO5m?t|NOv0bWC-%VtSQ_5@SIbs~v{ixN3>1l#fL5qD5((kXIqi9KsdqWcvDFt#T?{ z9e)pK$G7ZWD|H*Pzx`5P%KoMFaFv_{SXeCUFV7c4b#fsYW3J+B)s^KHRdLvjY!AJt zwSuhvNZ@}E2gy1BaegbTda-p8xGyBEI3eDk&(u72;8Of-yCVOA@bKax9%8)PDXv7N zZsyXvv=AQ+U#+v#s4x)0M$?7*txr_WXnr02wiaJIqFDXA z<_WM2JIsx|e+jD+_}@hjnmWNcv|He33`7pFoU!wmCWJnaZSOI+ym#0tpFgRrWFlLu z6_$SeL$z7Wd{sM;mT6BT*%*NKR^qU>av8^^n+qME-V1TzNT^qcWX*#4S=u{Sl>KaV z^_CSc#zeg3BYm=KCpYK3Y4#tyvj56&A9g)~M9l}HErWD(Dkmtu@lWxPL|*eeDB497 zbtlLfgOyz+F<+l6ouzK~1XZapSl{#HJ+3KNn}N~zfN)=)glKZCLYMkZO_4)f(0PZoUjUiZ2Z31>!b}(GjjszL z!sL(T*U?k#_DjO_}c$ZE04iEzD4dUaY!+hujnZ5AH|SVdyw zSAFYgUd10%hDA8S!`ynANw}t(%4bD*dDL+6)5-T$$@>+se=VgVfO2Q(|N zd;L%xpfsS5ITFf!*7m14F|B)z@--({iZYjn%^PorW=YE+DZT`!-b<$6(H0{-A*f28 z>oR@ccqAQO@ZI&2P-kMNivg8m5lv)T^ESV9-R17Q=rge7omnleN7A^6|#dS)i$8CDN~EO3^=t}$bcA0;r;oJ2RU&;J&$BX%;?SI2j_pmR7HpF?Yg@n@2c zNt_R|(q_2|dtl-vDot0cgo!}XH(AyXK7VuQ?|%4?iEwstr=g9R#@k~ z4KDh1zUTmh9to1kgICJPT@-7hBtw6H%NN7o>;kV`u&Rjxr0<+%03%?Fo;Ov2;H_0| z-U3i)B<)nktPkx}Z)++`NTc@iE4+2RZ;R}|$95RjB`rLY{o8uYaGS#*(|oFwShXJa z>5=(?r7xCJOd1NxnRGI>OmSYze3h5JaW_~!E5=8&KA`E|-IIntnEcK6lRyNyhH$d> zp-j??X?E0WD}9KOO3n6ggP>uUL*IIMq67CsT6pI@2WG|tFhu0w>xuPY zjWx)FRrf+}Exs*r5kY)h@8}*eIT|nnxnibHL!6{7c{L02)4UUDZcUOxH>V#sD$=-% zm+d=rP5;w?nq%MKIU5BSzhLD<5v2Z>E&UwG{D+^E)&ADI4B55J?(*(QHPxX6y^Xe* z7trTVRGG`}cc22NttF>si=$Gp4ORNvpMeYzfgZD)?)Xu}LXjhR{js*{D6UH{#Qaw&Sj?yh!O6NKoUSx25%gf>r9^M5Q?U811g};}t*stgfMv?ARQ*}#6 z-5h6)e1T6bjP!M2eHM0wLcJm8{$na8=^aLt_fs_W0CP~kCs5qI9Znhzm3Md>I}ZQa zCEvX=o#W?bJju0N1g%pMjBY}8@{dv?Ci4DiQ6*ArE&HUU&DG^g*2~nwdZDioTLczq zafhRZW{I|W&>^tL`Ix5D3{^~%=59peImjfB9Nt`?A3VP52s_XXqqMp|S8UI))6l&C zq}4P*alW2)^}h=S1y0uEMs4u(ik1miSoW&`b5h@y%P>Qv=3SiFtNq3P39_3SnU}dM zd86j{=qWlInhE_EQH}(vOHTCJ0NL+lWQRAEOQOFII#*O)r2Ubhm=Be?j!auQ$Ja=8 z{)qY66gRsSX4m4?kKX0qLu|o&v{W6Dw=ZH6I-D7>eqQq(u!u*&`Ujs{!I!PBAS(IS zLhayLq@@k){l8h0D!;?HrV_L^UO6ic(4i}(inL#}cJ8~bT3r2o_$R;0Qw3I}8by}J ztK&(~T4Sy#`gbNAT+YF0Op^{n9lVz&<8lkxXj-MIuHUuudePGRTC3YP2ayNW%o{gf zmxeu3(<4lo0vD2HH{cyLf;obBMFt(Xbzp)H!pFYJBfPvEFP>gS*6WHsDH?e6IU!e{ zDwAW)re})z^%Q>{K}^GvOpsAnk})cl6mXlQteQ1V*YBz(p9tHm;=DG9BvA}!x&$tC z{4_LWS1t^|3X<#<2PXZF??sWdKZR?*ys%b)_t`=e#X)k(G@jrNO70&J!kti~;kPUp z7na@kQ>#9@zQ(v&wbo2TBIXbHVCPnqUWdzwb&#%K=09DoU9n_pkWv^XbTht*CC;St zowhXH8{>c+6PiY>-y`|!LYpL{*=@6yv7N)u?3c(TGX_s}7{$+0c!3#&n23VYzSe&^ zD2DjX>6r@fBIRG}&k2oQzz903=Km0K)F?@Vh?!4=7q}t)d$nj8xg0IWeoagl z+ou_3V8@cJsbMZ^PPs>}CU&&AjY&_#>q%xalr&M?bT9WysG|RSQB4;=Xkdr}%O0A9kt#Z6CpJSd)^VN@UUz`3me>?Dg_3j)-iU1rpc+x0? z^v@VaZM*-T=@9gO=2*DU@y|jJ{_fPL)yT;_v73L@rz4cDEHRO)D&`bjZ1OIZ0g%{W zK;*`^UshyO*_e z$PeKE1fAzBbcPpBE-qp;I5)qYt#ZX{VEUGP#fukzu}#Qe1aU+jy0jNj)itBJ$qKn3 z3H)^*wr^&QJa}N7(^z~oFouYMF89vxP5pw4(;WIMw)Z|>o)oqE9hmNv_At;J%}1B? z9cQqE)Z)k>6d**!%%a}ROAo%JLc1UfZFG+A*ullpJV2!V?;WHA&egue2&~wMy_|JT z%Oc44ET`41CvU0e$w|xz9yF)fy!*$bD3d0gSC4u{#sz`9Z(wuu)NNMudEk=pv}AVk z_|cO7zj1%pH&VE=9HtuENC=wHNAS;aq<3cDAbm6Px$DEGL-P9Y><5O<_%KBj3?eX2 z6w%`CLFhaGS6li;?z@k({=!L%UZ43uk);!K01ZsMbvKw_t*YR4t~(2P!|x zmG8z18M)xEF0LA_Xd&ND{tD=$8yyUpA*&}ahBp6g3y{jHphXLS4qKaK)||k*G!}*x z%_Qs%ZhrEFhDXpGkRlkzkZmg;Zo~I?hu0=5&!Xu$%QU+zMixUdLPD~e>($_Ju>y56 zf+XK22n(%1u^qpC-DFVjqk|6xS3YiqPmIWGNYpXhyi&gVag_6t} z++)r9&e|B+3G-zZ8sF@!K25$Cl9X@T6lsz8>~gGLHQrfYa?X)8^#w1*!wtU@xH^X# z?VXn0>%fmOH^GivCN`r^JJq4%YxO6oPnw&pM$S5?kBBDH&$N6v%_SACnq*FsLxYJC zULG&pSJp7O{D8m=AdX91(8b|q)Zs4v_gMgMr_vL@_xA!g_=g87r<-lr zvd$*iA#1tn!1&|ICKo5c>l_9O`GTAUU%U@_#N|9U-`ddRUcLXHuQlC`)6gvUOWjdk zlZOjU{}-p>|1l2WR^;t2qvptc93UR%G#sa%MgK+v^qg_7Mi3JwBTBq>VyL#pS>MG4 zM)mCeb&_(l%>7W?-yErgyVLc5BHx?DDI-#0kj8rkyZF>eQVy1OYH!+lqeuT?tgKO6 z!jBO}OII$Z$MJuQh1qh1VUl*j-*WL01-qzN)R~4?)EmG+8rH!TvG;V(hOAAi-b(<^ zM#~n{Vkly)%2<3JDKm6Uy0h@0Ftkj7sNkOA=AREOxTR?6I-JF=6f!sgkQ(fK9r%@B z##XzvQMr^`rnu^;h_JJXaHH>*hm#wcrnAb9u>KTKV>uTT&PUrp|3sAY$03|CW17v_ za%@Y^d1`BN{*m|V6G=&5k55b*E|pv>?x-C*91OgS%o`5{a6=fXYJd=F@dqx{JG0{| zsV%|sfqs*1{$6%`iJFmEQiA@hXSw1k>cR|?M@AL4@{%FxHW%j>x`lUAx$l3Yjh>5n zh6}@#x&0uV1om*JiyMDs`aU0+Fp7Tl9&wzr;ri>o9x)k@{8>+qcN(zyD5@=E5Pu5c zf*OOhqAHMWmQl|IC-#F!EmE9aP61a#mTY&4PA;;lb&$6$#sfo~je{=rGydR3LB9#I z4#YR%PsqI;w0}!W9-E1i)9E-W^8pwUp8Oc`lMU5#p{TNLcfecKq@w&*XMB5z%vuLK z?+HVR*DN-JXc1EHpgY0pTahFhoWEe?$%V$Tywb40a6J7D!j3BfRgGJh%wY7QIGe)P zyIobZujVf8&_VO}ujlz4&Fd~PhDWpQuimZFKZr&D#+UE4z((DO!{LlWw6mpxm~fif zbJ?Oy{_ER@F&>D}PMAo>1@c1JmS%Am+t0nn3ZeNt1R99WKD~YP~vA;uxl+)uY*nB8T$)xWk!b88z0APSZ76>P4SsXM>spjz8%iIN-=*<~SiVK1ewDLvsYT=vxG_ z=p?jiJ%Au$3FTEeF{f@Ncag(#oc}SA_k2z<(?1Hy)5@x)(}LkUSzW{%(_jvOC<}6q zsiV@!Lu=3Udt1v#@%U;@4Vj1O#)gYpyT5+Xx=}wp)=^dV8X_`y_OJaY67o%V`59ZO zZ3FENWIqt9qsn0<19?ciHx^IKg7N-OS!mmv9UwzmTTv-8Iiz3re%`IbQti4-M4U|FW0oQ*Qc{IsWUBt@*z~C zw|mZ6PnwYkKS4@fqf$B1JtSW@9V-NjFY^ySCr!<&pI@q9^CwxNgwpjjKaT`&X`X!1 zgA(Zoupz}Cr1LPgtj06e-6=-v7>t=gJQURmD^FfKM^$euDiGpgrfHE>(|N>Vysx_* z@x2=>e#dey{w@2L%LzXxTEpaT!U67QKiz z1m_DjFSt5CWlMSIOL+c5fauF$ZzD5yMnc(|18L4Z0v7UP79 zKCA|rlRDaL&c!}-Nk;a`OGL5M#-G2$Kk-hQZiW^`_u$KsP3km4sLfFl;my#Yy`Aq$ zzq_98DYU21R)c`uzYn7g!>ZOnmY>1Dsd!~e{evtVlL4yD!$<|janw=sHmSkOEq54= z1_$b-P<71dJR$~*X#oaOC>kG_rjdAscKM3PH|!R{e85>2l%d~wK22ceu28q9fw(cB z`p78_(DDI%%vGyxFLJYc1>S`TZm1p-dT-#~2+l|D(kQITCrc<;|?aH=%;AltMW=!b@Qc2?s z&LI$ZEPa{uFXD*n*W1iGI=j9-uXeziiDZpmBV0d&dQLk7C4y(PL=TMheuOn+!lnfN zm@t+Ha_R>r(P0`gAFzQbz_Y9IFcb%V4RopRqR+ckhs+#-i&VY;b_M?N3Cp}pyb&l< zUUy0x+6&)~U})Q*Qc(v6M#^21F2}J1`4*sw&s#f3in0q(2V;grt~_g8|JmnaUkjs5 z8H|~M{>U(>-i~t6Gb;Ww?Gl1Q$<+ot786TXE|)Az3b00De1JLm*JBED7p`4}y%c@z zv40A%2#BkQN|p}!&_*rbW9jH74286{SPatXMU6^18~H0>y!KI;zT(*=OH=Ki zh27$Q!`%dum&3Z!OWa{#q872SSp_o@JOJ=d}H z@*e+*(l%Z@PfhsYg0FK4zD)NX9Pm?+P$d!f8dDJ-0#^;V`X`&Lnm8?M%4u2gb-@Z# zdE1dX?xl43h18eNb}}3;iKd#7jQ2Q6y(sqwSzDd=J@={*{caQ+2$9EV#+D=p!&xm0 zF6@QvJd=GoqBp9d`T9x2JMS@9`y^y8)4I+1@dZ1vCsCi?uOL-ua{q#cnr_@4%O3| zQH%SjWFyeMTzm8G@a0MZtvIk)O|97|Ny<oNni6}oS5>VcK2 zm8Yt=k%XGn;HbCobIH}%E@v&lTL|(N+ANWJoyWC6Zi)zL?=9RX!qQ(q`Pe?By!lw?A&D#*7OLz_%=QFq6 z&*%e!aY?0d*~H|PHbL*X`a?_Fq3DjJKX)(LSX;^ zgAw_Gep$-7{h7Z-l*;8T?w;ZW@%t&_82Yy;*BHsy0U67d zrxRB(*=S#11K&38y{LKY2hqF@h_VU))vTuUS~v&+EmzTfSFjN({sZM9#ufeN8zaY! zE2~~7@idqIkPw=4Nc@$O(Cty_p0$5~m+%%nK#1#6-uxE*F zFeM&%9GD;Y1;9b{69I=>rcv<2dtFF3*QT=}dC*l}MRsu@XKt$lIy%0Na`$DXSkqI` z+_ost$Knc5oPH0}>%Uy63G#wFt%-YH4VD@InB?-+Rx`Fsh$~oEN7%dyX^&!0&qm9F zgeKISL~TvKunpg&*d+RRbJ6!B3KO6#QZl%&VP__2SD>b8Dj+DV4mk<+JkGcJUfC@3 zRXjRnj_LJJ9pRU_-JCQ^6g}Og!OnrSfD!TaljmMhqNn#8xfl<+NhF`ynyFTL6*~g+ zx|=S&MP(cp1I1`Ish7rUMcku_r?W;*wanl`izX|K7MhQb=vh{*Ce7}>I2n+OSh3gm zJ@ii15)&zBk<#C0HWx)ha7MPeC)`cTlCHp{wguv;EDQnxh9Ls_P$^<9pVD|K_s{YqU zC_W}zrwz6%lM|nu`|#n31Zko}zRi6@))rtz&B;i5B^DFf`nP+@UvgQD_JPXf5x+yp z#AhKOxn3}3@gI}zrriUk7wUzRL{=^$Y~zc($+W5+(A4>5EJmE`#2cT$p`r7k3RSIgwm!r z-`=Up%G5s_Gh2V7Rn7a`{@vst;gHY-xX_2_ywzdwtwUE9?+w4C7IFQ69gzvGY;`&H z*~#4L>zux6j?I*`L0F&qlWo?=ZJ4u{&C_CuYxSqBXQa&Wu03@WANe>|pYkqi^By4##S1@A<0n_f zwPN3-4lai6(iKW81}-l@n3@qGSKQ*6Z4%0|4QDwbY{%DZdYH9yOS{heufM>SzlL;H zHzk(#^T4PbTV=i;NWR!qfqRSP3qau>{MySHQ0ck0puB77@C5M=_emrw0kB+G!LbGT z!%%LV>j&@rtk4`*#)hPzaiv31D-GW}t$&LP>CFB>`U!89Vm++`KwLc59v~)q2m~5$B>Cim*6msBNdiGUWpRyStF6tyR5pGK z{mp3b7WVrj~=3t7Ro2?3F=lU1c3OZ(^i=?Y4b{ickRFIY(TM9S{;!rI} zqSQ#6Z#OcwooOsNE23vpc(yhz+CGSmS{5hh{$uh~fy_{2$uz+blF?c^3sg$gvuWm{ zgzhGXJI{6KIGm}gV;+rZ!_&#AGLK@}$7QuSV+1rdo;1qI75Xth8NH*9_Ns zA=Ltsr?~%7>o{HScOt_*B+UhWXkqHN=HYjR`>%+NgFAfGc4JNw#`8AhszOya;nZCoVZMlebVUMpnx_5w@qGykByq4-|NOeGuk5$D_`}bD( zN>7g_U;TSc^>Km)fqrEj^Jc$4chDXEEHi2fk5zhjdAnh9IG?mZzaqBN&|INJ;}ZAr z3+Qd5c6Al5=F4x+IWBzcjuVfm{S0not9)C(-PM6DrvuPY4ROO0KhJ`1hbq60AurzaptL6gON|xUE z@d-P*&MM-FVFG-GV}ATKiv$xuQ19bMIw%iFYHKK}s;?Qvoy25bN!m@#-qXBRIXO$) zQ7Dw@y*MEk$94vetTd?LCyPZ)}=~D)cQayo1`r%=J z*>~|l2xf*j`Y*VjcesKGtQC|2n*NMYMOj&>A-ll)Izuj!u3({|^h?mSWR4Rjdd*hP zE~5wlw9W^4LQfpU8&bemA=)Mp4ly+U=lD_XZoFz*reWo~*XXtC6iWb__woqr8yV!S z)D*sboc#ES@hUJ&p^5k&tnt_?yrfB6qdg>pfK?PML8ubD0dD4CB)@AzRLX`tqO2QlkIcDVsoLls4*bV{oGWghiw z#c0n|Rz^Gy<)FmtDwnkf`IIMV8%m_oHv3xjHzvfG|l+mrlNovn%IXPbQ`5 z2h*-Xt7QDl8ul4fI4k1Qy4G=_(utZjg)TQ3Jy;~+q!tmNdYd!faA=8x)PPG+^6X^7 zsttF)mNf6D@F5i^?@{g&0 zF^y}e!EeI+!{Z7{>Autc!;QT6b`lgcooj9>R(ZtVn?_<;HbhOqK((%dq#*v#@)?%` zu9D5)K8^cYMiu>hJn(Ld=ULcBC)fADJ<=kBl?$A&BFNnq%it>+cFp?7lyuOT73hWi z2KNIC@UE~0HnKW^d#M=N9 zJm`4ZJ0_chOAOER9a9k3f*Zk8bRfihb#rehgO@~$e{rGY?^;GOJom<1MYk7*Me=V6 zm3eE0Bu8s-kL#20VX1N)en~|R&j91_rFeq0!wAQ4sCk=4*W`Z)&um{_Gf!Nb8A)rE z!M06R9cTQ+mxj0CN2$LoaF<2S!`4N<3z^jPv~4plx!S*X%Ot+UZP$U5{1XkZ#z8=V zMZ8VdE~%mF#bn$3*O1A1w1F&Lwyc^6XT0iBNCF)wYiX+F|5)@j5K0g&s35pRMI5OYvef;TMmANb!9h=R%=(i%QW2L{HSB(w5dY8&k_T@m+CX_41FY z$Z5LD47JMVF_Ln5itrK3Yj-xkg;O^=Gg6u&U2g%s%Z!wiJ?5{rwn%b#NY8|8`zjNW z;Vrl+jM~$@Qh9m|`GOk^KVkH>aY48{}Q9>RjWTI1{s__iTfw z6nURDc1H3BN<$|=*s{ksa6u0M5hQT0y~mW$-QRd-X=_sd9PHA7%Gn|B6@ScEH5tU% z)j(-#EA)M)d|Lj7jBV>r{fLLH?n9L zz(53sB*?qU+lS5LKM#>$=VdS+nsNERXilV4jN=hhOHA`S`p5S~Y?xdCjyph~@U)Gq z=R_j+G4l4GSK~<|hqD4+7**3zu*zh$`;_}-Hd9wxIBx#_-70lHD52Y!iHWNd-&@XL zXQP@;Fop@0h_{WhRtAeVV%k4vwEu{czg_W^TjU$FWJ=QK!Sd31DJ3Rc$@Hfp8wI0D z{qWS8ioj9wZOk1o5&^RYa?0Zll(->4!VnnUPnBp%Y5bML4->EJnX|X|Mm(v0sy^$~ zbLzx>e9EN6&4J2w)A1XB&-xd3{Q?6=BaE~XG>_iU6;9~B%H%VHAJk;95UJcY2I%Xb3ol|@)vM3)19k}3R5^y z`NU-XR4am5j3Jq>%0m~E*Zwi(`EKg=7kLp*`^McleMR-7_qy<{Qpu#11*H6mzbyjy zniKcuqaDt4Y*_*D_k;^L#0kFje!lnR*?morlP-u`uRaB$C$&4@gB@cYYXHk%PkoMQ zd5oSCoQw;TWAsnM!qaFP*+(>D=7M_1UGJVi_`9od(}uz*of%-}@6&{3h!#R@QgIyoGmL=1Op%p?{-|O}Mv7TGw&& z)V(M~Ee?`L(s+pIf%MaLbR6R8qT;B`|9GyQhuC1zFQ-o@&RTQpJIz@0in?sD z?mN6zoEWMWvdnPxo{o>O2N6RXMRG^bcm|;7i}&e`m?!j0obabk*fY;45pm`9B0U8^ zA`ye78ufSJ*KSSZzsm#ubghXBUifmw11mQ`j}=?mUMp64RBGX3LJKlq1%z{O(|F9v zLlRsabAw*|pv;YiCMaE$QfD8iu&;*3&@x{f&spwmjApY+6uANqhKq!Q{1$ zK6|DvGvyS6vm5MsjuEM_oq{hxbclfGYF3<)4g=A~c*T3)Sp|Iar5764)M5?aa@8Z4 zV4Cyv84&JXKc=Su%G>+aJw(NG*YB}6L{C1u8C8l?Dyil^&EU-F`MFH-kRTuJn6cC! zdZE?7;$m}>=2pJ%1*Zzkc6y+mfBelF{j9|wg z3FF!!6&K@|mE~n+5sNkHuPmf=c};?UuNklxC^-Rx{A3z8p27Z^d`sUWk0$xOuZZyN zpl2}sb7|lMp^tqS>Vh(=4ncAO52^FA2e4e4uEUiqxGE;ItI(X)pY`mbj!A#M<$#?E z_;b~FUu&pcR0&afVNB`nOWaTOtk#I>Ql*R#!>_xTKKS^hY2U$|b?IlAqX-TRaS-4z z`mHX)|Co3KT&9s6&4gF7Q>!Mm$bwf2H!y64x%30{rdLEc}Z1#Zc7KQub z(7&@T)j51icTC|a7)8-LcdngD>bKq>eHrYC{`zC;T}YGi0D8#~8}gofz893)w?lOA z?_@EfkGwxy?=8Tba1PggeF%Ok@I}=F(zbv49p~8pC-s)HEW}B)V1w#@(R( zSMW8`a5R6yJlJQ3NBMx6o*soemFv*`x$lIwHCfKayTu5?J^A#)h1n24r^Y9XcWVpH zpO~%aT(RD9hU)gP`4s9v(-zSXtn>+!x+*-{F=h5eoyhp_X3BUZxNO<<^DHsqua08@ z|7~)~F6M5Bp^aM^_ppN+^EcB)peRGt5N(U{xSo-Ah8J^mwC=z@5%q@IOuIWkazUGndGLrmS zZJK{Mes0n5J;ct=XEC&Re0|g^F8p0k&l(f;bM`NXU6GoZJoLyAgw`;C#XARklBki) z6F?#%cT|<40&Sp2)X`tSU0ntM+wqkQ5U@31%PCZhdVpxPaE?n*MK`5?6IovKD|+|i zwidCnDT8R`JDkKbi{OrySjt2)X%#qVv0k2{yW`CtY5QEK6Dg#jtHkiE}UGAo70Mfr$_ zuk~B=69>r+5}^JS_1ywiPB?mI1?wBp6Bmh;eco)ZalJ3AtS9V=z3R`9^k3X;p|0j{ zIutqNbu_|y(|WHH`da%!xBJa6oh^^6?-5?U88i>Cla@sR(k@7;>k3gtKdI#n2r=xa zta8b&t;U`-u5^RjwV~bAU;41Z-L^2L;px7166NpRrdZ%e);bJ$6Yo< z5Z2ZIGP|I0i}0Feq)oqN6&|+^g&L$Es9PVPcizW^qsS%5vOrc8@y@CUn9D9lJJHZD z&r(;OUZupVt|o@vJ|T@OK6^CL!hG@RCShGre)G2v8_dcEvE#rRpXnT#V~azSY-fgH zfY(w-7h80gm11S|Cd8I0KX~6UrsY6+1oFqiA!HCSZIwU3+nyE?S)hM(GTOSPuT<}C z0Yzm>BG5Ul(Wg1H|Ls$yKI+!-Tl&P2zV1$Rd`IKkaSaOwHS6cX2pO~jIsd``6}xVqD*j5T^F$|IACkqKLT|RzxVb^uRcQ3U3cp6thBpiON0GxING+TIKG5(MB;84!>TQJ2tRWN32 z9H($hv1#^RZ)(bv9l!q+Jt41CT=v}AxYVDJ!YPACW6zf(RV9$o1wK0|7+`; zutbD(*(qfBBsyd*>lur@_1~VPFx83 zX3`SsmM4WmXYI~iLK~Y1Huitayeu8vDj)dhg{Aqr_T!C~$rh#*5~Oq_D*6puHMz-9 z)oEt%YTxTUxNn6qV!!e!M8nN35+Gd20V_PYNpuoWMsaRPcw2F^|E{ME(ZF~`2_F34 zH7Y}O=GC4x7&T~;#QWkPdb#k#Q6+d05rCNa|E@3e3siT>P-zt{&U`?-ED!AEb+)i9sh3K;Gl(=Riw>qFCR20T} z8X@DRQUH^zst44h3bu+{ebiW6C+@kwlb9|uKy{*jSGvGv;$MzbVI$je#V?~8vY<@0 zMm&5*fxS17d^2AE%oq3clP-TbA})2$U5Ou(F}XjVHPcni+TYB4febbdlM4+aEC*k8Lkpsf!_bE4H9~^{g)#Vr$rE*$}Wr882@*Q z_pAziBk=l0>tzM0eQH|{w^UcZ2(t#=-%g@pG^uKhfWF-V{1~Sm04a7Q2iKb&Ju*jr zw_6NvBu#L?1^mhPgMaL$?;sr#Hx4=+X#d0%aa^}%cB+>7zD# ziyFg+(%UeofweO^i zyK(pGQ8D@!uze95=xMuA5qHKZ))IfOn44E;_w=sn`m$WB%O`sMFI*O~%LgpZo0lxR z$T7iK5oPjGr(kAu8grQ1-;ywecxmXB*21Hrs);T~jmvF)ZuN}WbFFpd~4$%tn2DKTzX zFdW$Z*h6rp;|Wi`F2L_`xqR-n>p)iI*o@_~ibq@f3)|jZesT_t;CLkBYrqS&Qwigb z3M4a4(M8*{rwIC^s1K{Kwl-ZESrS3{M3iFMxz57qNnPX535T}j`?DX|YpE~IoN9f9 zs_bgs)36_4V^Qx1O)QaQiS4weDw7Xu)njA%I4Hb?Y#?hmz$<_3*SYd z$X$5q#m=ET)Lmd1)F8Wiun^lZMHEzH%F7=9*%%!h6nO!$>dDu_DWjb!xRt*x0ishZ z(B&~=T*8jzlD(g=RIl#G95+4v>u{S{{$;Duy&Qiza*TyW@s)QURcoWbu|HS!iOi~q zKg16Flb#<=bYKP=G@f!%c}^a{Z<`Fr)>FifFrJi%+TSc~!=o*f~3yN@qWL7Y=JWEVHm42jop!ulomzG7H z` zx9_VzW3R_F4$Io$;JE6N%HEZ%B4X@^6kc^x=`eS>yt(15Q;^;#H{scabr89$YeAump6E?pXf~(etfxOLvhgXRChry>(Ozk%}av3BxB^l zW!_)kQ$xRrF}@(DR`13p?GphJ#jvGwRzfjd#`;68$yyyl5<5r|C*CFEf=pUg!^9n| z=;`%JRVMOwq~Y&8?q<)CL1#{$S&R`K6!?JKps-?z?xavW5NBG-VS2(hmmcT3#i!bv zm{*MS42^rDOKiq+&D*4UJtD0JXYQi^3|!(;$nk-9_*kmo@c?AD?h;!bs?%npjHW4j z%<=+^ZmEw3BwV=3SMK*A?OWsCk#enVm3`8WE(-~c)cn?+Jc`Sue1Zf$X9Jn2_bgk7 z-{`$`1XCQx-WyH+r<5`6mr7G2Q#wkC&#;G0AFIT@x^Gp{>y^Pbc)&zLFTE=;tnDGn zPpS=b0EpbiVxK~FW{}sa1XOn(a|;L`fzGrehnx^`kJs$UA!dg<$){W0Up5J5>DE*M2(-Lp6i})W%of06bh4%sQ_%5K_q7g58 zn@$_<@A? Oa$4t)F_-J83sIG&pD1amFY9L)?x>h-a9{NrW)-EX^U6jOM=8p0^~~ z)TgC&({HKPW<|j_?s&J!%?Eclwg{VC;QFkAj3Vpxr%@@90W_&O-tk zGZ`M(w_m~zStDr5%djuuJ0;g1nolCkj##9)zh(XBuq=a{&YVsLyb&387g|uwP%eP& zbF`zgb^i(cu@R0(uPz|QFKlwjuRSu=BpaUaOh4H7-mf%u!^p1xS3p0Lz~k1*8r8$~ zXb~32RQ6oUm3-3wdWfl7XrlYRXN-;h^f79OGXirhT;ZS$4V{msUPKKg{{SO$l!h$K zF*xhjgjjchmoLS3L_ytPye88`$d31#gqUeAF8IWL#|e?V2DKeirUuMOMUiV`JwGsy zQeVVQ=IdS9yWiF`Myw9NUSC{t6}~S9zYZikol4iwP3pfPawd&1w4;oV$c+liJMk!! znTnBPpqK$Pb|-w6DNIs zl#|yhZ-qmjn>+8CaB-osF$QQcb>!O0YxLmHor4eUHLKsTT_%J<9EKorfuZ94S4?%u z+vY%g%6l_JQH}ZVc6BBjCgd^|Ku5d=EQX8Bvr!}TTyh6L!m6_GOSaF3*ueD@s95Mj zYYVk6ya={Vwri4ZbL+Nrhz7}*#q+O46Cfc4{W23azcx(rp8+7BFD=Zd*Jp59(y3z> zHNJkdgyp^ffFN-C93t;>8C362i@w!Yw#nd0^mO{mf!R*5_arPqK|5Xd&-HB(6tRaN z^ulheRw%w&$D0dT4C6aU&xzzgKv?yAdd(iXMN6+(>bvbG7JqmrB)v@mxz!)cL{YuU znL$8u z0eIgZ-ZA|D%bonnGjswFwLp&C1)^dA>_IjnL>9nU@qoUGo4#gtqTa2}&tbb`B8r5fP+XF>j; zj=3@5$?QN*zDQI3@+YR*dqvyM<}h-CdnOIx$U!Hp7|~-{U^eRkZWwreM`C~DJfOu#Si^p!@zmtFZs}f33)UA)PSuAU z69-QCag_(1;2ZUq`7Z4|@$J#$1-7)$Xk3k91rO0+N?iWOW*_=R})f{eZ-w8q7U4cOL2mW+P2xtH7Tj z(<`Xupk=#yu}_mFS1)AxX=Gu??R<~fr{?E2#S&RBEf2qX*|5J1?sAkMM7j{l2%*pC zD-rH?e*#V~RrJv#=-2;squS0oj7~b&-GjdSd|o<>eRvMEJ4*AnkHeERs2Np)XhsTS zyx=gkv;vrL?R&p!8>=a4radu6HuCAV4{q|T$V2Wx@A6#lY}wjz_x9ERN8YEHp-Z=x zS{~erIX{!yb>e4-6pv9!9`_{!_3b;X6DKcn9_&14O7fmWpQTrh?yTuk)IMY-v@QnE zturl*Og^C48OyrZH}CyE=A3w9+)=d%=uI}-K*aW1T(3M_CeN`#zo5)UqnMqIOeeR4$908f21bH3mUJxm|I5CaD;rS)ivxveuv1du z$HieJ&Sg4j(&Pu4cWiOT>)qW9*Ru9sH??SXVXVFj zu{XpN!bO)({d{%N$!k{({sW;28sVCb%cMm#rh1keW!ea-0DJ7_{c*Zp2E@v0e-E~4 z!?u@#2BH0owy67Jh?jRe?-FX2oJHd+FJv&E;E)363a@| z9%2;&)?6ox66#lnk0$4w=VG8r^+il>z4zl~FfiqrW z7HmJ+$|M|MUJ_dHp^%$$KIiqxGqSNyoXWXEZ(eKP;>Y#|B%qTW2GsP=`7M@%Z3*Q#Lpm@4lb# z?khF4);ny?!AkNb?2FCKPn%Zr-$HW7JZa{npo@vr^QfV(V4S=B#Y@mlLZj-@2s#{X zJ*FX2$b~4f)4GUAdt4fo{xvIYFPr~$=Pn;4UPOUD!-FqBkVay7Y>zo|pEo;nQ#+88? zBT|__A{cRMPCo@Rl$tf|^w8g2URCTi|7?$4?DxZ0e`@J?h1hnI+OgiYcB?GQCAcN) z&X($pJ2Qpnyd0qQA$Yn1|c5hatbwh{F`hN ziJ=FDhxp+ihA90}8`;OcSal9aae?-PQ4rg5lC)KUCp^IaILY4Q5DkX51bpa`S<%Q9 zCaKkZd9Hk7ziLK&#G#FrZAug@H`y`#jnzpZw6Ge3ZB~&}RZ9@5f4PlA~TcxZDlmWt?hQo&Te00B-w$D7H5<`=y7ZQj!TKN~0 zw$mx;^~9mvX2bo|@>4f~X!;^_`UYTS=sd%Os z3iLjv2(SXGS$zu0-C&qOEp8J2-nB|nr9WEh$Q>(p6@29Q=xoyA3QX%EbU(&$`ltiS zvJn;H!PrJBG`>R?V*y7l4{kD+ega65o0jMTAY;0Maq8{nmTkcSUh>b6L{o^I2vm7^ zqfq1w5V8MDrlyZOl~KPWmtUo6WyqQgZ@^;@L_F?TJZxx-^yO7GlK0>h)048`zijon zMs8c`?x)KI{W^sXEvrC!Rq8s3Z{4abY(5EEvc88(U}a&8%T#12Ncm6OOAbg$De3v{1KO_ha#5#c5(Uq4jeIA)tkt&Q zb+MydVmLKBJ`3g=$9*Ntyq`W@nBr)V8oZ%YW+^uCY1-Y>AyL^nI$0EK0g7>Jxl=!j)!^{O$;@?mqM@%+5WjEov zu6}8?z)So-eWh*hmDn#aoWnGU{w z;|rgx^vu->@$KS|7>k(T%Ots-K$2hb_Q3M&O>!b1ON>_(tG6(Iz#-G+HC1M zA{q?Zl3+#e$Sue*Aoo(2_7etA^aBPK?`{{v)`U$>au*!Z1o+z~OAi*u*-RW=)!1M( zk*Kx^SW!C`q)i@}2LqF&!R1|}7R=g~-8xLAStRHt5zC;s{)A?{7}F_V&eyEQ|7s#? z^~HO_0$PpUc}D}jM)7)!N(|cg7IPH!;=MIU{5$mPs~8rmnwqixb~OHO;^6ABnzYJR zeViOVU;Ws(O6jLMJBU9zJ5hQMqE@axe{#85z&w9U04T@j7*udPR5uo}LY-2DE}^I% zi2B3RV0c(8kEXRIJqjx{XeSGr#tAL%Tq-=^%=+X2waX~6c2gf2ghR*hR5!JH!AvGM zoK5SQQp2MMJx|{nio?_>k~;N5qR|||PCg^Vg1SN3erC&*fX3*9gtG2$3|xP}gNS1S za`PXf;Np4Q|7}E_aO5?oYgN)m*}fGMkssKZ%tt+;$A4z~81P7l$kg_;?TJ(yLR=x^ z96%@x72&cGZ><*7{{NmihrPK~@?6Zsxwu{An@>w8dcmL8GKGCdwI&myYfml?X-$Z9 zdv6B%KjgEqt6oMONMxU{94O8Uh_I=o@f2&^c0ijSZR{zZaCN%3+)^&jF5OvuJ|yeb z?<4ctY!+i8anInVrN;4(QLHW|6a1kHi~(-6UpNh&M{%IYqiUms>GG#f5K*X6tYFt^ z3&C5qi+)7+n#$MH%Ucf%x{-<#Jr{W0_h-Dao<}`}$i4h`nN_%#B+04ggo!Y-?ko7M zU+d#JB*rEh(8QN7r=lFu&+}L$9v8W_-VcONdV}kJsKzVZeQz937Y@s9N~`T7_X*FS z-k~|6x&r*y^ssM`G+f-Q%gkIoRPn92+#BBQ7kh|#lz^crFa;CXdxaUN0_@mAD%U4p zFD1`ls$4b=EUtdn7C6JzrY(Ecl4;Vm(DH#Wf_sSHjpNSG}-o}4B zv^hP`>%fgwWSYY9Q>ZA16QB(u?^6?b>m&O6e#ei8Hn?5~C3Ip0*fL4Z@2u#hi|z4# ze#Gp!{j*<$_sU*6_ZZm-JP}75WDii$6yh|rAKqv3jmnPqdIDUdHej)R?hki7^94m* zpPst)I_R~je7Z%Q?~V)r3~djxYW02DDmVYd^oZm{Jg)`o{7S@3$Yb;@KY9VA47+te|NAm7j7aCNLMCNAQd%R5p~f=U zYTbCJI(#!WVlh4UFUO&U&Mb3g{x+E~A8b8s_T4ZZ#STMv@?%oz%i`bO3mcJaL=wJNM6*TpBdn}#@BkY`xv`M6~XNnPj+_%#R zqmO17#go|;pzl2f9Q>NMu2o0>*E{8!n>L^`eJ;t_G5jj;qktypQ*u2Rgzy7sxc0IY zFb%uWyUWBkbAF&~kfC1sMRa$vmqB)=Ka^t(@#3k*$l+hu9z69LD#AggZ*skaz;BNf zT0DbPG|IX+r81Fsv@>3?^J+^e6i5=%zlpJ%V$|HdErr>BI9FmBr#_FEpm$)Mn?*xv zPo>L_7)?O04GD4%?FdBz;SUx-kgh1Vb%q|hO&DWHooPv&LPa*)9~qQ!EaD@2y>$u- z^!j@I=iND;+iyD{!7%~+Pfna0ixXs`Z`u91^h#ENFG<>^H?UT^n>b~W5zZ)dcI#l! z*78s;iRs?&{1v~(;9gnJ(f2GWt(?M$7kx~2$nBiYU+*~D#Zm`nF~K&ivB!wz$meX4 z#j<9I-_@J-eM%1Z#x?0mp}hCJ>V5UmUV&W=(7%l4X+S(Bpkgc7KYO-+t6dvbGc$t2t(w*Ym zC_X3|yV>CTdTM6GREqXML8ju1VU*I=t#cO@)rXPz%jQ*Izbs99@GMO^N2nCxHEtB& z4so`;trC>{W8ScrMyR6?{bqSvpnuP*+uTF1WLYV!g2u!hV@Boz`Vh7YwWc)^Y>Qy= zFD-L+S+|f%5X{(t26meG|FXqvlP&B4+~7KzoUW8RuK6N^e{u7Lu@B%=)*;)7mjws! z$W{E@v7WWZMh5)1#W1Sh@!dYN?)gz+@y30u?-DHp6>1`Tar3pVW0n3B!akt0?F@wz zOHJxJ{K*#0vFg4S2zTmR=NPiS=fIf^40h$#ZKdYvL?@n_ktle_O6JUohKsnM8+{+A z=%_ZJfcg(v-GM>NcK0IdgIDc=fK6M*#cI;yNtWw%58d!cJ)qo z$K;K)v+tow)t2jMkAXVIbRmWl9?oS&*mmQ&2Mvw;cnz&Os+d*ew;WfgCdY?-osd1! zh^F{o#~F$#H*nMQSi43EjfOt@D~E-tS_x`rq`vc%0jgeBxH#FeU(-FZ5O{6$RTp<^ zME*l^i}k)*aph^U(}6V+1S_P6fxDxyl1_sTsee&}cC6-jN^+E1WoY)j|n>qp3&dcrYCvyVl7;!J)qMEWU-Lu zSH4UnMMp3uiU~B^zbMc*vDwCqe8_<$x1eKvg8p!)aexshQp$h6_wu*ot2I8-SMwe? z4)MXl&FPBYX)mgd5gzTiLV02MGi(J%+oBV0!lTm!iWCvPP%^-3+%55wt*}g7A*>Vd z7SfVy70!xuA9}Cd@{pI3psZsQ z>kr)K$CRPj1-WdIcK+L(@e$p7{a2vVAHSK>GhfRcoM{FQzO)b~@pUM9h)gdNQ?V1Hp5k^E*V%6Y=ujWUTELxD+2SV?1O4LOj3-5K{{^hzLd90K$p_=+( zxFh)C(5Ps&r`-FE`VR^l1c06%&RxDv?WIq%Dpf%LItp0|JJ*jGx4cyIICT}h&$W>9 zvHZbpq;mVC7;?ybe=-U7a_SISh>jlx%B9aqbhul-`^tpLlzO;+* z2S@XbaABHSM>Ft0o+|wTE&N}NSaDs1Vu+f_42+*B~ z|BmnMdFUzd&3U^Y)?@SMfOwh?0s!^41QXls zKu#0SyaGxhJduSL-K3LpQ$^EPTSQqyULw5LyEdk(KTjN+Er#zWf7HvPmWdf4rq}BL zi_8@YQ(=BLJcMztsjh1^M;S#8^0JbXR-j+Gljv&!z;E7}_4G-=IhuBqzI%arxjRqw z*YarguA-q;WONEUsn(&?eywt3| z*1-KnL!ng(ac1PfmCu5byKYqy)=bgW53Mj(S8CNHXsUB_XDE{DjYP+*y**|2yoe^A;{y0ui=>~2C&;mn+Fo(RKqo`(t7h1{c zf&S@-%DxZI>2UpG_f))g0oW^Nj2x9PBz>RkN6>){fHLN!cs+U_^LRInj@s|4rF-Sz zL0S7Zdv`gG?m_Ww6gXx>GP41Mf_nWxm_he3x|_x z?}0+?p9cKybZXiBH2m&ZuA;A*3eqa!F9*{7Ml|>x=)}wA@b=coyx=ilp*nkNbRC4r z)3o0fE9lHKNQrxfzXaK#6H4F{_y7`v_d7g)x=IKVOV6KZReBbO0y2>iAcwTAQFM@;xe8B_@&QNUjB}ID zu;(^%P%R3adl?@!_zqCIrN{$`xsX3)q+|<^PxN+2YXksGBpkTu=k9iy)YXV;(aa1+ zp=kK>H$=WU2IQ}=mbRy;#D-N_c;c=%xkd#~&%r*-%{deM0olAmSRC-LeRYEK#5wI5U6~8`pHo`W!-cXizHKgi#(nTb^ z*ZrYVF8}-cL&qz+-#gBC$6gWBs;ap_`pNb$+Ob?%?I9Krd7-*oJkqA4m zV&(l1bP{m>-Mc(^0gHOHupD z4A%5-yfM4#Z3NGOW%dPXWd^J+WC6j=oZm#fr42ncUr{Gzb^6ziD|*MRn9wQ7c&}PA z_0Y}^u#W76{x>fDZ)6($5@n~6s77+YoIs&&mBC9wv|CmJE-B%kglMRk7DRr7=IvzZ zkZ{br@hB#t3AZH)O|c~Cw?H*ijMR%3etj(7{2P2$&Q5(YO zPIFLe;rY?Yi%)1d8*C^82Hz$3KowwgNBDsZ(I)tlDQ;&@L=G$o!W<7V0}QO}E_j7e zejYOD>A_Z1=3pHhTyIC0lf_rMsfFPp*PdSbDQ)d&ds`(ZVv__Q`*92iu+NpqEltRg zT+0YgQl!@*D)trJpe(kiy2*2U)5KM!Pp~_aS^FB?Pdj#ii~f+6K+s1{j^Y#=Ne5yo zHPd^zn%v7-hCVIlg!e=oPMo{QrSy#MysSs{Bs!yq1}DqwB<`-MZp6xXm5Rs-XsQ^; z&McN!Uu*pntn8-C$36){!6$TSGE&G>pXH zt3?0qgX>nb>9XXHtG7=F8NbKZzH)x0>Xp3v$*5P1;byQtI!&2pm#=P(IKe35XkCco zfs8J8Y^@T;7)CRWVmozP8a)H$dYB4i8PU15L!3AD+zR;e$yPnFjS*WuA36eNFKOkO7bECoG;V#>%c+>rdRa0BXV zOPK19N}7auaO35Ocqgwvw?<9l)Rh-|1?yxlI3LtxULI_N9bZ@8a&95{`-$`e^GeeM9BJ(gfE))?p!rs% z?dvZF8P`y^JBXnHo`I-kXRu(Us6-s(LHNhzT}A3^$;wgRByAZ=Cp?#Yfj=5s)6}Rl z?_3r?Og$5@=Ma`MQge?OwhEsRZ}OJ;w>m?=!o5?g4Kf$a6&ht_20YZ1RWlgf9o%VO zdgrgl7CaaAl^3mR!e$D{#yRfo8qR)#!P6u9Q}TKpSq7NC+&@^WZGd_1?qSt1xxvK< zu--l)#!N^G+PIfhc~l!^AKos|JlN-bRlp)aVdv+;&6uZH-#aKH(6WiSM_tdb?D90SDxLz#i;$R`ey!s5y^J;f~EfdRDZZH!!f9N2LV`Wartqik(`t zUY2_ys@zEDV$zvPWMJ#_aj98yW29-~e0i%#NM=k(H`{P?@z1*_MOJ|`-^R$l)D>e5 zZ@q8OxIBB&b2-rKPx|}8sa2*}0v*Jx7q;*&tWdx%#xh%nwtQ;zY*&S8!IR!Mm6Zo) zweB+B8ZM04cRzdnY^Q~0Jq3lb_cOHyYL%E5tL^yroh?nGulUUgHlDH)htoRSqE7v4 zR19qd1&DPkJ_5J+Tw!FU^@>maf6 zoM`ATN7gF3$(t$poJdO`iJ*HMK=u`1eFaiv_hR+RNTG%&9Y-(pp*G$vt^cP7{;Bt$ zMtUA-6d|hFic0O+GoMhqF;4g$PB-MvWiTV>knP|^t9eYl-i)Sm^byTW-M2n_wIUN$ zI%i);9VguH+hNcTvWg9lV(Xxi7txH1n3Hiz?P+HmM%L=dZ(i&)lQ|8lW4@oL;`{kS zCNA<}-Q{w+253*!_2IvcO&pA{9AxrOyL;aLuKPl zOqp*TtQ=x|^IN}bP6Q!&E+FP~;0Xf+rMMYM^)sn=mmyIJwfu`C$y-o~rd_Z!3zHvQ zjTK!Z!~iw~u%;4X_%oGOpR;9q*1-{KUaLi$YvDQ2WwwP&iv;EN4U9os1`tD9U4O7X zb;XWhvY|c=)JnnBt@YH^X(L$?DjmBzbUHVL2unPEoU&~Veyly1>TiklKbA@JqyN^L zuMByS>QoZMQ(kI}{G{*umjgA|Qxl}GmVABSw4=8%+eCWZ!))+90|8+F ztAtTvOI30cNkZQ>P--)BDY+fN+A&Q;UEU6B#H!Q$V^nKLrzkhqt=>M8uTD1|#Ru+_ zW}P(1PyJYP`4-=So&#oo;!N*7*t!$MIVhvS_Q9=ph*;XYe?0lsdcKn>q&La=Pqcvh zIIKqwJiLB4qVc3#?+N6I=u0>I?%$-86xJdJWquHTyRqMb-DiLVW_R9V#|#xu*cit6-N<3E!~i(U zKLLw7VKfc$N;1O-V^BgWghjmBbfoj65a6fuuYbpb<(hU<*kCbIBP2fjx%S^r#J4m-sBZy)f`5bvZyG}_r>y(NZqNaja`w$-E}XO6?wcX zTk`;p?!F;c2a0De6Q&Tc!Pw54ip`zcRkO)|Z&PEFA)howyv63k_8_@{;6Eus3lhb0 z2PCyO2C4jUt9LiGN;z`z%Hx)Xh9h7C@{wZY)~*7$h5nkAW+>|ow)JPzs{$@qYZAK) zRA6N+%&@p9d{OOXv0Ewf^dEc@K7L1Le?IS?d}AasK##o#d^x+DsfR3j*I?uMR8FZ^FUlz|6G)#Wx=u1Ub(SXbtTIUM<8-Pa|L@f2OeBk1Apr zTPxt2eHyk7lM*pQN9m!$F@;kg=hguPIMd0H_8fR^%RXL!&vi#Dd$3Ehsd^8@^99PA zE!~PCO}J9NK3o!}w@nO4UAj3kpc!|XtjO`1_p3Bk)Nw{f-g;g0->GyT)A@V28YWRd zQqShq6Gzv(p8KF5>6`B%Qsw_XRzLq2AO4wG?YTWRx6=B_J&(t*sL4qcS}n3a9iXI{ z_xa;rj$KapJJWwT)WpZ2yQ9S|8@aorR_KIDxhC*7eDM_TkSg{8wHIumUX3m6#Nc2S zM9QPM&p-fIscQv?+&~C>v6}`?BLr3!Soo|9py}tKeTM1YpD~%^Y%rCaujAZw%eaUg z!@=31_v?zYzV&ucKm&Up$QiiJXw9n>d`wECR3*q^n_NN3NJ2S=?ze82Rr$xJ@>oUk zSV&>)Xh{XQ0lh-d{xeLKeDooNJE%neSQ9EP3v2uS@wDPTV_}wMtu9dccy=F%O4G9F8W=-V{H(|ahlGK^?BeDtl7hOnF5q+=ScaKcnj-DcY!j- z?u!E`VpllgRc7L=HJj^~qOk8uZ0=rvtwsRU{^JNXQYN`V%tYpwB zsyfF~TAv`E1!js05M^?D0L%*hhfGR!N8>l?Flw@KCO90_H3$Chy5>Eedfpq)-ix9_ zG6kF9A`;Dt(6id;g$D3($bjl2^grfd)W$*{v@o(pRHE6j>hW+Ptg#%R4R?02>}V2P zXKTI90-+5qbf-OZUiRp8ga2I52%5z)xK2W=Pk!mgqVxKjgBxZcLnPHflhsm>b0k13 zy2Nxhb~uq-?feJ6VX#SJwOFtBqATt|;lh))U#&-oKtb{u*w{;)DXC=Ci=g-CJvA8T zyf;PelO(?N_Kmkozh!*%#kKCHCGV=H&+kYINy)8A+?`JyPa^1;jj>8ZM#BLG#-{mFSk-V zFTlps;G4iaR=j>Eq&C&jJIiHjZ`nSV0i8kZsb69v)d?}_!GqgN@29z#8k+~3IF8GP z$?z5ieg~SjUB7OJRHtQjLk%)#9(Z5pOUa4fR0CpJ7H}%P9oWMUPU_!gw|@q41j24$+6P_vH|EmwKj)AwgINut(^-+bjZ|0T(JgE^pM{3z)| z{FkvIsZ2owAo2k)VD19|P!rCjzyAiu#A*(Pm1fA|r$V$0F$@)wa(71kHa+Xzkrk}V z7I%CsluLL9#+cMzV)ysd9uDAR%UBx^Q0~@ztuEL5m_G3YAsrbaap?%LvDKo%LCbh_zctscf?j(t>f(7r(^WRuR`|V(v*#a~_)och@KE0R z;Z^?a=-F#YZANXU8aj`k{B{oA5$on&Ia@sC`Dndy;&5dO!?N#KaBBX$vFc)TrPbG_ z5-(MMjm^Y8PIp%&-v4nYkxxGoj`!C)~Fp zJ#cO8Ze|y#x+#(ze}D=#yVO4&0o%~@ri4AH!28zV;hfiL#+fgH8>}8pP(Rdh{^bBf zhrb*^E3x?rI(DFFu~wGq6|Eg9bs0Y!wd0aoUKV;HbDnwvADq!v)?}2ZjMy=7l~hfb z>^Md;rfz0|VU0++(1|?*I+@0kmV#%kEQdIsq2K#hN1klYXF0q-lUAbwBP|oCa|5mn z)3%mvD;O}%0GEJUfHu)P*%_LyaUZHW+8pceF0o_kBkcllp>GDD*n7b4!;PjrGmWU9 z&Tb^Ff&Ig(SA`A~Cp7$ez_U$czm+nblQ}V7F;#n{JPID9UXKDFCmDH8K%ikW0o9rN zFRybafeFJ#sRx6b-aIafSnBsauk3hi=t!HD2s)+z=4$^z53Jptm@c^zS2N@bR z-)Y~aC#jI(fa`g11trB`e7WnABRYu9OpLhQuT+&Ed|bY&CjLm&Nt<_(e4mMpreqG? z!MD{{I;6J~9Fk0KRM_!ldg5qJ{ii$3jQUni+dr5vGiyKNu$TOf&s3x+jUbiJVf8BJ zRDArzr@;{v)shfNLKv!I$4O$qyhQQ!nQDUq7=JarEud&-OfBWc?=(Zx_Y&6M`J>=U zV#Y(Ax7MM5R2q6M?i+E0e00|*gWHQ%M2g^+b#7<-zn;!drdzDjlXc| zo&QKYO*4{OUYjuu$sr;ZkXH4l&lKsC+OULfpy9ZIegALshW>vW&p*|!WRd{p92J{K zwvezf`LrO}QBbB`R=mGOvwGg~wHDp!9zT@m&lEAD>(JDeUt4(`;d{~*k+vjy~7p{aBnD=C4VO6A9=PLaa%jyC;> zg#sUE)Jjrx7$he-iqeAQFJ2XN@Z{=&5DUq?4n{)nT-SE(v3C0)>8=Td?XCP%`wzEp z1Uu#!zNIi1@c)gU@EM{}dR`N=ZVQbu%SVXMIgl0$8&k<;2#Z{c59F(y8+nIA-Cv4H zJyBip`~0a@p^MXas!HvL&-J92f|d^dVEAD)Jsl1&W*;zhB zL3}$2|5o88G<+gAP74URGDT^p^}rT6$Uyxcvc5bX%Kv>=QIe3#l5HwP#*#|PHf^$% zX|rUT3L!~Clrc{#WStL%N=y7|)@=6>GyeP8!=U)K`P{WQL%5;*}(Y^>j50&4CCk8`AcnWzls8wC@G^uRm;l&35^ z|D8-DLd3FQlOkBN%P`@Ss{OnwQ90pg@Y-C1C!B@e`{x-8@}Woz&;V`=G3-A?xM%Y@4cz_? zcxM0}pk3e&oxKFD7x2Dt!O(g07hD_Ihc77R9Sf~)QqYKHGWP#mc}1%!&OBjurP8q| zEA=1+e`x=j(VG9NFE@<68c}nZRZVefZ}eAkGDs|;^LPn|;NjwzpL9IDbK*x;HK!b& z;B9;7Je2Gre{^4QE6BV2M8CD*IywH{k&EVGD`v6}J`W>izw`9<(s)PtX6SjYzZGIl zV>>a3xJ!`!+-r_aI|fAT=)>AL`y?O(#V&&whgTI)B4{v=J5qqX!$t_gAvGtq)?cU+ zw()-kJ`p&_r;u(sn6BMM<+5p9hcTQ!Xy0qvdkHOoPsQf94;D};UA_kVLLJJY49hOO z6Py~l&wm0j>OEwE=^(!Km$U#$f5^Li^aUf(v>{fr=-J=C{l6do3Y{l&_cVeIQ{*-G z1T@`$MPT6DiJ7p%Oc#OYr2iJZHvGL{Elc4uzbtM5;Q5%Fg~tYOeBg~?3{G*-Yu;(bYbng1MXY-$s21iQ%chT4hB>Sl0;QS&(19IC z(c(vBS+;S9{%~g*kW>Yn-kRmaz4{Y^Eco-sO6J4e5DUX(KtCi%{Xx(&C`?7( zIUpSW+FUJlos%!(@K{?lPJZxbji>2hBzO3}JXOFC%}*8`Eq$tlAIV(tJv9IhHx@9qYyn-dQlem-xUitAzcm-#Y`}LryiF1Jkpv8X5 zK%Fmx<9gw{gbEFQCb>n(7GjjMw;<5W-IVEA!Qj)BnYQnU%!!4eSTm=B`-c5hu5(VA zuaM3z&Xvi>Z>W1%oRjgq-uhYfxcr{wmJJ_YI1E?jes!a?Q_ka;*T)EZFawMI8jXvGQYLa9B z7Y!D1daZx$BkmifOaqT{C*zF_DN-A7Yje)uNnf-t8PD#%AH#f)I=JD{CNcXhGx)Mt za3W;^yRLY*4T85g8EB#h*FIBp zr?$CXl=B0GZ}iXMht_YZR8m5MeewpZAV}+^h?bj`+O=oTvejR{5qc|WMRQ=ufhYwk zs`Fq`8ydK@pE|aYM2R{ys8@&{ z3;mj@ZaDW2q`L43hK>w=Ru7gi235NFEMqA1wPYqr_}*{WPLPD_^sC@t+5HvKWhjAq zGC2{=J;FpWID8Cre&>0_+F?De!%8>r8Zhu45fQwu@%a#%%hc{#u&gPr7`p56wd%1Y zJw|A(M1RUi(stJlR62LWLqKAUd3Nf1@*Q5Wj5~ zYAYa~+oUEbM}LjLrSSDGVh+E#n<}b&(XPqjN>MH$X@(Y9cviurG$TpyGPD|&>?k|Y z5h_Z1f?D0*V?ZOnx#(B{#}i-FU;t8ShZ|6cn&OMo5SypL)fB6;;ZZPIHU+`$z51B8 zahdNHkM6znvti%tW@x+fyyp9Xh5TqcAbZFB7ulP|Sy&gH>hs6v4cbqTl(`@QZ8{5r zJ_7DCldlG~GAj0UUtTsf-=;ub#aQv`sOrn{sGobSR82eUF}JvJCt3Z^YAy$J5Zk~W zTzs8ZTUpG+ZK&$e>s7J@+=3`a#214l(C7xd{LI&Bj;Z(s!oNc1b-o=@7gMii3?H)? zi#Mx?|Clmp<};}T5jrGajS5Nj1dT2eI}s8f;?z;J9Fo?c;1$cJIpEGTr&_cG)aV4H z%t9tu`Fbn@)nCP$mo~HV=iZ?xAqCS1nLz1Eu^kj-%AaE2SUigDE5$9lE0t_tm`7Fe z@2%rrggGqB(8=7RPvEV7pZwsSa@H?AVU6Eg;Rj?jyl)a8uon0Vc|2b?cX&rTB2>3o zk@&~6=Z+;`<(Kq&Kf%nPHM`a$h@#qT`W8A2OR>reM)7jfZd*FCmkcCs%!?yE;N^bF z;+D#lp*qmaV#v`ru`ZkfSNIe+kUlrg~VZ(`jiuZ7cRaW+l6 zLwaYcb&L>QlYGVJAa~DmRe3IaOE#V&H6VKKNt2V|ql6c-Mh0EcWtF6wC67bHXF`*Y zxyRgZ;w8n6f7zu2g^+e|eOU{2_Oo8J^P;w$d(F2L|8{D=6A>;Hb7cb)$Z!+jEdmhg zwmm?ujx28wwyMxjw^y;OfYq>*OLryM?{kL_pn#ZsDWPY1mcVSQ2t?zPG|)z z*~Wbm{1q$4aa?h5huA;8_E{ABGz*Xo+3D$o1u&{)|#Iu|%WG zrPnvO9d@JePF~~tu=NBu^0eD=uySd^D?+gV+X$l%F?X(bvB8}C8oqJr|CC^VeTp``WZ(S36D1No?>Nkbr?4&#vP8`<9nX|?6o3>GKnq#(4*ZW&JN> z2WBOLPE@6jO%_VoeQ`whhF$PZ^^}L zD&DTxK6b62;vP>JU*?N`pvWt0>JFaT`Hz%%zsRG*_{PDvv%d$zjqF+3KF2LEvmN=? z)9M}8z&C(N z4d95+z+!ri8aAWjjm%PhxgIGx6b6UC_5nwVDLFE{8gK<+3AEs!pvXjY3}!p_>L<$# z`)`iZY{Oosk?A_H23WsJ6qRNiwL>$hohp$kyIX8Rf5Y3+ZT+z>P;?peeiGpOT*EwA z%#McEwOQEU!i{2}s`nRsq>?9h?K~;QcQ;^R__AJLw>BE`B>(ZX=U!wX>Hd*HDCt{O zuFYfpoORcHQh7LeWso};fx!Z2$d6hMHTNT$M_nPXQhL~7 zp1+rGMaaAfatZpS_q$kLN!?Zf=t>{kgTox1Cy4S7UJ%AwRaEx!r3&f|=I)nM4m-~z zne)n@l?yq)jT8CGdvCF453&Gf-0AZBjhYrE+4s~`&)txiS%k9Lkx(k<8Ml_mzs<#? z1`G(&u4O&Ez%Tlbk{VSeZtROycO2cVesPyq_xGuH6;H3+$X5O*2U!iKHdm|!;5eu+ zfYmxo6xrgVP@KudVCB?|Ja2{D{c=0rsChrPwbGa}!r85~GxK>azF^h5>FC$eLrz?b z_099TKUe`BL+6YhB)jLaG`AX;KnSuZ*#{F3%7~x@|5Mz{1%QF#%*D?&Ucl_%eWuO( z7aOVlVaD$2s$Yw^5L&C0331*EvamLx{r&L4-n}7;)_;E7&K_J)CZ-a&Ol&OOu_pXn zgA>zVwXfjScl!0{D9SQA-|gzufcvadQ@)SacJ;}Z&lnesOUidiiS$kbl{MPpprm;?Hy0pOO~aG@^il(#Dx{7O)wT zr)V00`Qf(byUD;uH%v0hf&S5V=0cSaDSPdL@QTTm7l0~p^7>2D1Vf*E@0=T-4okca z*4fqc7pD6!sICu&!MhOp`}lOBLNo_@t;d&BBn|#u zb_${KO5gRwnZa{M_428dZL!J{O*eh7M4j5Q&vdS>v!PPUq;S-@@!GqbxHO$#KlX3^ zw0E(~Vo5`wiP9TkkiY8vWw`>+rm{3JuoM?&R6JWTC4}+Jy1FRv+7hY1oK(Q_=PPP} zyRV1ULS^OvAA^8?-f89N}BHnHH|g+##( zvK1`CnTc&#_i9iS&)=Cb5IK<*{(Q!MV#1uSAG=Oy0Nga_H@Z&688AkukLNt%=6}4F z!M{N<^Phsr7^&b}Q)pQfZx?|^RxMB4)HBlw+aj!wi~7i$y`-#7uZ20R_hH=yaVSp& zo-8$QCeOEHNk)J4k&~8Bk$9nRwK;UyU;EaV6Qf+W{q{02230;e$M)mxhx;;7f;V$| z3UGhJKxyCov>kp+YURhm?mzbyCx7>#MkM^9$Sl_&TUbBUwRBCsV3H;#k1MhAt!<~) zZh{GOaXzc0K57b(I|z@x_i+vBN0vJaq2&x@_v|m+c8xR^p;0<%H7FfmR{!j%-L*^{ zy#(F^O1`v z*&ir8>N5AK;Ohj>f*H+L7Z}U}kcH$?M|@E*jUL<&mBM}0ooetxOxYRmyzYC9AeH3H z%8k_`z>yn|q+x_+i~rlHq6(a-;VsECVG`Ix*MVK`3wqU$spnK4Q=H!8=3q ztXXiJkIu?`R=!V~>_KzCjcLi}R}aYV(IQm8Sr*Dfj<)`a(oMk7txzZOt?%A{{n=sN z9YhA1f?`=NJdGA)+{QUgWML)(Fw>DB&+-8XB@)NA4lkn$%w}ZPh;gWSkRo>QU#+}B zz)O1QMby=r2YBwO;3y^si3jBlOIU`t#NNQBo%e|l7Hl3}j_)4>y92!}T}_fK5&I$iGzI; zi}N@w?5(IBZ$x51 zlf18g5C(P0<84sA7D;8xSSi=R|D%h>)nMmY-Z0g1qSD)fzZod>RJF(OP3hH_!x_dr z-z}n7b=56j;IvN=GqH9&UtB9rN|fhJd3`!%ru=z$5qTl~Iw)#eAB_0LJhb%(5MI_0 z%TX)ry~ZQd4gcS>bhj360IQ(ENs+SZS<+ef?)i*|9P4>lur^>-+^v>RJ zMd)pqBq0)ai<<{;ZRBZ8ZAv?vX4k)E>h|1^YXD@lIRRO(RW}2dgJFezouSCOKwDIl z&ER+(!f`YfEs*>~khob!tv%0Q=Ksjap!xV6Az94thz}ZNF){->`j;*<2wY@4Q@vmw z-}w4Y%=^7lZ6YP#P_#=nCBR#+|L2INfDG6cRJo05uwiutNUIVr?*18_>YGYf?=OmG zQ4x9xANcJ?MEUEgtQp89L5aU?_$$ zBz3+8`>RHa+n%1X+bePv_lCS+xWr)>4s3V>r$hk!4>xjM0d3@#;M-C`0CDj$s(>g1 z)mwoHBUN1WE&F88O*HLGX=U0Xqyz2&%Iq3A*dNF7y?RMzKA0V(12d6H!QzTJWs$`% z{VwgdR9$Y(6?T7YW%|?N(Q#q0b_8Hm;&wrfvsay-+)fJLdO~oC)V+tUbkx=GQhIeb zGcPy#*G;99&T`&19?-Fsb@0j}xC%oA>oWdvems+{Qw>qK{)A=LDlnID&6rXH{9G3A zG8f5-?OCSu?PaB`4kpCgsvcq(o~Zu4{rGLcNH1r1nb?(vo~MSls1eOik|Q0D`{mJ~ zg1khNklautdg%^1EMSkI2f6wNj{!oLzUy!f!_?N*+1{_K;!cFcTd>UFuo%r&g(WfE z1V)MJYLuDmRVS)VM#(Ls!is06s9>40d?6^v3jdHh*g|Gu@PpK5rw3m;$aE!EW6FR= z_mRm5asp}vI*9PvVf?DbhW@v;;|qxEc!okSLi;|dyyM;>BZsxGxIdR-$iD5d&x5I> z5_jQ{rT&W0C3q_VbZw$sydMX%oqwfIhw>?HY#I%`a(|l(JJb(k=I8sq&S)tq8nSut z-~sY{fXTeuRnLFsy+V$Asf{I5kB0S6U6n;&7T3_i2TlZVr>Q$=XL zOB3b8E|JR zjCbZao;qjfRC#t!#mj@8PGf!}W)IB#BITqoSrE$^XxQT$ej+&NZnJ~9pKh{WSj%LI z2IVSnv6dr#OTNs3eWzKj7b-%vI#zSeZG1{y8F5 z`qMx6ME8+{^I{X$7HtZ5JY9-~)g&1tU^{Nemj)CHQx=c5@Qieve~ERT99eUm9)4)j zt~&W&p-aU}=WE~JJS!%$^Zb6sW@%*y9ZU&|4V$gHH@0?LbY!yA$$bB>qJ-!4g+Vt%R~s>|K@tA&Ewg^*@>;|Lg|R)y~rIl;knK9W^ueQm;$!er0*7gsu+j7xeq67&Dsl`kc#(*`Qo}xyQRFxZcC=Xjw5?I42WX0 zt0SNhE=)B_d=;RcGhEtMZh12@)=Dvd{Hpx+x@F?Dl;l!xY&~x``5_|h^x`k|HsA{w z%2W-clb;2Vw_wk7U>!aUhL<;<>(ROMN#`>veA`ZBO{fWWs`>$OWti*AK3s>{jZ>LM z?85E7^^dQSWVMenGhw)Tv(@z%p6Fs_c_o=^S00^|flYgt{p$f+L6}n{Ae3GOk`5?_ zMGRIw*Y$X^v#G|Fsnnc5AmMbtT)H8c8R4L6>Vhh=GE8 z_i7~qGJy>tO-iJmL)`6auW;rxcj3;B2SU+^hKs-?)5*t5WuXm4tkgeo6-`DsQmJKWPCS-^INK74(Ba(SKPhENr|3 z{3yIwt~D`z>V0!LX|TW?iGqzW2mbOV+d!ziV|jajg0-EgmbE+4-DSp~ad?@6GbmZ& z(t!bEm<)fsAavY>kc(;`cgv9yyITxv4=z`%sf61eD&kzjFQjoEfzBrDuf)1&=&4nB zH`q+>$9gjM#h3dA-aaBQL76`o2ky%^RtEk_9zIm-%e~I6U^}#smCQ7`(W7!rzQl50 zl17iN0dnoiZTr^~rxwbLw6@W&g!kHAY<}=lhVx2v>-9r}HAcOwCMFcaf|!cZzbIKO zK*^2*lq`xQOH5<6H6|^$yz)kAUaIcysw78iUZ!&%6NbB)7Vt?Yrj#{oP{apZ$ZoDQ z*N*)g&vhZG1Hq!F)OA_p_fTPt#NHXIk|h-Hw%_da(|?85mBamcN8z?J>CiLG&LYl- zrqt9OdU?YSPBI0*+eh)$7_!MsqWA{eV8kB`N_EkDJSE`RD^1-zmNdJOvq?~aJBaH+ zNeAPjFk&XIg}>cG369C)gW)O*bIGc1KkQOzEqnJo-m-U18QEJS2EGFkkTMHHCq1Yv z0KOD6sj$<`!_0L53iyv8D;HGI!El<7yfTR}Igsg~p%jiBEeIvX4`du8&W?rq=}KgA z-UIZAE6qai6)3z=-*?CPehl(vE^cS9EU4fBf&P(&cQNi#|A3kBUy8t7_Ts$Reync@ z%vW%WQl;=peuh#D?F5M)?yrl2!#v%^zSbn|O?X)Sc`HgKWz>N!dNjY3IhD4i+{5+S zgbpSA?2kav`WCq0sEIcDT63MqZEU)fcO{k6H6X>^>ip+Dh(G9jcs32i-2*uE>2;@$ z&*5a8XD;uVaJ2<(W9{%tqHpR`sP2Nplz$B5L;f>b-<;NjGwjsKFlzFDdkCVI-2k1~ z3b6ek8rSvR_bPF7vkLfED3PLs zSeJ1%llke(zW^OQQ^MZ_>b-emDpyamd4)xob6`k4?c1>-n317(DK*&b-LKZ(F;k;E zFDQvk$gTeh9j$`PA%E0>9z%+0OqY@iU1Dq{89JfLkhTnkeMe?t#MLa$|@ zEFh;Py~K%R$<0wglv#7#N2$s-4H@^1ZO*9vs(Lf~Izsx4RA3LGD*Jha8zNTzfqh+f zfHf&O5?wsYuK0y%W%;&Ux`C~o%gdyUz4YbVGaKSI;5K@eSVe0WEy1wp9vVW zW_w-ur`dD7jK%Wu?}U9_LO9EJqPd5YsZCAKe}CQk{+%t4c&R5oZoQzRz&QEN% zc|h=`-ZET&&kXzMV%k*HXQdd1$iGeK`>1lvw5oq16-YU3(qZ{!aH!&;_Nx~v23O}l zWq-i}{)$qFDu`*KQZLX5r0#nEHH|0SxNcrs0u|Uq1enD9DhbG81Fi4dqQv9~PC+I37Pku(wE%bLgsE|BEf0eNd zMm3(zDO&gQ9SYqB%mPJLyKA7;`tV9V5>sH4Wh*xXC-E&lP2nU<6{6!7{Fc)L<88vU zobyc0e743eB6h&tgc9o0v1h(2y4~rJ(7L=6QB1%Xx;T+%NWjbz$4p00ZxV|g#iC<4DbKN0f25_M+{VOCn=xVb$ zg2s!hhSt2)R`YLDS(?ea(-t4{lirm&dMCfFMr1h=_`9IdO_U7Dlx*+5;;(DU*_l>J z;f4=d6SytNdJ`|66WpPKV)eE^;iHKU2K)x*l6RGrKK&A`ArV%fFw)RX4QoJ11A`at<_^|4(SDdQH`7rKGX;-%&L*V3l|z;^JzNj2VPoSZ4r6yKWEIVQtahO96wKijzXLtxEh zd(Fv%m~toXJ!B|;w{&}mA_=(1=h@P>E-t+}ofGvU?aY18p5wD>&<&&gG1*)%^HQ*> zUNrjob9Ywa$jHbp>E>Pfd(I^63U9XCa7FS-;)!NR5y7%2aJBOiaJt+ySHWwPXRYOQ z=H@b=Qs!m*a)0Wqh?H(n;>lI*r4#}6aQu!D=S{>}QO*azCI)d`qS2%*PM4ivn^U6= zjvXOtGXq08pFTeu+#@=2w#!mz(Ncyl0=E)a=g|YzoI-9XSt1CMz;N&Pwfa?+7ZiX^ zX}`RE*^o&kVRKsLNiew?XLfr)WC%|G4W zDs0Lr{Po!%-YfCOwpa@avQ!Tctn*emCk1$tBYcFLUh%$ zw}mM(yYsg9H~nmsDt1(Qm$4<#&SH|#vcTP^$KN^1fTIm^@pL`zsg?b%#kY*MbveIm z_vg`R4qL1IyN!Yp-^gOFSu# z^}DV7o;J!raCIp4%2K1E;QcgLm5uFPL2Q;c7!3jh{DGWOV7#C+`ngvy zi~FprsEqQ`v**^O;JyC)&|HHBx(i8C@W_Dp%pt%ptwlwv=Cz#ag43VVo8Cz^co3F8 zp<&5P?%Pa7K`d60^+oKc3zyIw@asE)?3?HGqib78)0cW6;sSBip?VD03*>*OK`-iF z^O(zN`mFElhxRK~Jxk$tqlY~!gZap{i=fshfDaO5pj%SW2w~>_rM;|*Xs@0eyQz|y z{!K>-r^o-a;Np>td-lacXJ(2%QHLw0!j4UR@WkTFNK0$Z+0%pT7oFV{I}A{=p;45E z0X4Va`J6!IoGSMP8>GldO8%a$aCBx;HR|xiURA+KQ1m8ZB-82s{GD<%N|ah$n{UEK z9mjlm3a2Ld7K#TXS*Bmgs}5A+qs%q~15@lVFym@WP=rlQP`Lv>pYIEL-y;DD?<2CR z8B!iCR_Uvm2?;^nX}CO=C(k(r=&`Lqe;Ew_^MUw(g{Sdj4Dw?X_qabt9JLAMjYJG0 z=z?6Jb!HA}*h>@VnJeEe2Od2p&iU=|Lpj9qIf6l42>mcSjZwo5QojKC5Grv46BnXi^~fk+_3@!CV;p;V2u36M7Svf;qt# zzb|g(@@?|0yeq<`*E#O#ml`hDRLq5&k;~-fuG0u{uUDH;^0Vy>@#1_JD^H*6KHA0m z)sH>@aW~15IyBHMIwDdrJXK=@GW9>)lU_-5gxJjL7}VZfe@jZhTKZQ=3x04y$X!i1 zei_CF%^l0V<3`EBwK+t5E^~3)Z)X+?aD;U2tb=;8Cl z%RJYne$2YIbALN-KdAci;QlK3Z2Ct>?>f$xFY%-7uS-91fuq8vwPAM3zL&eP26-8n zrVwc#&Lim4*Wg$bZ<o-~(Y$BHu>kwk~%D;+6C3HH$ zWL+h&XbCL2WcuP#MqxDp3FW1;nZBc85#Z<+GkvlNivsh>KP6m;veSAwXzpE`PIb@@ zo-@D7KgTl^2H+SfO4G%!VOiznC`-vpT?Z<`Bwgk7wLvmPjUIwBmQ>%AH{OIha z=D9_4{Q1H$Tne%G{iPUz2(H73ltL#y$wWpsIq3T^gWugc*ifOp?~s-H``Bj@?60q$ z*_heNJEE_@e1sCOj(=TeZ?;YCe^cBh@ug_N(U%`LXZ4^_T?#NxQ) zMQWdC+z$4s9Qd6U%`DN@7;HOxMR*GN-A4b$82@KK6(sDL+)nhL=G&2X!JqplRo+~* z*d=M(Y3F-8rfRDuOi0@WS>v7#2VFN;He$!l)*!&KMj>o3lbRz4d8e(9_w(7^%$%>l zGgRGu-TrdX7XF=<&Rl9E@!Ocs-GIs@s3oUY=C?#rU?}$;#AapwLge9% zA}2&Y6NRN2!<+b9&-1%X6-;&;vfJlTt?hKmoEXoa?;`!3HA>fr3wHG2RY?2D-Jr5F zvK^xt4J01dAM|M8@Tk8G?^z3^a%I~ak1pasqi1{WR~J)dw=&lrcMtR|q^)11@>d znBMU67S!|sLZBo-ChS2EINN)-H&bI(6gnKp9J{iS53Q#9hkolq3OqoDBJvep@wBGi z8FGni)uweGr8nqz;qtfi--Ssd$1t9E!)KQd1AjBQ1z60PHTvTTEQ{tadoYWQmfghp zKzMHC)F4=CvIYab|9G(3g$%d>@DSH;_2~^-SFoO40ep)8y5l>)xj( z$CYFt?@vg@bdA0*ie^HG=NZ)^*!Ms;6JY&Vod_?>R|O5Ha>^-@w4+%$Cj9q=?mXL3tKYc2DTrw~zyuR~@C9Ljji_Tk{D)D*D}-8Kw_MXJ;{!lRR*Pc|y0Pe)Aer z2~tOR)&6o^+aI?px>m{_NEC)2bs9R`rfn^H#$D~&-Qrl@U$rOJJpxmra0-2$HA#n~ zvoKXmo@;M&GuLIsfwKfYHoI!Jw{O4eW9P>r z0cVWNMYKfjwiv73+7;YlMeHMT_sj96(_5f&^VBr+8)8le zRG;m=emnEVAiKe6g84f7}~DJgeNYTV{q zc10@Bem3)gNq;kW14-KlL=Lb%n1{r;`#oz}(A-My(C{s}iy575j0{(g&r#bgTFrA_ z)w(2}DV%NT#AM$5y?QBF-^P=;hPeE+QS&(H@_OMUYPlgL!xw`MLn8YQTk1kjrDC2YFzTrL|E3$A|Cc-S z_uqg$?!hWX<_UvE46~h`2Jc?r7;@9uuMc432vVW`7r&O?Axz^=>wp9oaqgRFUs+>k z@cw*Cf#t8lHTNIuhzWgOIQDj>Ad7sP8bg&eZ*=){3s&p#_}nHd?osl^dBc;s?N3Fv zZS>_SgX8ZG%AFb+!`lKiu(^1;tKQW<6K96_Y`WqgPg8e)V(Sm}WDBL_>_cxe#ReZA zehrI-3J-PR|b!OdkG_U;?L$4O8J?FQP<9Mdu3&C_Ti6RHE zRb^IONQttU>EYk_tC1f*R=(;`(jD`g!dH%A2btxwAezC8%EcQ!WxQ{Kke5Y8wx(t_Nf?T0 zn41E~GIxRiu3cas^XQzMMa;q9qLJFT^+M z4Hh#|_SkpY&(l{sL=G=6pE}CixM@q_p(N2g?g4Fn;%j;gKzDR#bY>Xg{X?)Nn#a)YALr}Piw^Y- zr>VHkqJuRM%~cxZ2re^}lgA6jbw4Z`ip%rWzxtlP#c!_Yjb2`A7ey_Ox#Xb3+ly;M zY$x9);$IuqVWQ;|Coj3H_YTH4dwR+pc4-yebRoP~P43=y=_Cx5Iu{M%dPVROb%VNf zU>dQt##E&0ho$VKYi)vO&%6h`f9d5-1Jmrd+wNahG~$l^y2~}TNS&K&Tm0JJdmADc z#^Uzlx}~q`vlG8EY{HDo)!r`+pT4s;w>uTp5SGQNnyQ2|3xe4f#TWQG8Jp_DT5Bz|vT} zW+!|^jjy}zt3n^S9etSN!g+o9dk@OI@4*dgfcYjKgJ4f+ff2EMRzB!h^6^7pZlSsEKQ^2>BkvRf{q3 zhxB|%EdarrshpkIbpYC7g`M+9OC7)>S!U{-Y(yI>5;yywBJ>`L_5n80hwg-8ELktm zzXa|M<#$ju_D#c;>oV(^Kbjd0h@wlQ^cDuJ_^At+3h8^9Qk!7AKBOx zkn3O-w$k5*`L^ui;h4`9UPDe(a#Wu5k-?S>$cVS7bC|3xliXLMc-hcckD1h;UU|+H zy(TeZ2-a@%@Kx3%yyY;s)AVS0q@Lcj`UUl)-AB1#1gC^+rIt>%C=*!uB<0ZbrjG(eo31>`fmO&IZ(fC;ZcJ?}R6 zsRgQuu$zl+{Ks%4QB0ROiTS8gqmO8Cf@x-Z;SSUWSn7-iw`9~$v|%d4w&=sapAFph zeGYY)9l+cN)YRxGb$TC3&nILtZ=F8s;y=4QwW9rr>6Ca>{Z_!rj7> zGy%qMz(O=ZTw|a_w9f{>E1L^%pI{tc33sw7#E0rIi-1LOZnRAd+ka#N69gfp2FH(t zg;SQPtU*}p75H|!TH zrB7EiYDy6ZkK3g=V zMvpFGPzK4|lCAQ513e7OPnJ)c`E@FmoBr3C7^H|!)>}&xFLriAjGBqb(T7hRmJTfI zUw!ok;a3Jb!u#3yIKCf9Ww*KQVD>=-*2K@hn~+zBB4a*yp_Xk=BHsJ#XeRLG%l;v@Q=fs)a$ln{Oozqa z%RQcdYY!`VR^k0o=kOo)ELSf<=RW*HSd%*@x$MwE#~yAw!1rnE{V!8u*&E}m(pqW3 zL97MHP&v>Y>W)`_B$ZI-1~d5xcrI#Wo-=NSS8qK9E&Rl8Vk^Sppw8ZOj{90I&YoNG z!RIq-IL@-Xr2D>2L}c^Y!ZaQ;O`)43CKrkK6Ew}8#PpITTMIMijHUh+ zvNy{hTWd1caaZg=9@R(|Y>ahLIN2`D$Ftt`{gIL3KhvzALsCf!xK>P=Xv?((7M|W7 zQ*$mkyubTMdi4kPEJ^d#iPc|7u<*DQst40RT7lWYPJ-KKuGIs-T|SlSE1YysB=lu( zWmO`8QLzpg7{dl2Qff)*&3@>pu*ALA{#^@M#HK#t`>@C@PWVRhvntnLF`uU8Bz(iJ zKiC}`y5IK0RNeM4!Eq%Abyv1ev#stl=lGD4>-$}m%xM@}r{<{RUPJW>{?^9(3lE zc4HBY-|?9{Sz{Jg`%lnp{KYjB2SW-Mzhq07)sPQ9NxFKYSUUTB&#NmwRmm44R#5z% zhv6+Hqxb=ybqRNv{cC2lfh=|N8YgCpU5|mCXjF74a+@d|CSx2ubn4)&0Bi*jSk~=- z&=!JcCgChRP3lol?P2HVXI_Vik4LxnWG=SJ9XwEz7EW1Ei@frQb?}FNMXca<`4dal ze3EkQmuC0t*SDOxBCd3yaeKqZWxF$OLz)O{_&m6HZ!xU9a`Dlz28c-1=gB0WJjZVx zR*dv#sSM}T%Td1aYMa%cU1O%&1@74`CND(z=`&jVl7G(A<_u|cP9wUJ?)@nU(oxpW zJbYq|{(vLX<^38FOWG5fDIwlyZ>UgSRA75a-1(T()x4DU;Y7`Pb-d?AOfePADrk=| ztP6TERU!}PD}1uJsGsNC{60~Q+A9#c?j?_B+sr=lEax}j5-^HqquHMBWBGJ1)9X9g zhIwwtb-PZHpJN5PWLJ-}@#PfKDd`=SyR2&> zG3vZBoC~fOTtxpt3yxW*CmU+QOYF~~w?QidEpd8-I(t`NT=$GH7EM)iVZUKx-jMam zQTmJVIKn^6%4_;T-lN{f%!d^_3oc98GZB4YXNn4o7t;%Lq%Lx$dE2SsB^Rn3%1LaO z6YuS38hvwLTW%WP|QR`GvbF1m@uF^2(TKyILs|EFau;x3JWtgOW!ja%=l?P@_oz}8;70$`X>Hf zIZHcP`bh)98i0#>n?uo!un;RTjfURpl#`nBZs6o~Pk+Qv?7)CQtY-fo=@TjqWOcT} zmCvEx*EHu>aB=>#-*~#a1Bv#Iub5v@uTfkhJ0$GAzHSIUi#Uuyt7Ye;hD1CTi;~KZUejC z*AusvjsCEU9yyWmh)7Wo2uFOWh-|wqc4x)R7u6i*a)n?HsK3TQ(*aXb@Sn@#H|3_? zyJs<2?^)Wan`-*qwWpHiWpCSqz4ur-41f=5CKZ9&g0*4uMnNef7HVqM!*?ifnm#(Y z`%2i?RMSbp(;-KWkk?z{N(sUs{Nm z#Ks&R%i5;J1EdP|ij}c2?Cz&E<5$;Js{xmb)xEq25Xgl7pNH^Y>GJ>C2E_~Ln05?b z=_~u6znIM9JMA>2Pe8zE`*xoJs}pxe%!~cbo-(a)+K2FJhE8#neYv1GdO|ci^eKlO z2CN3fuimXX`aF#2a#m{Fz|%Ir&EXwazbA4b>-A4!K#A#B@>CfQzY|q7$9j&+(EELw zR>41lVy3eiX{8;nipmzGr&+GKsPWGIS9Q}7b9F?@HKf3c9XVhO!pu~Ud(@dmThia-BLYJ&H5jbF{Ivf%%!IO1hqkVxW%NJh8XZaPjD<E zRBr~+au8(;Of*|+)FK+UEwti08}n>M=j?EqgxPshTzg=EYF!o)n^bl~E4yRsJRhWl z%~MeQ6%(fmGw`-QlGY?-^!qmdZSb>az%-7w2sEQl5blWWOWCh+(IjsDQFFN z!oxQji5?UGb@r?&ay@~C;wxnHZrqAc2k4b}gMyLDyEbP357El7Fb|#OW5mxjGT?Py zhiVOa5O$gwxMA)Xzt%A}PaG5-u=+G_7;Y_}X2CynHL4~4PZ5Rn&Eh_s08g>2`UGE*$HA_5{^LZTqOL_|PokuJ^9ks5mFRk}zLn)C!A1X6x;eKT+7&08~nBm~wX z`EtKo_Bm&ty~i?KIpIOm);C)zo^a=X=sJiGjDZ|Wzz(193_G^wZ4~}vrpV5I#RF5s z@7cw{x`927jjTZ+1*&&Nw`n&{?LPmHO?mAVH}Q^M{XwD-Wx!tdofdSzLS z&x~8zyUHXlfqoyk)6mBl(@?*aLWsa|dWzh6d+h>q5_z}j zb?sNx5lc3bsD}{pOI*;lp(0CxapMb5rM&mB*B_fK$uohuRu;<=?fgCevE5nSFK_rk z=ZzlER8Q%Q!8}Ejzhv=ZNzyE?EoQD4E*Mi;l|i;?i$AKm8S&8BnO(VwHC#HYP=GO2 z(MVU@up6}4<%q|iI)m}1y%zJo%=kH7K2!Gts25Rvg*Gecdw`;h5BJe_=?~ zWq`xzmG0oZGn}P1PHcMqy9`48<=Waaspo?Z0*whsdG$I6d4QTjyPAn`rgGusf9M&?P?FuS&v%KK z1;lR~MNP3gc)6^1B5}ueXK50%01XnW*F*_`c+W8x^kRHUJx8Z1E8E7|OmbM^cSq=- zCx(w>ZkA>X_Hfs%7zg+#8eVVYwP^0Zrv>*q-Nu#|m+kS=<8+^Z;(`eK zjTn$$TJTJ%7Y82x{3D5Jdx3N_`W7p^t~yYj31ToOciMSCb~=NmBFMY?7g3l7TMx}e`UZrvD2Gj06`2e&JGZ9 zEp(+&b$h81Z{57>A9V9q-n_A0IjAc9KszVs_VsA@M0KAP%XF8eAG7CrmAjLh2NIOx z{yf!WsGX>#z=k}Vaa>4I#FbK3FD>xv@)Q^LU+LvZr70-AnAth@jpk%4czep}omO0Y zCN77*_vQe6fdYu+CvJd|x=zcLg!3Stj$cfmJB#VZRlJcMAyva-JvZTp{eU=@K?fx2 zOyz7ljL!us^(7?$S{UfzH{IZQwb2-%4{0g7GfUTd+5lJ-5`SKJ#2f{*8tMIP1_07Q zZ;`El2HV*v!LTJKCV22GxGG|uspyu&aXGLcizXNkKr%!iJdb*7P8dPB`;u(J`#H6s z@o)B^K}O+U-?O;ki-Q`EWVN~fb|^ExH8GW!nI~a#yO&sJ9&2FLnF=g}G>aJdz;Mj3 z@qh8yD%ExPW-t-bGRw1Cq%z$@=(MRB2lg#;xKK6r<%+ZFA_F=S~TMArkt zAzIN%$)Cb?k~PY&2lj$5`wW*TC;zA9x0oX+5-rRyP1UPUla{DO$bo-fC>YQW3t*>u zOIj%>Vca%GpbOh8P`zW;onBV2(hu8QM;xOFQ)iiFNJa1&cYr(8jGN#a(AGOs57FiL zeE!5*0Y@dGS}*X3@cCvl7O!Lj&!VZQwM`e=*I&|E@`}?!N(~jpDtO+>vY*($caJch z4_Sz;{`I;R{Y>ht7an(8=WRh{Etk>aSu>NH{-O@ojMNXlo7GXit(0Ulc_#3quz_jM zp2aAN3-%Z_4V&Xn1R_q*vox%AD7;RmD&EKSKQ_VMCsGVci$A8Vuzx+W$G=d{^dtk( zhpI+_>iF!bj(n$6`1*%i1Fvw=PwG=m0^Vh0uj(QCyR97k8JA$E|KQzCMw#{u6)+lR zu;*!T2gD;lxH_JJ;|nF&cszKWjfR0URf1>Ofu2v9U#ROGUZviKi}DNP_9z3 z%p}$ccX$+zr(TWd6Tbd*RL#c7ppzx~v9rjuL=bxf7pXb*{im$dF0LE@hN!Iyi@M8F zYZO8?=ff*6lsLqbHroa`X@T%iAzS`8sgVt|gqIf3X38c!1HuI({KQS#=vih14l^j4 zG2*r?j3Wn(A{rvQMCU@N7978E`YgbX2|y7Nz%57g_%~i?FP_l@6I3-oczR}HJ6RJt zu-7gBw})iG5H+Wte}4TIk{1D`n1<(=){b`!7>5iLl=XNK0zNlWi^Lq-XbtBC1IHMB ziR^M&4QLz`@T;2-fOp4mA!_igsCD``+(KjNH~SfabJ!w>t()SYrx=RljUevh-Y$j3W`NIK z1$zWO=k44SPKO1E(#;@UOa|*PyzwcYy-qc`uOjX7&~}}S$9AG+ZYrwDkr%b19}xK; zn}1q^BdVK}7yyFpbGwk`n}s}rwy2%LcBnLKc(y#R+4AWR8g359O|yG+uUE}je`=_t z0crL09`dcl9-Lvstj4`WM?bmMD8qO_u?N{~*NvTk>T1X>?EzT>dV&1HuvLnu$Px}U zFxDU<{T?s}lSYfTuDqZ`!dr8Zg{bD|s2A?R;QAr!dn9xe?w|pN__Mlv@f;=7)Yo|56uU-cmpuTkYnAevP*euVXN({Wuqe)Kl+bT4d%QB$CYfc@D5KsrHquu%@L2LCy7XK!uE^ zHnnSfoyO$qz&M>4TP}Zl%XRYKXD^RbHJBE5IKHUVyYdo=Vzthajj@xhR37riY?u&= znisL*z%%O?xyQSiJGU4M%{*$jeAe$z?)CX+xNABo%d>H*zB@bEXpnz`OpjAtL0SNC zV2^Qkq&-q#w%wr$DWpsNbXV?p683uinx9HnmAKzby8PZzW-g9UhpNJLV1$Hc-zR^g zc@wAoeXT8>qeRL}?r$~p^_C`nqWqO~5eck{|NHQ>Z5DqlVr!zm>ahWr*e%RV-%L)z zPEjI1C3~!U8fsWbm*$RU-Cjxi$EZ<`r_fxC29*KW@ zOl6-b_egs5pL!%WASC(Y8voMXHQVD10Ca;x5FEvbN7XcbvPIKFlsm;ef}!W{m}8^# z+b2PR_=x*%#(a3q9J?ez(Zyb@&FDnbeEhCCYu?1!(7e@C0_-or)-VSvHSSZiRw{`S zlB=-B1G<*RKD=`Lbg}yHcT^1|{&*SrMrX~QF$y3t^8MMVv1zIdGo0Z`nVNEpvI^>1 z6B0^w+%cN{Wt{*g@m%)riu3<_y*G7quiA}0AC_l~-ii@3zF|3%;4MqXgCfc?ufeOO!~h*Gj^blTeS@bvJ1y_*#4n5_TI*YjgL zS_=Kz6c(eaC-O??4N6gdUqx2RwFuq2RCtR8^x!8jlP0E-xGKI$u$U5Qozp=iLJ+=g=)CdSJ~TykLjZp+^|oO4on-q~E`c3B(3$mF z=k5O%TmOr-*@1Nb|0})@Nqa-Z{*?ifYK;L%O(A_?>E56vdAlfmaUyoD*Xz8{T4VJE zX?{N&s966Ojj-6a_UmaEG6>R^&8H2to%UB3(#syNRzw8}`}M}oUMnz0eK2hU@+mKX zH7ZofFN~BF9_wCjm~a20TAGyt9M(HMSF8r+;6;AM!HaD_u~X`-#yJwLzxF|vX$|u` zIOuSP2f_eo#zdeXO6RAQA}T>|$`PGALJ!28Wm#PT!+BNrr*MVjQI-{ubcYd&an;pp z?PLQJCI;rTDTF7YIF~m)I-NUG|?Y78jYt65N(~@rcoz#@a`Q6+{eJ=^M{kG zS7a%e+g_l9gn1R=32qCxH9_VycCRb z^M~{p5bNOlejj4bPAq?D3!~K6RzHFfu7LYIwHFqcH3BwGgFOiI1yzP-mDKWU%Y6~o zNz8Bmkw?DCn$UI!C$A$yy-X2^l5xJUMn4TSvJCA(dxuV~Zq3lf@z8(Nm?E5utHB$F z768aP;*eVRseR_Zli70$0J*HhCfU=Iuho~ zID*r|5lUcXV(&nlb!>?Qa%vJ}W=_aJa9(g*;zO;#vTJCnc-JdF%>-STW8#y7Ba4r+ zu%<_7?9tg)u7PR2~)8gqh-r=2wxKcFkHnunTTROC0`U^bmU`--!tId0qVW z;cz8!8IOziWILN29Ut!jFQGsa3HA7Vk(ZDf1zgX(|IbUx4;I2c?#8hmiV0CJ)DdP+ zP59wjc9k6Roh_8@rus2&WC`F6lrr=MA=l)Y@*p{nowCsfi}sn)QO29$Dg(vJXMEHd zQ@&j?=ECB++2;wC2I;q5)fi;t42tX?7OgfLp@9KA-IKyTEd^=w&wUzUmB*~7*G$%b zFh4#jiaCN@b1%UUS$%qotA>&9KqjW!FeL+P#g1h6j@oSN+-mOtkAf90mmkyZYMpvp z=M>q`-i$c#7)4?A!V&5;#f+;kZmB7@wWlx;juf9U#yJo z7z-jnLz^c3h@3=?JfT`tUYroMR|O@^CIwWE2IH&-N&OwUkowQGclNYOHjEtef{&+= zEVj6Rq`W#B{V?%?apU{(y`u5FX0@k(40RIm{5-mH5nr=AAz`2zlY{kQrhwfG9s!6l z799!@LQ;$TnSjC<12KJZycH){Wh7ttj)y+(vRF1z-|{W8BM0sz=6|^< z#{%h*U_1u^tk>N+0K7d18oG6asXEonxO&sf+`>_IP?!$cUtf@ zitQ;#*$aztv){TGSD^69=_2Kr_d8pN%m)2ya;W#{aO5G1bSt;mDvq<>nPajNb@Bzj zmgBY#qAtbIS}^{Ps9>PxT(reL2f_jDa-VR7R6sVh@*rVDgwpfoluoN2ntJZBJnnPn zg8Wu-d;3qrN>ilZ7D<*I*q z;l#t!ku)r~(-)lc&g#~74LGM-^bA&J-?duol!EMO5#lNnQ*TItkW6E=E!-UgC>(lW zrdld?a;jBhDtj#}q*0r+4qcJgx9U68zhCCI4=L{T|Eao_kIoh$| zP)W*7_hw@o^ZEu`RGs#B*lqG}ZunmoBn0_5gbaXS&kF7}vFfkTrY&cC_^{vAx8gWyWC%x{$#0sAPXtQnt)XLc9<$7uBxn!wwjg zpz?yr0nQ{+?yHRGWV`E;GMPPJ$5IZTz4z36O*6C8wy>l@i!wbz#(sl6MNbOP@vmo4 z-i#3Wn^#TMoqf6j0@WXmwT$~}@So9jj(2it;r?}L;3 zmSXnP@sRcMSc8z&=T{t0b^qR;l@Y*hpJakK$3ZC=SF+n8j0%Arb4B^h8|J^bFT?*P zr2m&{IoPvf4R#&u**1`#;g|nBcIbjcZRtc;%X?!k)9#=qTB$-bY0%y(06E$yGFd#I z)GciN^Si@+Xx?STG5=b$%)u7TM$)-IhE6a%n!$$wCfpo!N!Cze7a3w*6{~dS=I%ID zAUrYNz_lr+PWM68=Z}x0iAm0EFQ?JV?5yL(;GnZ7qt{@>2%tHn91!S03N1YHM`8N( zjhuGzt8Q*hD@%}?_jCTO>cC+x6!5)(=`5Bc)lq{T2q3}F`h6t($NJjZ+0B?tn70J= z_f8y*)L~WI}0w@sP6!8~-^}cyi_Mn&K6ez4+h1yGI7( z#lHAT{eZ}@3yA8-Q;k^109-s|M*}8>eo4kaBX!BRmh9Uao_68GjJz(YK>sMzJXYa}P=cU}vp$A?M%-I4Cz&TlXPIax>%}IDy>Vx#nrSMr?3nss2nHOnVT`j7YbLhwvNMis4A8HArkc_z zzsC8A)kejNy>->PlgJ6`#|wu-&T$T#yqAl3@53x?5kA5`x`IrM;WO@iNRP z{iPIXyf60K>Ko?#`0nn#z-2c#NtIhmqSYdE)dz8zaM~3x;W<&yQB!h|NonHfW(r{E z#+Ur$bdvS|zAmd_T7PjU`oze~T(_(vxa3+Uq!kgo^4Zkg%=-TH&`SjN%p!6ME2$oC zv4RJ*W2!~T^%#eRIoG1cF%?e+3pMyjKolhT0yE2>QW>cznK1V{E+lWdkuX7~N z*70q>B`wQ~>@KAAsQoQX8tTA$=`poT7$S&Q_<_g0;URdh!K(8={f;C5QyT7n{rP`8 zI{eStFM|dqVIDxSWA)$#l3_+3Z(;(mtTv>~__ou~%L=ACU-F`E!L7vmDsr*fKc4v? z(>%_v#^Cc4W*r3)%3&MIVplQr>@VjMoRzoE4U>xLomiWkPpo6`<-CQ|Igtlh=RDvR ztqcdaLYADLUva>HY`4he+L4KJnF;#aYbcC+5Q+dS;bnXy)Qw0DF3rEiqL~XnaJA{? z`wErW;V8kwfoKAnrrO$`bXA%bPbnB4#9CI&@O%gD2jzCH=#C56b@j-NVv?e>Qs@h4 zgy21*?OE3iOivz23KK?Q6ZqSp3UW%@m&9})FPLfw`DDEFu=2+%*o+6EK}=PtGqNiwsCN<>|j!-+cWPH$aZo}aK>KZHbq z5jX}1i~kSJ=M*T@=e<9H-<7N%CNzEs2&=QQgdoPT2va5+<}?rqVP_c8VfHkOJ0pmz zq%#9>?~+)&>SH7DSep({_<1R%``jAr)TUy}piop)#rsEDtBu?$h|_1*X{Zrg*V z-h1C4*(2eaPY*{!=EQ0T4)`s#1L-}6OEyEFhC(SO6#voiu%DP|ntmEe;^01$N6*p4 z69$)0WqE`PGjJ zh*@`jb*n9x>7;JuZStwGoj!LL^V5lOdX9CR9rU5mWYBBPc?gS=t(Oeb-1QNLQ7(uG z&^FsM3eGFj?EtDUN<6NHTesb@VhV+vQQ*TpK&lm zD-N+;35c>SZ;rJxY7ce4s6-M`inY9Rv#j?`g*}FdBAX6+TA!@cB$4UZP9`VKkTOfa z<1yUQk$_~7cZZ@D_09dVNlomNFYD)JXN6C$tQ1{+XF{u?j1ZQZOgX#exTAkdjnu2< z>F3l>?+u@)u>=Owwlp%&C+dWtl45c*0`qL8X5J6$s`^j;X}1hKsvXE-ZL@xz{@{m3 za`u3yytlsz6?V^Gnle|49hqy|-pCoq%h-aiip9KMLHIRvAy}z=@qX=4l*pNQOd8?n1&)KKwT}YMmJ86^$*YZ8mg9^ z;(dUWNPqkKA4>taHtvGMm8di!LHHA8eriPwMmglH*pUCN^ZssP{{%JYOtX&TBU|*& zbBrtq-vnWV#6ea!^i&iOJ_UKxwm_d*rqGmm*HF=-;1+(@EeF2SdnLy76AbOUY2p5N z8J^~GA#fdDlD!QrlRiPs}bd8c9Z~{a>})$T<_(_ncE*aModVWnVjH@5qjmET;sF`Jnbv{uee=fhjFk_S4WkLe9Fe#BPaaSI+$0~(+a#yE z%m7@9=9e&6cnl1JbG{1ZsJ-;?CPS9!&OTG2E@V8lckfqsJ?i-#EpT%JQ-iwWMDlw| zVUJMF3MfL2R#Gc54>>Sf<;_UiKRWH6OH04g<>|zb=Vcvmezd;C{aYW(+HSo7S?PN| z6ykTau_psZZGgF~6+i%tX}$~GJfK&Y@!2?lP_*%$BoI8SR$qG56_6R+b^2XzVpavM z`=n%t_83F7neb*CWUGMowT-Jo>(o+Md>dbq`gpSO&x2k+3=9?Ks~vv{z+{HXeEW~j z#mB*y0Dd_vMS-kkk)ix7=nKRVYH+z26ie_SQ&j{_4GLhpydD<+~Pn z(`EaYTH0t)noyMc3O*|Jk%Y(d_6l z2vz2%ieo5;y}6$ZSfRe+ceN)R(@IJlJ@WQmvf3}z1#Y!jPJNZZR%(0Z`lZ;A9~Ni( z@!zvMV_93CC@iX*VG~lBHW;3Jsq{GM!nvxC2d1M*xPdIzF_3LIEKZA{A3&J=9W*MW zov})QRc&AUh4Nj0Skscz>oStriIYH`kly`E4xZy^#h~FQy|7X6V>`Dh(@t2vH+J#l z_Fz2kz9#oa*Fyh5+`ETtLGoMLbos*2;&rDY`29_l)R%H23)YaObMyzS{@IG%R_>{= zhhTFV?x$N)77I}g7vL%z#&#RCJf<&cvzJoIpM|Rt(HRM!P&N>7%`7#~7HHYi+ZrzoHHy>1ZkE-L zYeS{&6!)K-eOG`EXFNOAsej+^Zl0Tp7JU!CZa}|}5Cc*Tr{RdFw4QHKMklU0R9B;2 z?CfoMK=Vk}93mzb=OuHoP41(5JB7>ESd;iozmEkCRaNOUi!ap3*vkOJ*UC&rIRe&k z+0JGMJo-%zE=1c57}5Q){$@O4*E{)igwy5kFUjJxNR`K~!%mMrXplc%Ay`b~s-4z} zlwc~ERt8`I@|+P>;e3zR8^xFt@ve{EX4sY+tqx+p$O?ZZUwX%*j(Y^5N$lV>vX8K){x|FPQ#H|*Z7}U{;-L=;&896XL;IBM|GeQvxaAZ9X`ei>8B{aOO!Z#WF-fQ*N zh;2z)U*Am1%?x{n$U$=F>%W}Hg$TzsZM}a94hE|E;!|cJ@{;tr6(!GPhCJ7_d;8_r zG?;JUi^27$;rYbk;zo24V+ZU23WK*2_i}@`g1%(J5$cV**-oa3G41 zd>PpZW3!=#(W>(TF(m0W8!wccSKHLHSPx0QY@Qm)RFeOAG;;ZPX!Q6fglq+v>fQ{Nw@#lgu&JZ%A+qtZRWzl zTPObvF1EH#xrjuGs;fiqrFtxO#Y^_W7P3Cz&VPfTU;VbTxZ0bpbgH4fka<4vyK`jT z+ZMV|Km$4$>yGsn?($Z+?v%c`FlCtYVIyuP)~H;}NX!#spLe0a%v#7KK7X|~rz#@T zrpwpgUryKkpK;C8xS;drV}VKBUAEPB)D3!RNMZ!!OX19})56R_W$uX@IRe6{%1(=zyIv{ zsk&-x-4Q5n4(vHxK0RNE)H{h5=2X#2KPn*}``%&?Tx7EojF7Sqx&daX*|k;Vv~o-s zQpMR@iR;(vhljboY*Tf7tuM2EQK;pMX{0>DE$4@Js)7D=##v#i2rYn)LiX;@HYtg^ zR@WpTN3&k&u|qNWH?wuMcICSaEebn&zs4Tj|IJOh;y97+MArF_4J*;wzf~EL)!isn z$}+na9L-~7kEIL{lG?D7J*3%i=uw9K$gB~CKlFQY|JISELk=zysGeKd53K;%@!Vm< zLl&PbwwbaWUZb~M_x&LsZ)=x%Bu_B8!RhP>zRfj8AQ<=+W=;`Q$q!)|NStO?K_!Oo z65U^(IOr4nZmGGa2`}(6e^0a-=xkx$!}yS53?G_9CIe10&(|yR^KE+=_uF!&dDmW6 zr?xP^yY>=i(eaW)JaFJPIylZag=HO|Q(7~o@SvPfvAaMP%`f-Qr5_(Cnee_4e!yfh z873<}c%*o!kn#9GHZ=Am*svN7(c`-yt|7~u#{=NA785arJ?)ul~Bp_bFAVaQYO1lUIER}!;nkgn? zfFex{Z0N`$$G0UNa;{eR0>AwOl|qLu^qa78tZg;=X0BiHdg8}`&MCR@zHNsEto_Fp zH`G=m97WysaU!N$Y%%5c>%dTpf}xgg5^`Fvrt3L3gy7$Ar)19w?0L&(J;n>r&zIn> zu62cvFG(|aOhKW*iVkND`f81Lh?;cRL7Jm@!TgcHo@eIWk4u@K;So={6&ZX*L|Nu@ z>ql7qgaWB5M7-*5Tt|la-NOeA*q@8IvYmW;V4CeoV{l}7HH2zyv!nT1yTMqiY{IjO zUj_2XbAaTzrr96B;sxOVJMIPdTX@BXF&xo=mf6$hPF$T-W$G%}mfw(ctvH>1aHtXH zD;IobZr&l10n)TV`z#PgiZ&iXh>kR#M92=iSFFo-bV+JqBj#*xh)AESpk?2vy|QSo zAfXALd6lxof^VIn!;phGe%Leah$GOb%|ve|6Ii!2;(6hLvnsyNo{C{mf5lVS zHRlj_*gI}%_HqZRdbT6J7=6`*Yytw2ug)g=4^>``-+%YPP5OWl z=ZDBb0m*|l6U1=RvU$QWtcMu#6?rR6+dePK?*?@ZSOcG+YplY{P+^Gs9*`jK;Y^q7 zzJ{Cz7oIE36ElMlXf3I$mQL?#uK`iycfsixQ*a5QS05eFQ#`ub9@ws}Esj0xuvG9M z#ZWyEWUme}n4i?()0e3QWPGy>Pe;PV*_UH^CfYG7)7-CLLsvDYIi1RDEq0{nwy5St zNI9y#vW5a%8%2Oj`cB8;kS|=`*|+i4pXU3H1e#6qSLj#bK+i|%07I60vJ5pP-(K|_ z9^NSNGI89mwK=QBW^F|WIym{IwAoaK%1ck1GZmpJfo$n_ZAS{n!}7uHs2c+5)cvQ` z)lq%kkvdzQ=KP*z-?~IT$p1xbLJM~)N;dT}p;*Q4SU;Q!xCuetKn}gM_!pKE z-!h^Yja|SOJMYyFFLbp8Ybk9+ti@k^E!G3)aeyq14C6*!L!(G`!Y)VExS&+A_ZLh0 zPGIw7I(wFz=dipqL3$vnnWK>d;Wi8uwUFI|Hh%may4LP%G<-tV?2o-FfAJ?nT>@Mu z&$AdP<;N=vzu~EH$*mo4Fd+I7@|rTK&ibX3vyj%Z z{!und(@`RWDj^Q?(}6bm%J^lAQ8K&QDptUG-6ZE#c0I{xuPMx5-whTxX})juXu z_En&k$KKt8W(K?&InG+Sb@_T(&wYk}&-6D-1RYYjwf06XGsuv7Fv3cHTg04t`46_7 z0S<#h%gj24S#FGtK2495`z~}r#X0FzqF6y$rZPkqt_kuF#uOt0wE>uZRkvffR*-@} zOMh6eB*t6cw#dDHr{LDML}ZIV^1We|J41uXzb`<77@{Opc%#CIHU}XiXlZXbEKG3r=NST+pz(*q)iq5#Oo^%*96qu zLNk3Re@p-5oLnmRys+`lN-klu>d}fX^Of}#agL$S(>~ssk2-8+`C8EAl0FwI1J5|! zN?j@eDUfG2N@rIuOEWGYg(vg;B4<3}U**&5Ds?+!mk&ISmU%^q4qsiQ)$xSq=-rNu z>fG77{lHN}u-J5AlXU`wrH5F-R1?}NqyrLa{i=e@9h>8SE_CMLjl7i`OX-7hZ%gs# z7cI}SvHgxb*ujMqn@2+OKOuUCj(DZVOZC)A%>lZnbtY619JjEyz|0L5)u z1{rrh*mBVOn_hd)PY#)tH{@%Mj^c$`Y@JPoZ2IzZoBt9$hsm2%77b*Ljv9n72swps zqgCc~M?dkYC!H)16h*}z^Z6DR7_4+2hXzJ6IL29e$7~544R|a<6kUu3&9%4}H}&R4 z+Q9QA_VvWKWJxw|WlgIGY_FQFPz{(c+$mt$18>%HYI6@0`Xtm5R@79j?wiXy8#PMngbr!vz~PU!cuWlJp$8ieuY7-iK4 zVG|R!y_}j4=-;Lu(qd1CU3*|9cMpb~SoafTdu>2Neu3Pvm|I$ZM}LkmrzKG`LDT~T z(|#!yQcNwoU96`VV^IDIg0c4S+iqz+iRz6A^SdLwv3ZbWiw=g#5d8Ju3!$geg=dT` zs`nrUh&!}_Ii)G#x%aarxV4N|c{aD+r>j#g#W+=v8&j0J@4C&1Or@@H`V6XHKlL=h zr=uJ8n8k$XmgG)J^0te0t|5Ev zPuDPG0ZG8v$f?Z^IWE+n@p4u@;;{;|?eVU=q6F;1mB`DBh@7k!G@-_EwRQuJ_x3BS znlIZt9xZ0xs|fg5*n$N^TcSLN8}DFDbjW+5!lP7voVzWIkbIMgVPVAZp{0uX&(_%buLEtdU8Hp9bNi- z@{JF`K_KlO>32z#+dKw1HpBjV&-lS1Ve>STmA9LQL4XDP_XdQYx{A-rz(TAb9>^Zo2hZ70@(df z1fU=hP(i4Zs_^$1&Wb-3Y3!|I9+nf}8q>g9(tQLGGS*|RZnmXR(7uS9~1oCZ& zmd?7hugKS}lig6MmpQS0=aQ(LwG6iTBJv|h#QZ;knE$KeEqD$4?|*Tk`QKxSf4@5q zys);Gwd7v1V4e6^Ji~xm9?^-RYSA>KsnHHOGIdFvj2$RiF5U2{Y?}XdBQXHEo){in zqmD{18tikI*L@YLaicOJ^r%GXz?~VfU^x7dh?p=jl_XXKSPKgXA8Hl4%*;AHv->DXm61J2s8M z=ZIh$XkO46+VZ=RxT#3_UDZwOFC@xUB#sH9&t~u$go-$XM!X?26z8I2dEwnT8~5Wb zXC^tCwenoP9kYwMoV3})IO|Kx|!#f3uSs&u&j^&79bvyT1`K3Zgp4upD%S+Rq-nQ@V!Oo|#O z8vKgwVCnzK+Gu_?!_i3-cuX4?A=_q2qnq7F71N*7zl(vS)4DC=8`xbFz!j^(o9?s2*3kJ5n07ss_*a z_?*A+Q_KdQf8T9kSa50_|d&;^?sh1eu``8k1G)G zY~e@IR9}-(iW_$h(4i(@Q<_*}oef(X)mY~x^_W4z8zm1V|DUMV(ne;RZuHCt|6r@` zoxMKl^F^p&*BjBAhrFZTl5XrK84P>%u^Uzndf9kYmX(HHb@7v6lk~WGFjEHNZT^6b zgPqL_J5OhwRAQF0ri-4#OX0zaIN!$$F39s+16H<>5}PZBAPEA!-LIj~KL<{+Dw>uc zIVgD;`7V|wKHK3A!qBtzdi3ka0hS874MZgFIX{(D8@$^qR<=_2e8$bby^A|ADe^j7 zm*@JXChy)j*Jy@_fsSwKPECbe0I$$>IRgh9z^i6(7>F^b0eo{JEZJn$V{4s;EMXpRn{i5s+C|xIcM*n7UA`674CSf(Ver$|z>MEK&!nOgm+j2Fh z&#DjR(F2FPT%}vzDCC7z+>)p)EeTO5&M@IQA8}njQsgZ^#{{l^+gP=a zF9IN)pzSV=tkeoTTN41vp#}{SE#2#lU3_z21RR;YvQ0c;bGx-}@4l)Yu14sjJ{!xe z{s8ga99ryo+6j@7st?fBE&|k}{q8j~T+Dn*Si_BVFMzi76Z! zsP(-$1--(!`~&}c&~0S7Y~EvgX{uffLK503J^p^()%3n{cYaHfw&~)VcyVuWFXo$j zz9#Vql*~;RkP|Jz1qpPwcuFFOZ$z#*^BViZhe`EwkX?r9- z`NrVw37lH&`TIgMg{~M%D?0RN^3I%cYy-}eO#K?vpfA5$M;K=a&(iNA>}k%#fyk;x z6>@QG8p%#^z`QcLa`ny!`|I%?Wd~k-)wF^>D$?HwM}xs0;A3zgDt5*d~zq^rX|r8bD!4<^L;m5yLYI=^PErpv!<= z9g2H_o7Oj|p>Y#^8;6U(Lwh_6?X+wxb;Zqn^b>C+J}N4@C%(m|3kZ6RxEw4^eU}<3 zCP3>Uf$cTkDuu66rpyj*qTewT@OaDQ8>~U0ZnJZx#p&Ls(1e zmo`>Md8*tl;pDG?Hvyc-$^Yj3Q<^f}lA+W90#d92`oV~sd40Y*cIx6{2Va1d$LD$7 zp)ih&dpV!}V`H0T`?ptBNjKr5wVc{@=QdBdkXv7Wd?DnM|LvFE)U0mZZmE&T)~%d0 zqbGOVqSJc%P+4Lzg_g8n{fhC54kg!sL!K?Bkh-`VfO6J!Yu~HLQ~P`4L63ypsdfUT zf^x}Og3*g|F`cXfW%@GHbuJQWlHQ@PWc<}OA&8)OdU15re##rtx_Gm@U3X7dmLV-h z1ejowS*12G;XwU7%(`&dP$RUGs}>c0aRj5@p}&{&U{jUzP^kllc<%K&ixnJ0q?8~k zZAjZAuaa~M3T9TVMf(yuVA0OI2eKSu^Z50UksL2rFABF)4<*VUZOHt!ckzzhan|My zra9v(iHp0v5mE{gPHtI^Bi=ISYb$HshsIhyyGyCPL@1O!1XMq~Q3M>OM1!HA6gtG3 zgmV`C#nNtbOWQ|EEv_0p?EIm*kX_-BsDlWa6u()ygd@A>t&_D4iOaCZvk|*raKm#S z|F)Fhp1f@RdYA87s#DkEa|_H2zdfH6c6nvLrN_K9<@QgKQHo1=vh3pzM+_xCoH9$w zlC}M=*i+CU+ztC5eK^Zg5f=`&P}b%^lvxLVROV|aC8 z{zpuZ*6oGJ65GRU$61i=S->(~zfT&NMq4mLSYpf4_s6{oOP5|a-tkd(##R!4x=HQ5 zzpHA3tzL9((~lfRB#?#&F8!D?m$12|w0dJG(aMHPraQeF4CjdH`ajewO3A}-3N6D* zIT)v>{1_l{E@*>S7!QAGtCJYxpdznLnoF?aQ-A1hdg>_~=Yl!R2}eiknIi6Y7Y zzr%S4bBca;_|#^2ZnQ$*c47Z0L?hl+P`dk_3CC~O4J&jQs@djWGYsegO+|5~TqDz` zU2tbBdwf`cPhTxntA4Gq;kd4mx5T`O&@CD8J~&HF;^3ZNkx|8Bo=hDZv)Wupz6r7JG34Ghp zg$!F-b>T0UMv3w7Tzxw`J&p^nFKG&x-@KbIBy&8npy1$J`FSbf?Jaui7#kBO*-G|z?z+13t(BW`#lCi1FDnc}z5z^1>@aJ}(S+ zt6D1zG{6X-F=SK7K~(b<+%q~X#QH!ay{$H}XO?HB`-jW)`j4p^K|b<9hj*srh49~N zF56yCzPAH;8MtWB3}rU@ZO3UPAt+G?PjCPK99nI_M_lUeC6)rc(Fh^6q2l9&y>|@xRxvhy=n+|1t`hF_r7=1{C!e zUn1_bm2O}jyoEZhUaeg8p^r1Fbnc`2VLL&?!?&76c;HPM48A4&!T$EQ6)g8y_@r zu7=JNtXJQbuqg5}{#t;?IQ9tI-DH;eLVjmdECnLSSKN!T#F6!tXiDDWEdpNHNdqeJ z=lls9lh0lPu( z!uuj|-T9u*zAmpm%jt57#A%7`J>(c1r>#2(;`i47kKVpK9Llh7drBpQ7W*~|scgx< z&9qq}Ns6qKBwI+xHfEHNbxNU#sgPuu?6Qr0&AyKqLq!?PSjI3*-_`LQ?{U1}`#$gU zyx%|1AC7~2T=(49bzj$S|DEU5+eQwYOa*ES7#QHh&Q_y^;J2ohKq0kAZ+|@dNZq5o zAC|Xjv|YZYhv`7tri*TVa(gw&w@iHQhk=9{QG1DpJ3E1InRl2G4#Jdm9IDrylKGKOzTD??23ZKyLwN zN_4t>6JC&>%m(%_TPW4Gy#9=bZ4r6q9_QLP-AKTD4@pbg@MGc#iqJj#I(YQ|Z4RW6pWy=O=yZ^ALp=#S)c-GA?NVfz+5BxlaFpslvKSoemsISq( zP0+(R)B2QqYa2yQ7e@L@FdJdfOAGC}j`*=`p)>qiQFO*+HAzi_LS}@h;j~B%=owPO zJabzw`~=RKzC_^!S;ijfF$M|Rf#!D#_G%Ri5nsj~@%bUoP1XIW^Zp((`Qq0^rfeK_ zb)BU^>%QS~O=4pc6Mz&nG38G+bLt>~YBlU6+=^8O_v}Bw6o{1!$Um+{80qSG{=tq^ ze8qTLS+g;wtmYwZBXLMQQ3cS|6i2>qOs%xOpCd#u+T9^w$@}ps=k(Q2yB`g6NZ70! zfMx$q52HP#^tV6{(Nl+839iIe;-PP^iq&zae6xLQwwq2=Im^9K= zs_QT9a&OO+zsdxT#$#a7Ho5DcWbmVg?Nk~Id=Fp*rwnf^*PVB+Uh}y}_>P-QhFs6q zjz3f91GzN) z4Gn;^D1uS|G=SJP_(3Fs{z!`uh(3(yM+Xp_rS8gma%@@d6NE56ed64WA6Q^>J^leU zF&44^6^%0embrU==f=XeoW7=1hS|&koN9_)DdxgTN1(~qkSyu#uPBETdgF6| zg~CI5coyAaHX$2u4s0XoJrPYsEw>&uK~*3LHqJa#Gik>7n2LK&-IQHF%~hln)8VxgBSiS73wkQw4mN5M4y3M8`g29FaS(V`eB2 z1Y{cAo&dNK#BdB5{ny95z>q^KQ};bksgrr{5RRPMkJuSs2YQ`T<3d#jV!w~y7Fi<- zjJi|y2QpW%RxD}#RVUn{vNxCA6Zjo{nxGtC5FBq?PWc(>vlX zqU9*S^XJH)wb?OYH^NIlykLm3yBmb*GWR|P>II|53oW3RHWi*p{is48&Tmy+VB5au zW?RE&BcLTjrgA?4wm?gyhT~q7h%w?)!}qt3j~$0rb5!l9JLlaCA78TwWsfkO=ZJj4 zoXzRjB&17_WOPclT~AnuzeOiSNYMG*2WkKEr)@eYv!-J_nwLvWZTLCyuXKZQk%Fp%TuE7dmZ4agnj=u_yVe{<* zj~tx)l2%!j&RTtyfc-Ko8Y}}{8@q`6&S}Pag7)t9;=D|U&tC!<7Q!^Xx5Dh5)4Ocp zf^ZiEA#XM)GHs5t9ST`;5*thps1UY`Drn3Kh)A)(a>|}uCNzwt@y;IkaYJJp{d|46 zm-^6$09Cgcz?Y&s%#CSotQ?T>nYuiUJ@EUP!`J7mUrK5}zO}-aa%;Pw^9Tb=1j0J` zr34^_NK<@?*+dL!c&j0XiUNCu?tCe$iw`t-g>9(?2r^ki#EO)sDS%5HcBji^c>k^o`414gx{3`hnsb1?k zws*Adz$&S>zT<`DTkCS|rASX|Uaf-7(uaNi-`w^Dg2KmPNj;RFQTb<)DwowfQ|Y`Y-OeB5IzD z(|&pT`g(au=B7U-Jv#sXE#%Zk4hY}w2uQJ&J^eXly_qdDtS>%i9%eDT>Fx0~G%&Gu zMAgBu=5eciGg<5!MD_`90@QR#{;~L!TV=JS?6t(g{oN@B4|MvBs~c$_d)4f!U$0+x zsE^zgm1jiF%V&b{|VupOUSU@)ir zgC8%pzls>>9bj?db5Xg^$FffyOcX{B(7(1HNwit!Ba*6G5hBQA^heputMss7!o#X} zj%{IX{?O~GLNj03YcE<9?i+I^DX^r$fm$W>G!ZLtc3M?)U=&h^Qe$94d%%js-5%4wqBZ5|q=30s zX&2JYeKXK3wu4D)iiVTcMVMF>q?4ch?}-z326*dGi^*H>`fZn3_reFko&3S9Mv-c) zh&sJMfDe1IvO$jiC4_(F-UHdnA)y=F${M_RtGA+)CJvG{%e)gF6Bjb6#&g(K0IzUR z!#55q6d0Bn)~frheU3(3uIoDP-`MYZW7qlj5l=sTiZ~S|(SQeXOyO9jev$#{*>eY$PafjiE!m z@=ydsMN$ezblRwl?ti|5pe^6)`eU%hd|0`+NKXU8-UORtALFW2kK(FuR zF9VwyH;c)>GQnf*MbM?Q$|%RZ zt!TNd*o{dIevZVgP<-?(uTp!Gj$3uP<+|D3y0SbKe)*qIQ2<~BM>BJucyr$$(ap&e z9B1>c46v0i<+BHgpA!=sQs!c{9DZjBVD1Qo#)tn*?$9<1i)3DXb3x_C8WHA3ej`4| z-xAefpbKAz@feVD0Q0IL5<}{X`$koXMjFrdqSl+UR6bV^Rha9#Rn?Z)#=cGGa?x|r zk-0A_YO&X3kBejhf5iP@1CZ<&}ssLi^jK1RJV{0}Wtz zvIr|qnoDyEi!*?b;(opOK$^qv;~7{>4+do@IeP+3)F$l?jSve}-24 zfso-04fZ#X)*<1Sg-H_??lRVzK~z9`#U+9+EU)H3x<#0-i@Wd@W(zVE(Mf_dSv*(ETJ=c&L|YPS>h5dBkM(X50?n(>F2d{>01>%vZeWLi5&*gQO1}lkqf?h?bozHtz2s_S% zj+2q*Z#~Uw2()I`boot({B1{mz66e=+a`N%<4@$q^;V;BvZEl{nv7qAH3!@eNzj>H zR%!hMkY3r1OZ|SYS)$@RhhLWV7lO%YJUotuO`(=*azW7tEA5uFm%@ zfjdBmZ% zY)=`lsg`6@!gRZS=MJS<_9>d1d$T%2sUSYv)BcU^6=eapRd>`cbW^u3nA)UXW6kR& z&iKR7ybR}n2_~|Jt+KRZ0>1hKnD_CYlw zI9L5Mvjm5WfDn*6#0Teng=^}k>$qGR2_#A|4-W*mOZ}$N3_SNsCk9=TsjCSEmE@Nc z&q9K56RUKY=+7Gc^y7KWwFCY?CY|ho<)Zn{e?K(BbOVclmI`7z&{O;*;SPkrG4uCG z)ygrI!<_ARM&-R!Z=3#sJXA{V6A4IGCwr}(63LC}==RMW+)6pUQY>f~A`TpR;-tmp zza8kVa2}?>BNyGmgFMrB-Knt^dJ+HE<&nA`0uTb+Q|Nzp_yP7*OjFk0Vf<|(#PQxp z-No3C!B1Q7JAXF%12L|Y4bbAtatUab7MW5)pM#eRcku5;^`iGOFHt+@1RAW3HPRUj-^CK#tniTjR{z(a1f!G1tp&Q zWhLfY#6sCz2N8=3D^3#&_$#lLmY3`7Lo_MzCOx%%#;jg5sPa{lTz%i-*3mu@;G8yiQ9VemHX^?+82fLi zc;=H>Ih7?0wK8ZRYPAZwK*6>6(zWgQ_1Ih+eHN<07;`j&^op+`qh`JW4c*CG9%OiV z*^1@0t%~-O1E*Nnt;Sa>znm`3ttLW68jjw~RO`r691HC6ZekWNmg1j$!v|J&lFz99 zLO4f|#9kJhO3vztjPIMjmL_{EBvarubb(1VLj;UsGR({^77=XTG1hxL$$>qxGY+a0 z3i?Z1=J;O!v_JU=@;3UF)XhfDM-9_?U6Ll26Wr`E?d1)23fqCW8>Q$)+|_V~Za=RT z)xQ%ItzqVEq`@iW_FVdILRQx?-UN7!32tMZVw?~FeVWBHghZ2U8-{b29$IhFk`tU~ zza*F)34M@r2=SVl#gkl9Ty73^em3?qm&|}JlmN+ZO1GI;ne1V6^UP=JtgZ(2MA6M|j*df9ZlyWHV{!p`3axGB2n?glZb-~+oCTj9N7jnagi%USY2 zBw>eljrMQOom?zx?(evy?JE!LuNP=Hi)9rq`Vrd4!M7}0hI*F7NTxCDBBNMw3PVaV zd~;;!6vmWoa_!QwXX+OV<7`Cqfopl%ShBjYQPPXpd~5Ni_WlCTa{WT1W5Jh=YfIQg zbz0r%cp0f_h|U7$9HvDZ_XE%723Fdh>w^^;N(kCp*0Un%VcgrpPF_wDjwR<#a|EPm ziXJ_dM$h6qwEnUsF&TwK%7JH#fZyYfLPM0;hx(mA&@oOWECAq9VV!E*_SWH;Id^I7 zPt5zT%iZ&#wYz)QZc>Ao0<~=JGAbwgV5X+;A*6B5T>we+aV?-g|BXVA1j;<|(}HO7 z*-X`}(eJL?Qu2G@UOFGo0Y5GxXl`BX28m1(FI)GmG(N{tSjJ7@0l*RC276{k1sLoI zS}5gPRqAOI00v@P%W6+?t;G?PO`q37P`?d^QhtfHp)LPFK3g!w`DgJw8ZoUFQ^PkA z2NA`S!YyL@7|De2=4q$mt`FL?IWW7(KM*BZQ+6Q?%G9HyDKDC6aQ+67#W5%7KtMIK zz+Mt2P*oMhc_-<#Kq%Jf5>DZ1-R0~{y1^JDGmqUHhV&c;R<3 z6W1r1SE${2t)+dfRbgAd?r)rPBE@^DUwSgWa98*G$p~~)32&YWZ(C)C~ z!8?kr)%`6F{aZyfbP+hMVD)Zrp!&&RWr{q3Rr%@qN> z91c|LFCHZ90b7fYo_0^VfIIFjZl*qL@-WA*J)*2uV(*PTFCzAaR*|_rK=od{Sx-2hwvR1aoOV&3M?gxtJwB&7y zbvNA?9?pe3bBN>@Y`19jDa?E6Zq5KGZ@5*{Esud1xzhmIzDIBO|5H4E;_W~BoBz*$ z;r9BUjQqdXF+SQK>&gRHBW z6Fp!9xOT3@$6Zoq1FxOMZN7Ba)fSBJ;E#ashkGDmY(S4!gKltWf9ni%#O5%;>G-V3 z0mbK^TiT8-d`~`89JBHq;ol+=^0a5!D6ih4%J;&8nvYaotW6S7>qn*NKr^sk+HCTG z4JZrbkS|IV$(szqqSN|J*RF zA_vTQ*{bTQhjBM^t`#T6cqyEOMMM-ThhJq?qINTk!;c&AvlAceDPaS zp#j$(8~X3>lyG><=dBg4w(v9t$RIVH{R8pBPhcFx^2opD&8DS#zaMt+xTZx=?};3E zeeVaa(46+nTU|_bGhV(QsT5@~a|^2SVv4=_mfk*`UD(^|8Z`(clZv>&xJL?+rb!Rs4%3^O+#od=FQXwVO9n5q}G$wq|tEyb2fFu zueL&z!~;iomPNnF6Y#mjM~Lzcp!ON&!L~>j3TL~RO}skmZSAjdE`?V6_q_*Uca+$z zafJp7Ki*|5gs6W}I6lAdiP7&Athi4jthi%aa)~W5UZ+slPa+E3{N_=UnEh&@vNrZz zx|T?HjW%CQ@j1vC%s_$VgLj8dEYKWT1t4GMOOS62g~yFr^(1-q1TwClLN6o^c~{3Q zNyK?=@33;Y7s6s|3O7V@=$@yo#H*jj6>wJR_>bv;OphIx$rs6v1 zQ}$jE0OnXIa2@@HybrIkpi2YdADtp}lRi>uwlk2&ZKOP`*IGwO=(<`fCS3iuo&L&I zWS=&UU;%2;PR8r?)NB6MW9eEyc29CimZ^wD2MabKuAXBmkg?HFiY0m~dzKyt--WcI z9yr~qCRsl^Pn`M)pnQ4ybq~T~5vs2!VMSVYM=F>YH{Ps|#;E{|6l{yAKM?O#`W2M) zO;j6^e`U)9B@#HIe|9iScEnMQx1{4nMQnU2i>TTtt&T*ijn>cM1(w*_RLytuD#FtZ z@A7|dotd6-`1)YoLN6`(eE7{bSX6ic4`}D#Ay+{@ZwSf_P*sCttZ^Do-Y3zsCu|;+ z8B6wW0CDr!_?tMejrp5nb>~f91LJsue`<-QGKBW!e{_kWXFr}x`F=8g%F}2? zJh1u!Ky{xi$>jVh%b4u1Kv|=kG|{tQf}gRC&Cny?`k)2h8CLP?D^-$Dvp-S6!rvuf1L(}hn7QKZ9;IG`5jLoCEoqn zK3be<=OUo=lyE*S&LW>1Z@OLF@PhTZK`q#y_LP-Z=SyvAwoVhW3yQ6<^<7?GX$~38 zsY~$_b>-cVwTTQyx1f$v?-mF}lDv2lO?Uh&6~au?JsfsUNh?}>7Ti%5tuWGOV3n-3 zu+`qQXDJrerM*Sv~%&++}?x8n6wfNWxAHL|Cd_k@?%J#;W z7e1tCk-5&OZaBphJJmw_$}m`M!a-(=W9}O>uT)%BXQ>;7mRe{dbc}O&WWsEAcA>uF zDaODau_5OuRj9-e`3eKgZ;yL8+Bo&)Ka_>84Z?C!$c#ml~ugt4}t(Fo%M( z*hI78!e?C?y(6&)RHc9St(^aa0lGK3ou0vcpJp2l#WT~kvBnpoT;-Ql4>K@7YH7e~ ziF8Emr9%k8Wwbu3B3oxlC`)M}EgE2mfvI*46 zxeaW2R|;v=NgGjjbFS_l>_Vm+KlUIYCW|Qh&;lCK*gi7azbs6f1&6$s-2zVNh)tFx zHRvg49MZ){lru*CSellZ^t|`T>w|>(n(jICxko9zO7a} z+6T5{Q8bpvLO-VA&@OGK!YjLzuIvr3ztnuN4&NmuAgop%M82Q7&q8nq%xTXIc-FJ* zVG#AQIA~ZD96DL2GQ0oyWY}ETBIbFE^L%Oi4=t;}z1N?P%w9TncrS_p7F7d1OE4F2 zG0$Tyj3P;Yq|L=rOkbiLPzmYusGu!v?XYN-|YI27Q zBp8ovz2nvd&4)j$`0MSET#0($}Xm>;SlaledXFj37h!~AWJWD(tz6^nFV*o z#5VJ)&3|}N#MBs(Z48l%XV_S94!V^g%AxIphG^`M7E?S9>b*xa$DuYVv8{EwlP^5g=>C41m4>_#1RmpbWo_2*3Uj;Qig==Lw~YI9Lq z@Cd^O|GUO05%+7&!|q3+R%Kc7c=R1x8NZdpXU1dr4lzQ=vb=qUjp`-web$!?cI!nu zrEFoF0L~|zDWEbd63rhC7e<0IjD33^j*lB(tFUmb#2;zq95h58*J_S?4&s)d!uLTF zW;cpBScg#x;D84umSHK-3STKO4f;d^_)T{!>`#~zp^%6Uw>Z@ylZ2{sFGMP_VX7a-W82n!5&8*F#Yl}1moWN24;BfiLA2}}=k2#ET(^b4IR&2gi<`+ zmN5^EnR3vBOmTPkuMr}s5?XxtY0!2{b(?jAXMe)!$K9d_lj8m7d=GjrDyk>H-U%w> z38t2nALj94Vd*>1G9I81mbD3PY^8m}cj8=AtjMU3Ot9>a1puTMOi(>Y7v0=re)l)5 z%*#eAGu^Oih~>2^Y^0GonlRwya+a@>(ND4j2boHD&mv?@N~%f|=2`IphK)Nm zR2ixko3D^nxY?G>|Afe|uS)79MulzFj{LOPGQH#>6+Q`^=(~7z7=Uf|!_R(1hzi-8 z+stLsVm#zc)c3y~s^m9JA5;Iryxfjs8Ta{NS8u?!{Iyy|xDhRwZ|GcH5-1ziq1(TM z$2Q048WOQvBUo zt6%Y74rty%*XsS{CeU6LW1gjMwHCR8#DKWNi2k{cU0W*^3a^zv=m46x+Y9T(#?>-a zz&y1FM`G~|2p~|XTU0@bdO?gPUd;TLsV2MtO|W6}y{EJ0MzI$G6Ftudy6$Mzso%O^ z6Z2>0%R0FCd(ZjO9IiBMGS}FH=*pO7D~7!nbP8JTo3x1*NC~G-ee1LC6EYx`#k-a} zu^lcURIlyk2Pi0h1pL_^ZI@k)(-ZWox~tQN`%poa6iEpx9k)A8Wy~cem_5lufR{Nvbh)xC3dqD zFNi6d(E=!~Vt3Y!@Un#6c)zN3CTd>H(>uf|NK5Tc6I4Q-#8Odes>ll0 z6`DOXJnj1{r0$k=eP>+O4ckoNM(VC4xl<1_IDVVWYV0Hoa{4a-*v&_=QTs1!txeMZ59dpXVoh0d)h;KAPe z?u=LhkWpGzL+aDFn;lhbDwBQ_0#C|Xod}+`xZ!#)pK~C=hUf~DcR8u-aNY{x5Azak zzAnFdZOfLeH37qZzvh=&zr@nTyCVSY67z@!EoprV6{2ZG6gK-Xd_;nFml`f8;WS`_4Ilkx~kKzFtXqXS9pWTSY8x(Ve{t!!E}??^prxze}S|3=5P zY-*Rv7qRnCu$1v{n6%Gl`<-_zBNx%9STs5?T=q?Z6){7d=;G~2+^9d&s9E$FVLNio zwi2LQ3uP%kb!MLU2}jzHgBlTZ4TagH$G1oD9?)3bs@UTqcgqHrNyE~6&|InLiY`#< zR`VHq*`{ZnzNPSvdj02yL7ap^Tv6RUG=DHW1?WVE8oEgt?rTB#wY<`B0I4;4d}yJ` z(caR6%u>(WO8vdqp1gPCNiA@kkE8mC6jS^JC(YI?Y!#@N1M2R93bzZ>Q{NpL!KIAT zv%l1-T{`p!QtWD6fqL{r8nSOM^0>!b&mYJuT?Bd?df~}z|29LVnyrU+wV-QQT&~cZ zImRkOB7I&tXI!L|Zu>pfY56UDsV;x9N3s-pNnyv8^&0N%JE=v}8Ek*f&b$e(~Ey?Oz8>JWV{w0v^$^uS#YyLaJ#{~b%N|m{p~U3x&^g~?7mMAWPF^6@0s77 z)7XB}9k*9z5LE4CM*OG44Ef*w{%7Q0H*fs2d!1Fitk7?o9H%IB(8@Hj7Z&I6)=01< z@`hVk+}ysZOu@lu(VM$-LJuXlPzpvYooxeQhV(=R2fG!NnIgMTa>(P9q!N96XvjvG z8k}_csR?`^?y0^Xu(fsr1!_enWj&<3vR>$8*M`kUH59_}CQ)6$Mmai8?iKt+T7TmT zCEZBge)FPd(QNK9JOe8LwpcNma%I{t-Ta6x^+STiw0-#bzKhVPwuGXc@h4tbiYF&| z`hf*g1}&&><_V8kwjK)t0gVPBF*eQ}cMU6g*Kg3;O}11*90)+TFM4*Ue~+GvIh`2J zy>scEp*%Y>H033HcF3Eb-k{fKoX3;_C$mSF-L_ba*<9851Hs}SgMw|9P2auDm^I!! zbkheYzsw)Vv#R!bBj26*_ZNiMYm@7Z&jA5xiFeg|04JTGr_J@?5v93_mFU&)YU7)h zhGz3`M!{_t=;-H`B(fNRa~(&5(=`E{Sp{j#dd!vTgU8gVY?@rv8Ye%jB<6Iuf(ozn=rddz?mrcR%pddo@{7RbEVIx@FyYPa|=D2lFKOva?SlUeAD_3pzlKVQ#EqoVf;~WYo%>;LHdLf!537BjXdm?DXZ=Qd=*I#9n-{vnsii@B@63%! z*3$bM(NPwLf+>mOqE%xrZsniI?T-E1EA;U{U`hW%lm7qyaQ-W>y??e7|J|ScD(h8jarHMg`X<}oUWs;R`#B8Eh`+TymQEs7dK)hr^W zm?Lc!RW&6dW>wWl3~3P|gx}Nq{?_`g^{OV^YQTsfK9L2@$u~&{`on27`XC4ZY+S0@6t8!wW|)1xyw_2&x^XzbdLMCH9e{$ zl{Wd{?WbFO_oYqajz~}6jK5>%>UZuQXri|AiEDQi0b9NE?pL1+N~q+cMtd= z0XyUbd074H@x)IAFW8ftk93aONgp~T%{O~EE^FVnHTUlz_S zuVN(*<-QH0QmnltN&V3m_x##8Fh0J|4Wl@NeSCZk2lfl|{dx4L10UZXzuWBN`|Hmu zhxmTq|NppXdW85u%}123)^J$4SntlZ;P&PN|M9I86@JC|oQ-G?xaKnNBJ(hsM=|~n zM*FvNMVm5Muexw&7RqQ4$&DX3h$9P4RJ*goWrie6c)o2cNvp5gJ%9Z_R{4JmNr3f6 z*jQo@=>!o}5(9@PLI-JbgeZLQDBzLI3A(aLotOg zRJe^LN*}Ng{lRkbSp`16UwZapdlsMc=ZOM(W2)>-X3|H0v_)i6w-A zGB^}<5^3HPTjQEoLkZPCD7!?TWQjsw!ktm;eg+?Q-+S;n@js4+{}ngS zPa+dbs6M|rQ2d>qy5RatsHu{}ujqzB&mwF(V%jWGq!SZmhVF&ayZ_AtR7c*X_`haiW|Jrt~NN+qs%NaqBnHl3FEmuvw~f`QF5ddp-|BgKj1U71b&crNumx2pR69 zYLKH20q}lpI+268bLP6Oshxi7lp~YXoyX3jM}6whOi8I9@wsmMvPC)Y`tfD654T&- z+tesi-!1?@C^K8!=aS*gc+mJT$inlNGN%R3_6QVSG-!^yv7f0b3DG@ZdF9n@m+}X7 zCoSY3-f_+-4(&I$Bh9ePSNpYDxr*3{Nr!>3L8Y6{&dn?LE=BxxLk)iEuos?YUU5a! zIm6rASqHuht=6HtW_XWfV3bs^9aLyYR?wg&1TXHXJboQU?fj%SupO1har6#GTRI*u z5J!E8xOY$O+idoSNEU5BA}UI|JbR`dihE7<_fGY~DN{5oD+%Fq&#PaGJ_>wzrL&<{ z+oHEN?D`<6w>F<1MN;aO?yW5nPq1KC;}R9{zJqNz9Um>6U;4enUdd`DO6E=(-p;45 z!%klZ*58a0zlI;pH6!B{Q^caT;>Ye>*e^WsE&=|!{rDhR+v!_Wo|Dv4$y89|Hm|*B zZ6`9rTOW?=t5qV5Gz4wX^|%X#2Qaj+U%y^ylv>}kBrmc*YFMtlzVH!-r`3$CzzI+W zlj)!0?L4EcWh;2V*BOwPt#GhHNZ&EwfH8hvH^34rJXt@U3hXu><-RPye_x}f@w-d& zQ7eGoRbXwAF(DzVZ3KwDy}fQpNy$lBaP4{UshIMlfT*agVKSMVr8gh~fj}(qw24;F z?x&Jew*7=ZLALhy#CHS77wi0JZ8;GBM!y6ecedl;d3>HWlw+~Yiy>Wj z{CbAfT3lRw9UQ8zuC5p{g`hq2a=t|TQPtl}M!CZlM{@*FBSBaM1vBq*abZg^#DM(^ zvT=ggIlpIPYcpnOPhrPpW{S_M|lb+qWZ%h77C`L zuG^mWcFq8N4ahYdSc~@dmKj)U%a`tXgx=m*&d)E{0fOg!#t zj3(CS{rNRl5s+S?6k zL^ZWo`r$0AL|&ui(p#)JM}udDyY9Hq&X%#MAFhy>!v%pa&gAPLmqg^5GSWlPYW`O13If0`Mv7r&W$-972<7P zeR+dpv<_yJqkFFpBK_>-d%|0%I#8SyH|?m^ueoYICU(~6OexF(WEY4;yY=;#GEuO^ z0m5-=yL4SsQ)CnpiS!N+-{?sF=pV??>=?d7j9`CErC`lrQ(owV6IFd?Q2AmvWrnft zNY17|{8okgom^=xUES^e`1ts45fPCA6e|J>=cngVxqMf7hmuYM^Th~Q59Gk9vfz*) z&dSt{(*|uqffpv*Z6!-m!y4%GGCb~D^X}J3cYC+O+-t;wPiE#=7;Dsnz1iUgCiiCy zJtWqmwaPNPn}9IT5Bu`|5lfeT(#yHwOD>!tJ|L{BaMU3wx8k_9t-CJn87If*tC0g zOE*fP%EkRI-G@fU|C#fCl-*(jaB2~Sp%kHmIWUb1R2qe? z9jEP-*fBr@ncU#~6-MWw(kUU6ZP_F{iFXB6@!d2wsm%xlCRb$)xdj#q>Erj7mPT@ZR?; zn}5j@U$|~5KDAEu+ISOpKq6mG_Pj??a&&Q#?G5kZ#n!H^%i!jsc0xE6MPd@I4LHpM z#2<>JRYj?L{qqw1)2Mh4S2+gTyOgwt%Y!v7g|Whp+B0?E&k;4UOnukBw`?+>HL%a? zo+_5LxLuLcQr1wQ9s2E))|wuv@xtQ}z@W8LLDpW*88Yd8-BKHip~KG%nLwf^y4o2& zJH8zL2a#<#5>XHlCePt+&SAN4Qz>q4h4J@Qllw&d#=z=XwsLBJ-^h7V423v-%3a}f zFlRI@>-^mvaI6EQ_EKe4Ga%GjxT$nR@bB@9(aroF&$OnuPB#_?!RRCSI#_aO|LNer_|AHvJ{#N%w;lXQ?}qEaywk%TAV zT`$ks)DX8UW94jExqZvg4=(vKPMAmyz&YthB~Gws1E}v#f;rM^Ls$qae*DZsOKy1vRnIP(&QjsOCYROE$P*`mUYS zqU_T-?7L(0pGyU90q4QbXCmXjucfN6i-kq@ub5OGYpn^R{E~g0kQH^iC@(GJ_>`CP zHT*F}1lDp0?*tNggSq-@|AC5*Fv5v_TO0CNN##PNW-AcYeEOlju{ACdTM2 zm4u{?5m=rTdpRtWeoWWm9=dSQe9bCe@LXgQoNa%ka(87PcDp30QjKPLgI*-1gyfB7 z*}0=Bj6O#>`{mc_6^OlLwdB?W!4vndbly}=O;VpVZ`6`6_b%qwXUw65cmskZ+z$rl z>A?T^ws91q5UKs`VfspbJ)i@ly=q+-B+UI5RjMyU22Euyp#fEmK&Ho;0Jn zxP0U%qGc(G9C69)7q!v>(J>L-cg}IRJ3s99I73F}@h`^j?&7(8-8 zskL@+La*T~B47x6mwfHiecE_6f4}QFV#5N#)l|J)1M_eua zJy)&F^19VKWZb#=*Ues1c*fG4D$`U@iW(9K5d-X#oA6#3JFC7DOKfzg&E;urQM~20$dBTlu1ix8(74g{i z#;e-_!`q=%%aka^?&l5zXD+hE?c}!X6wJy!i=?LDY(pZ5icT|$4TW)M+&{2IO3iw* z2%oX|C*#~`CcKzhlbf<(VY;_2Sst_1e$9$Z(uFYhBHIV!=T+ZL6HV+tAXATD(^| zG_EwoS+ZKEBvzI{$rk3^;VFn<|ESF`ufO*3YxKR*AGXEo>oN&}IX>8Ye6r>(DfGH6 znoG%V?-GO@Rc)Ll1*bSs>%tcImWi47>`3cSc^c}Kq7`&0F;>e-;wjM#=Q1b5Dk^Rd z)6RVQ@Kk5^V&d&n?a;?xh*i1!T@5c4;@p#<`5l`6TMbZUF!}s5z1M{|o>eHBTUuLV zr-HCl|8lB-k4cm2WIYE(@u%Bi^;8m~<0iZ;UY>qYY$D=B++Dv`-a0{^a$}S~fyE*e zDt`@>q1u#%rK6!8N^Y!a$*Kme%IsZrSun(j4}S=UFu`X%CSNUmfS3iKL(CBV!VA_855me70#<> zX3oA8f8 zrD@k~#xsI4!9qfI)?{@$23b7ZpX5`ksu)=VTFyS&>rM98pm4v>3qPgQlFiQb#M=<7 zzarLa)6@z^SPxSmZkuKn3+$@Xju%ipFC0goeJ+x|HT-Lts9;S-x`R5rmwcxNdVo7< zg*_533sw0@AU~+fO7X*id>Pv1Ibha|H~Z<>_qohfHcL5^oZ?q_Btld&_k?kGyx_i8 zykyrs3_iS6jXWz4+z?;RQbmT!ywIye^(QmLPL7Uza(eUkzI-7K^9JdBCqOAd1OHSP8a0@=w}cujerTe(X? z2_ftY=cq^u@^aJenANP*^rgUc`@~8+MPkEeL`buO{zu9E*GK*BAP=IibH5K2x%FcwuNu@`fX%Fy*Co<(&q zKGo8^{2g@|cq<(?7Y1@&Z3!MxsB=2@hr8ur$jj$p0dE;_Ibu7aA568WizpCJBP>#L zgWg{_SnqUn?vCUnVXXH}3S2RU!~XJ*2$|5bkGSwA~B zS1Y@^;5nL6`~(YgH<0B~d+eq%`1QL4=yxm$Yo8Jx3~<`cgy7$s2|aRF^t`#L3TBpo zzz}+_Oz;$PrQyVAzpL@jbsUhSqWGh|q8DD=o|!Ay>&(oXCd5SJTgtNU5N})nXG!Nj ze06EH;6jXW(4!)6Im~j@JnuUTqp_3q@;lnyGK1;V8!2n&7co(mQ@OVUcUCSk*T++9 zxlulUs&}s%B7J|<->I%xps=Y;n{2Xm%YNm zh8CZ}*ez{e--Y8L+*y6dvq?DnbHC~di~}N3%h1cyXj5f_2y#JhV!F!Ku)Q88x1nsPYvgf6{&aY?3GxnpUFb-1v^<)DC}rg#pbAAw z)Nv%WjJL{^Nd zJGtKb%hN>JViKxx$o)3~i5G_T6HL~W(7SbO_XzGc`O?nzVKbWV^MAZrhdrbkL5*E zREtkwpTBQM>qR{B5n7J@QFZLYg-SSvWxm~3-zce|ig8PsHh|ZY6!oL}8I2?-rb}t= zsq+NN-4k@<;6@d`nlDA1r4XiUmO;#ig_SA|458WeXUmgZ`Uz^mJ2Z$!sz#KOZ+z){yp{KIV-V0Yxc`%p&0( zF^nT7hxHw9U18e~K?1&NA|@ax#T0L7Fu zp$w4qW#Zi-_#Y`Jt(|#tiAmuR#$)d|E455nlX4)xOpy&}dgt#wqH`bOkzIp*tXp5J za3wj+OTo1ejhJS9O3hUB&=2&%yIwTQ+uxXQr8Dw->OEPrdpmda&b!Mz*R)eO@04(h zSM$Jx#IHRHagNCUbXH0B^W^wY(=|{gE+D zs!egIQe8*YZyn3hg6F)55W{$uJDh>F zmR1^sJH1Y19e1+dKorqcp!%q3c#QMKWq}CAK4F@eIY{{{xjd>|_Z;p8`O~$RuQVo( zyF2BXs!~$P+{=g(jlFc-&G1HXYEfBL!Ul+96?tKh`rt#loN>tD+1a@$F;K56AwF3l<^SAKxJcird%U z`!TAjoPy`hRvhnm@2u&%y>s0!2w{&$84nCP(dS37{P;asb-CmSGF{EbX>j&EaE6O> zx#eIfUGZ?N%|ry!w#)S@7o3!lT!8AM`xkP8Bb7Pt5VkD5(lt^|qf0OZHt+_`Xj{j( zyHo@gw^x0-T*?-EpNS$rI)u+~10A*%NnP+F&wJxZk$sDS>L#oyMT~P8x9gD|Fm~;G zSKI!9$$ayI1`PY8 z*Gf_%qvf*H@lhJFBbg?;=m`b&fPF`Z3C zwJ#K3%x+`wnQ=q!u-k7LJ0TZ>Uv5@kE17O2oFreZLc{7XlefD4#>FFkdigGY#V7ub zdRf=KrF0Kxvl(#krzp%z(7fwQ-l7cq@h@F|we6iQm~&3eO4wmjuy}$$uHp-a8fY(c z+;q5;jsCUWHk021dv`@Rwa@5+6mcmmpb9DlHy-?UeRKMu8|d2As(nA>ee*Ay(W?gQ zznjY`)6Xf*5PR>q(-z*PI>8*Y)@Gspb|N-q;jzsTT?8EXcDt?Ja&$O=H2~66)&! zD6Y4J3e=m|sMpx-B?Gw#E>o>AKRRcqbfuA-wOt%95C_Gg@+r=u5X>Uj9mw^1w9JK7 z4l?tlLb;#gs$~)H61-J`=b5rMQ5Palm#e-lt9n--VK+`m=~o6@=Y_Y_e*u!G^19;P z4lFtL$I7+*Cfb4!N4DT{`v?ldws&o*{%Uf(_}9IqGTItF_fz1=56O}eDpGPiI(s9= z54wF~dlPs|1?-_`nDf6k>dJ$U7YsL$pU^P|oEJmQ!r1e5&>U|RffLD_gM=`G2cppP zj-~dYWD5nhIUdxuu+BVG$Zu|Eer{>{=6CMe9L%u(26qFDUwaZ0_>SBwQnW|EUP@B| zyGM@oWRku7*FQ!v1oPR@PyG5#vQKjLh5h)_UNkdT!iMou%DLt#@H z_PT0tBDuxu%e7Nxr(`ebWpuh`yByOp#b&*4B#JPVKZ`4vNX9+>@vEmDIl-8k^9a9Q zER?zUzJ*&r8wrPSdQ;hXF1M32um>l`YO=GBYDQtwj*EOeVC6%8dhEQX(UpvTxItzH z7uKG=^TL$r1b4QFYhc6)so#y!nsrX8-<&Z0%8AZBm}}Yo@yk;{UUs@pH0KE|`I^b? zB%l&D%@hm^_S#vdt-9$3Aoem+#+?@!Q=iURI+nFp%I>_dQsBZ`bxj0kJlWq1RHOB> z+?pSjAdkWgy^2AS?$6tQv{1BQ5l~m0!eJynVyPDSb=d?6cWR%sV=q#c{GFap^k4VI z>kyGci{C4U3?tw8*D9lbTxWEK7UZQ;m%V49%I@y@n}KJ45#9H5w|RfoUo!kURgE}i zhVD1Nj!+Q-6K2XY4em(~`JS!iex91W$sP&Ju_j%z#RZ~iD%gyav#3tL0y$z}Db*19 zhAA@b8FrV;q)A(U5<~NvhRkNLX=3`(gG^$Cm}My}e>;y>qm3{3Z}<^dkoZ{5_|^W# zf~S>NYrD(FTPzE@BX50NuxDv-zfXCPh7Hkui}9fzOHI>AT>DArfM^WIpzsDT#TKp> znkzzP3})*GIouB(*x1c0M=9@cB8C-MWnnAmvf}CVvV`z$t8r?|?vK{wc3#Q)$Ku9> zEzFxD>A!Pu&$$&jm&wrDOAhj>X)74b7T-&!9XIDha(DY8F#VqrZOj85|GMhP8i_Xq z3Q-^V3uAwrH#y>)6mBKj=S|iP7ngi_QX)Uo@pqbbIwU7ij;dxaRn@8^gdEo$C&xW? zbh7dm%jlmBQ5SP-ZWUBL43tq5`#Oj&qGgEd((Z?>bdS+220-Qm?eds5B}7^^BNG2Y zrShT8~zEw7(qnnq0hQB~TL8_m+t9X(_1gFjK)ezc@oLR@bdq zeTZ04cp<&6HeR+bP+$^#P@&q9X~dzY}Tn*m-#!JW0`>)=6*MNnLpi@btIW<#r+Zrdq!N+%-5BX zog4{xDd5f9sBD7PoGV!9c9GNtG1Oh`Ms*Ep=>joY8EGCo*gJAs@!>7-SnhnJHj z3vrzN*z>)M{rGQ9LV)Fxlg)h^TI#3Aqf9-VA8Em{=Bq~jrQ`>wWCPwlY{as_UVG@f zh2l^vYfQ@H+xA}^3tZhn826N??kCu>i~Dz+33oT%)DOim8uRQcst*&U=?LYe;Q1eZ z-@|?EPMJ)-i5su?$ohL-U!L@69DwAKLan0m6bC|am6dny-LnO!-U$h@O9hI?I+*?| zhQ8(j3z#Y%B_Z`sq4HrrTx@WIi)kz50w{qoxcfOUGX1 zqjd$skZ1?JSsqTQiTIZJ^na?#A}+iway^IEM142Kr0<0TWw@KFu#3fBscg%4jqA}4 zm}jJmzq`qMUrbHQi=i+SK~cO+mbF*%`lwl`l4#5_;kCFAObqxT!fV$Y_M}OvvJdc$586hg(m!l4G+6J?l+c z+P<(8XE@y5N8eck`3UIEV>5W&GeH!!s>?ytZP#UUP<{k6bCFYy&w|#)=-Kmh&*R_V z&iyP3*rsKKM1p$a!KN^e%MI=cwf8AFoA0}$hZyPVvtD)5t}e>aM|;Te@I1s_q|{^W z7w}_&Udc|R6S^NOH=vmN8T0FIqappxHW~M7=QUFTnY`)M0}kiM#eQm;^emy$ zcq&?jcd=4=N?m@6)O)8wIKfXHt7wM{GchNs=0wygPqnHqr6a>_6^OPB9yIxLQ8v%z zw28VKLq9?c_{9e^#kY)N zDzG6-E3X?_UsChHF#8*Z21ql^%$Uo+Eex@=dG6MclC2k}8eO;qlIMEQw$c8*2=@@= zH%q)5W)LMK7?%UoM)fuYBw$*%?Ci>`&qb{iX{n&}?|ob(9!WYo3STEhUmP-uVS}%- zNtT3nwDRkWM{}^uGsFz1;$SHbk{*LkZ`bfBdY@P@i(K_$7`>zgdU(HZc*N5RHU(k$ zwKS-A*8{E1LRuIOm%5U6l&q7q_6v{hN4(``2natj*h80Ufn0c-x=GXRzGzN%-w(SZ zldjg>w^0l%2!);_sLtN;bwPf6L1V>q_|b~IlAJP3uQRZA@g|Sb`<3DmenT_~FCFeZGg=`IXC|=MUlKrjH6|^s6APN{MdrP**wwJn8|=Q$ zYUxtU!)jl&8^o!p+sknYNn|X$~~M#2>eU|NK{p<_;gdqxR)gnilC~eO{5nz!Hv@ zVrG|J8NB6_q(R9DhfGbA6Fq!b9pv0ckM7cI$Hkv|m;IazaS0cuxZ$|Q-azJpuu62` zHv~I&=J>kRzX}=Dm;eA!nl2Ik?U~zlgr~sh4Jouk{_c7y9zAimZ*^*=>g7C27zbHo z&NRjS#Emw#;lqa3UCTR+g4=oldC!L{Ia?bVw_HL6gbVwpH)R3x&;)=DldrH&7e4(P zH3VLI1&B(8JCu`D^-LZ#s?&fP3NU>HWo4$E>(#qX#67OLMSxvj_$WzK)7V+PuhyXn zP&I7dDExD)0dRYj?Zorjhxr-=k-$s30D|pCmy7O28QKj*3U5cedhTVFL) zY!dYYV@3&DIgrJW;fXczHlyG!BufbVNZPF}vgQH+!j$+iiSD|M!w{ELF-9oKi#@Z4 zD)ponZ*$Dn6tzgL{6U|h{6dUcnnX7|c#XjQ>DJ`qBH@&j5m=gE9({hS&Nmcim{}e) zXdVa+WL^!VaOO0VU2vYj`-6&ib~jkG>6%jO{BUpQem_cLt>$dJ!C0yR+n{hJ@;&oI z5Q7kE7xCp}KKs%)^l1G~U-NhKkN7?P!dPyh$#zKxCrx1Bv0B$z!3YN`IqYF)#`(YuNMA$E8Wn(Jxe?9T>~K8MBF zW>*;zoJosM^E6Ual;V>I_If%XdmZOZuFLm4w7oCTrq-p$9=0zftzlSU$!bBytgtx| zRXf1*ueoz$<{#TQ*E76{rFlqZ9>AD5g|AFH$Fd#^dY>=HfECdUx}jG+>LW8Vb|zjB z4W9nXfhf>r`UFc3-l-qL?>UHmBaVLsxhGTf*FLH9gGf88v(n&b=Zvf(3y)2MUkiOj zuO3t|an|o>+*J+A^e)#Mpu6Bk91uB;2jRNuxoHMlDutjOEy&)&%nN2Z*#X{>Dn41? za?g4IyhjH~b2|kkqChe^e+;2mSm!Y$mr?p`kL|ukNEz1bdhw<1=ma0=b^yP(b3-Ko z>+QUvjoMhUPdq8fRg2Fm3>GuckhQQgOI{z^PTJ)eT7Qie%_w?+TtuQ7v)jqz&NnOS zn;DCZ!6$04o|Q+=5U;4>U)FT%{fr) zRrP{_Qq-bNFlh!Rc$cYA8-K!U z7=Ny}^A{p*ZNPA^D|q)?a65O@^VyVVQda#)h4>0CR(`*UxH_ZS?IsIN2G}aHh9!U- zWYD+tORX?EE!EZE?GO^69>FN&rAEu0Pf$Do>ET`5WKf+3wk<&>|(Dq`fCCcz=O9?vW zRGCYqALy^En!no=v#%&;WMr7pa|a&4^633g_%}M84#kZ$Tb0fMfxHgjCO_#X&K0=_ z5==I-vpJNm63c0w(cdEA(aWnfp55zId<p#lY}}flW6K;ziLY=3zpse#QtYkB+9hjtL@pF*8Me*-F5lF1kS}1 zeGedDAn+-gKVcl;5=X|1{)H0~oHGQLxqcJ_{9(T<)<1;80Kx8rh zRPH6S0f@}@H1G8VZd9H~zjluFBOtQR8bpu6|C&ga(np5)C7FWEzyp}H^lAAAee6jeuGD<(jAbjKrGcvPF5D0>cF`MfRJ@c=0pwv5?wof z=w+f<$s3HxPgoPB4`6n{fPUE};M5oW+H!6`-mX*#hC0PFg&$}lYyu|_SM9c|eM=xU zk8rZGvs+e{s}OCAyrkrr3DL5g0YdxKw>8p;S`~YShKiQh1JEuxxio2TrL;YPnEr0f zZZ9>hL_S}4G$Yc(bD|to#-0Jd%99RYU@?`X=dNAF6gjQiXeIP~yVacK@rrv1V?_@? z`%wUASg9Dh(nWG#j`q}Vw+v{V$~Zl)Z%L`o`{5$^WBU7>H%EV>se!I7wj5}&sz-Ty zf>V3|y0zfM)=5;BWrr6I(hGKKXs+LV(l>$;SB(Z(Qev{yH}>5*$Eq2Dd7`RMoPV0M z^I}R6yN54A9N5(D1P{%~&t)hclQ?D$6M;=U)98Agi!qX zwU*iH0sowpCSPR@){j5?*MMi{0Zbj8z(W>70w*V)<@}_{Ni&K6ycgi@8$vWNL;Vj4 zR`?P0yZg#$yMa*Uk)QLHhz8i~ZUDo^{CDPoTxJryavHdg-`cS~z|&3BX~ z@?SN3JXwVQ3>3^jwe7an%5oykkMJ}HG7)Z5KfoUP$2|S34Av#BC1|E4wpbfzB9RS* zE>8j_XGx&VAwiP8A=Y3U9Q$9XJps_ze_{Oyp+_{CgwVeg2H@g3 z(dEKoP89#Rlm4TGn?S$LMgVkq#sfZ1H~!gQ!(H?krBmuR3jT3l#;s_|W7K{=)%yzt j|9{;+|I^>Gvv+^#cELiw3;+xA@qw>fU8^#>b^reWh2Y#7 literal 0 HcmV?d00001 diff --git a/versioned_docs/version-0.8.0/intro.md b/versioned_docs/version-0.8.0/intro.md new file mode 100644 index 0000000..52cae09 --- /dev/null +++ b/versioned_docs/version-0.8.0/intro.md @@ -0,0 +1,38 @@ +--- +sidebar_position: 0 +--- + +# Serverless LLM + + +ServerlessLLM + +ServerlessLLM is a **fast** and **easy-to-use** serving system designed for **affordable** multi-LLM serving, also known as LLM-as-a-Service. ServerlessLLM is ideal for environments with multiple LLMs that need to be served on limited GPU resources, as it enables efficient dynamic loading of LLMs onto GPUs. By elastically scaling model instances and multiplexing GPUs, ServerlessLLM can significantly reduce costs compared to traditional GPU-dedicated serving systems while still providing low-latency (Time-to-First-Token, TTFT) LLM completions. + +ServerlessLLM now supports NVIDIA and AMD GPUs, including following hardware: +* NVIDIA GPUs: Compute Capability 7.0+ (e.g, V100, A100, RTX A6000, GeForce RTX 3060) +* AMD GPUs: ROCm 6.2.0+ (tested on MI100s and MI200s) + +## Documentation + +### Getting Started + +- [Quickstart](./getting_started.md) +- [Single Machine Deployment (From Scratch)](./deployment/single_machine.md) +- [Multi-machine Deployment](./deployment/multi_machine.md) +- [SLURM Cluster Deployment](./deployment/slurm_cluster.md) + +### Advanced Features + +- [Storage-Aware Scheduler](./features/storage_aware_scheduling.md) +- [Live Migration](./features/live_migration.md) +- [PEFT LoRA Serving](./features/peft_lora_serving.md) + +### ServerlessLLM Store + +- [Quickstart](./store/quickstart.md) +- [ROCm Quickstart](./store/rocm_quickstart.md) + +### ServerlessLLM CLI + +- [ServerlessLLM CLI API](./api/cli.md) diff --git a/versioned_docs/version-0.8.0/models/_category_.json b/versioned_docs/version-0.8.0/models/_category_.json new file mode 100644 index 0000000..67c0cfe --- /dev/null +++ b/versioned_docs/version-0.8.0/models/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Models", + "position": 7 +} diff --git a/versioned_docs/version-0.8.0/models/supported_models.md b/versioned_docs/version-0.8.0/models/supported_models.md new file mode 100644 index 0000000..6077615 --- /dev/null +++ b/versioned_docs/version-0.8.0/models/supported_models.md @@ -0,0 +1,13 @@ +# Supported Models + +ServerlessLLM supports a plethora of language models from [Huggingface (HF) Transformers](https://huggingface.co/models). This page lists the models and model architectures currently supported by ServerlessLLM. + +To test a model, simply add it to the `supported_models.json` inside `/ServerlessLLM/tests/inference_tests` and the Github Actions will automatically test whether not it is supported. + +## Text-only Language Models + +Architecture |Models |Example HF Models |vLLM |Transformers +------------------|--------------|--------------------|-----|------------- +`OPTForCausalLM` |OPT, OPT-IML |`facebook/opt-6.7b` |✅ |✅ + + diff --git a/versioned_docs/version-0.8.0/store/_category_.json b/versioned_docs/version-0.8.0/store/_category_.json new file mode 100644 index 0000000..78b547f --- /dev/null +++ b/versioned_docs/version-0.8.0/store/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "ServerlessLLM Store", + "position": 5 +} diff --git a/versioned_docs/version-0.8.0/store/quantization.md b/versioned_docs/version-0.8.0/store/quantization.md new file mode 100644 index 0000000..3450410 --- /dev/null +++ b/versioned_docs/version-0.8.0/store/quantization.md @@ -0,0 +1,102 @@ +--- +sidebar_position: 2 +--- + +# Quantization + +> Note: Quantization is currently experimental, especially on multi-GPU machines. You may encounter issues when using this feature in multi-GPU environments. + +ServerlessLLM currently supports `bitsandbytes` quantization, which reduces model memory usage by converting weights to lower-precision data types. You can configure this by passing a `BitsAndBytesConfig` object when loading a model. + +Available precisions include: +- `int8` +- `fp4` +- `nf4` + +> Note: CPU offloading and dequantization is not currently supported. + +## 8-bit Quantization (`int8`) + +8-bit quantization halves the memory usage compared to 16-bit precision with minimal impact on model accuracy. It is a robust and recommended starting point for quantization. + +```python +from transformers import AutoModelForCausalLM, BitsAndBytesConfig + +# Configure 8-bit quantization +quantization_config = BitsAndBytesConfig( + load_in_8bit=True +) + +# Load the model with the config +model_8bit = AutoModelForCausalLM.from_pretrained( + "facebook/opt-1.3b", + quantization_config=quantization_config, + device_map="auto", +) +``` + +## 4-bit Quantization (`fp4`) +FP4 (4-bit Floating Point) quantization offers more aggressive memory savings than 8-bit. It is a good option for running very large models on consumer-grade hardware. + +```python +from transformers import AutoModelForCausalLM, BitsAndBytesConfig + +# Configure 4-bit FP4 quantization +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="fp4" +) + +# Load the model with the config +model_fp4 = AutoModelForCausalLM.from_pretrained( + "facebook/opt-1.3b", + quantization_config=quantization_config, + device_map="auto", +) +``` + +## 4-bit Quantization (`nf4`) +NF4 (4-bit NormalFloat) is an advanced data type optimized for models whose weights follow a normal distribution. NF4 is generally the recommended 4-bit option as it often yields better model accuracy compared to FP4. + +```python +import torch +from transformers import AutoModelForCausalLM, BitsAndBytesConfig + +# Configure 4-bit NF4 quantization +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4" +) + +# Load the model with the config +model_nf4 = AutoModelForCausalLM.from_pretrained( + "facebook/opt-1.3b", + quantization_config=quantization_config, + device_map="auto", +) +``` + +## `torch_dtype` (Data Type for Unquantized Layers) +The `torch_dtype` parameter sets the data type for model layers that are not quantized (e.g. `LayerNorm`). Setting this to `torch.float16` or `torch.bfloat16` can further reduce memory usage. If unspecified, these layers default to `torch.float16`. + +```python +import torch +from transformers import AutoModelForCausalLM, BitsAndBytesConfig + +# Configure 4-bit NF4 quantization +quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4" +) + +# Load model, casting non-quantized layers to float16 +model_mixed_precision = AutoModelForCausalLM.from_pretrained( + "facebook/opt-1.3b", + quantization_config=quantization_config, + torch_dtype=torch.float16, + device_map="auto", +) +``` + +For further information, consult the [HuggingFace Documentation for BitsAndBytes](https://huggingface.co/docs/transformers/main/en/quantization/bitsandbytes). + diff --git a/versioned_docs/version-0.8.0/store/quickstart.md b/versioned_docs/version-0.8.0/store/quickstart.md new file mode 100644 index 0000000..83492ea --- /dev/null +++ b/versioned_docs/version-0.8.0/store/quickstart.md @@ -0,0 +1,245 @@ +--- +sidebar_position: 0 +--- + +# Quickstart Guide + +ServerlessLLM Store (`sllm-store`) is a Python library that supports fast model checkpoint loading from multi-tier storage (i.e., DRAM, SSD, HDD) into GPUs. + +ServerlessLLM Store provides a model manager and two key functions: +- `save_model`: Convert a HuggingFace model into a loading-optimized format and save it to a local path. +- `load_model`: Load a model into given GPUs. + + +## Requirements +- OS: Ubuntu 20.04 +- Python: 3.10 +- GPU: compute capability 7.0 or higher + +## Installations + +### Create a virtual environment +```bash +conda create -n sllm-store python=3.10 -y +conda activate sllm-store +``` + +### Install with pip +```bash +pip install serverless-llm-store +``` + +### Install from source +1. Clone the repository and enter the `store` directory + +``` bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/sllm_store +``` + +2. Install the package from source + +```bash +rm -rf build +pip install . +``` + +## Usage Examples +:::tip +We highly recommend using a fast storage device (e.g., NVMe SSD) to store the model files for the best experience. +For example, create a directory `models` on the NVMe SSD and link it to the local path. +```bash +mkdir -p /mnt/nvme/models # Replace '/mnt/nvme' with your NVMe SSD path. +ln -s /mnt/nvme/models ./models +``` +::: + +1. Convert a model to ServerlessLLM format and save it to a local path: +```python +from sllm_store.transformers import save_model + +# Load a model from HuggingFace model hub. +import torch +from transformers import AutoModelForCausalLM +model = AutoModelForCausalLM.from_pretrained('facebook/opt-1.3b', torch_dtype=torch.float16) + +# Replace './models' with your local path. +save_model(model, './models/facebook/opt-1.3b') +``` + +2. Launch the checkpoint store server in a separate process: +```bash +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + + + +3. Load model in your project and make inference: +```python +import time +import torch +from sllm_store.transformers import load_model + +# warm up the GPU +num_gpus = torch.cuda.device_count() +for i in range(num_gpus): + torch.ones(1).to(f"cuda:{i}") + torch.cuda.synchronize() + +start = time.time() +model = load_model("facebook/opt-1.3b", device_map="auto", torch_dtype=torch.float16, storage_path="./models/", fully_parallel=True) +# Please note the loading time depends on the model size and the hardware bandwidth. +print(f"Model loading time: {time.time() - start:.2f}s") + +from transformers import AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained('facebook/opt-1.3b') +inputs = tokenizer('Hello, my dog is cute', return_tensors='pt').to("cuda") +outputs = model.generate(**inputs) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +4. Clean up by "Ctrl+C" the server process. + +## Usage with vLLM + +ServerlessLLM integrates with vLLM to provide fast model loading capabilities. Follow these steps to set up and use ServerlessLLM with vLLM. + +### Prerequisites + +Before using ServerlessLLM with vLLM, you need to apply a compatibility patch to your vLLM installation. This patch has been tested with vLLM version `0.9.0.1`. + +### Apply the vLLM Patch + +1. **Check patch status** (optional): + ```bash + ./sllm_store/vllm_patch/check_patch.sh + ``` + +2. **Apply the patch**: + ```bash + ./sllm_store/vllm_patch/patch.sh + ``` + +3. **Remove the patch** (if needed): + ```bash + ./sllm_store/vllm_patch/remove_patch.sh + ``` + +:::note +The patch file is located at `sllm_store/vllm_patch/sllm_load.patch` in the ServerlessLLM repository. +::: + + +Our api aims to be compatible with the `sharded_state` load format in vLLM. Thus, due to the model modifications about the model architecture done by vLLM, the model format for vLLM is **not** the same as we used in transformers. Thus, the `ServerlessLLM format` mentioned in the subsequent sections means the format integrated with vLLM, which is different from the `ServerlessLLM format` used in the previous sections. + +Thus, for fist-time users, you have to load the model from other backends and then converted it to the ServerlessLLM format. + +1. Download the model from HuggingFace and save it in the ServerlessLLM format: +``` bash +python3 examples/sllm_store/save_vllm_model.py --model-name facebook/opt-1.3b --storage-path $PWD/models --tensor-parallel-size 1 + +``` + +You can also transfer the model from the local path compared to download it from network by passing the `--local-model-path` argument. + +After downloading the model, you can launch the checkpoint store server and load the model in vLLM through `sllm` load format. + +2. Launch the checkpoint store server in a separate process: +```bash +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + +3. Load the model in vLLM: +```python +from vllm import LLM, SamplingParams + +import os + +storage_path = os.getenv("STORAGE_PATH", "./models") +model_name = "facebook/opt-1.3b" +model_path = os.path.join(storage_path, model_name) + +llm = LLM( + model=model_path, + load_format="serverless_llm", + dtype="float16" +) + +prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", +] + +sampling_params = SamplingParams(temperature=0.8, top_p=0.95) +outputs = llm.generate(prompts, sampling_params) + +# Print the outputs. +for output in outputs: + prompt = output.prompt + generated_text = output.outputs[0].text + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") +``` + +# Fine-tuning +ServerlessLLM currently supports LoRA fine-tuning using peft through the Hugging Face Transformers PEFT. + +ServerlessLLM Store provides a model manager and two key functions: +- save_lora: Convert an LoRA adapter into a loading-optimized format and save it to a local path. +- load_lora: Load an adapter into loaded model. + +> Note: Fine-tuning is currently experimental, especially on multi-GPU machines. You may encounter issues when using this feature in multi-GPU environments. + +## Usage Examples + +1. Convert an adapter to ServerlessLLM format and save it to a local path: +``` +from sllm_store.transformers import save_lora + +# TODO: Load an adapter from HuggingFace model hub. + + +# Replace './models' with your local path. +save_lora(adapter, './models/facebook/opt-1.3b') +``` + +2. Launch the checkpoint store server in a separate process: +``` +# 'mem_pool_size' is the maximum size of the memory pool in GB. It should be larger than the model size. +sllm-store start --storage-path $PWD/models --mem-pool-size 4GB +``` + +3. Load the adapter on your model and make inference: +``` +import time +import torch +from sllm_store.transformers import load_model, load_lora + +model = load_model("facebook/opt-1.3b", device_map="auto", torch_dtype=torch.float16, storage_path="./models/", fully_parallel=True) + +model = load_lora("facebook/opt-1.3b", adapter_name="demo_lora", adapter_path="ft_facebook/opt-1.3b_adapter1", device_map="auto", torch_dtype=torch.float16, storage_path="./models/") + +# Please note the loading time depends on the base model size and the hardware bandwidth. +print(f"Model loading time: {time.time() - start:.2f}s") + +from transformers import AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained('facebook/opt-1.3b') +inputs = tokenizer('Hello, my dog is cute', return_tensors='pt').to("cuda") +outputs = model.generate(**inputs) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +4. Clean up by `Ctrl+C` the server process. diff --git a/versioned_docs/version-0.8.0/store/rocm_quickstart.md b/versioned_docs/version-0.8.0/store/rocm_quickstart.md new file mode 100644 index 0000000..1707703 --- /dev/null +++ b/versioned_docs/version-0.8.0/store/rocm_quickstart.md @@ -0,0 +1,164 @@ +--- +sidebar_position: 1 +--- + +# ROCm Quick Start + +ServerlessLLM Store (`sllm-store`) currently supports ROCm platform. However, there are no pre-built wheels for ROCm. + +Due to an internal bug in ROCm, serverless-llm-store may face a GPU memory leak in ROCm before version 6.2.0, as noted in [issue](https://github.com/ROCm/HIP/issues/3580). + +1. Clone the repository and enter the `store` directory: + +```bash +git clone https://github.com/ServerlessLLM/ServerlessLLM.git +cd ServerlessLLM/sllm_store +``` +After that, you may either use the Docker image or build the `sllm-store` wheel from source and install it in your environment. + +## Use the Docker image + +We provide a Dockerfile with ROCm support. Currently, it's built on base image `rocm/vllm-dev:base_ROCm-6.3.1_20250528_tuned_20250530` + +2. Build the Docker image: + +``` bash +docker build -t sllm_store_rocm -f Dockerfile.rocm . +``` + +3. Start the Docker container: + +:::tip +If you want to run inference outside the Docker container, you need to pass the port to the host machine. For example, `-p 8073:8073`. You can also get the wheel from the Docker container after starting it via `docker cp sllm_store_server:/app/dist .`. +::: + +``` bash +docker run --name sllm_store_server --rm -it \ + --device /dev/kfd --device /dev/dri \ + --security-opt seccomp=unconfined \ + -v $(pwd)/models:/models \ + sllm_store_rocm +``` + +Expected output: + +``` bash +INFO 06-05 12:59:07 cli.py:76] Starting gRPC server +INFO 06-05 12:59:07 server.py:40] StorageServicer: storage_path=/models, mem_pool_size=4294967296, num_thread=4, chunk_size=33554432, registration_required=False +WARNING: Logging before InitGoogleLogging() is written to STDERR +I20250605 12:59:11.141070 1 checkpoint_store_hip.cpp:42] Number of GPUs: 1 +I20250605 12:59:11.141098 1 checkpoint_store_hip.cpp:44] I/O threads: 4, chunk size: 32MB +I20250605 12:59:11.141103 1 checkpoint_store_hip.cpp:46] Storage path: "/models" +I20250605 12:59:11.141119 1 checkpoint_store_hip.cpp:72] GPU 0 UUID: 61363865-3865-3038-3831-366132376261 +I20250605 12:59:11.519277 1 pinned_memory_pool_hip.cpp:30] Creating PinnedMemoryPool with 128 buffers of 33554432 bytes +I20250605 12:59:12.487957 1 checkpoint_store_hip.cpp:84] Memory pool created with 4GB +INFO 06-05 12:59:12 server.py:231] Starting gRPC server on 0.0.0.0:8073 + +``` + +After starting the Docker container, you can enter the container and run the following command to test the installation. + +``` bash +docker exec -it sllm_store_server /bin/bash +``` + +Try to save and load a transformer model: + +``` bash +python3 examples/save_transformers_model.py --model-name "facebook/opt-1.3b" --storage-path "/models" +python3 examples/load_transformers_model.py --model-name "facebook/opt-1.3b" --storage-path "/models" +``` +Expected output: + +``` bash +DEBUG 06-05 13:01:01 transformers.py:203] load_dict_non_blocking takes 0.0071375370025634766 seconds +DEBUG 06-05 13:01:01 transformers.py:213] load config takes 0.003943443298339844 seconds +DEBUG 06-05 13:01:01 torch.py:137] allocate_cuda_memory takes 0.0012660026550292969 seconds +DEBUG 06-05 13:01:01 client.py:72] load_into_gpu: facebook/opt-1.3b, 93b1932e-4b43-42cb-b82d-7228ef21810b +INFO 06-05 13:01:01 client.py:113] Model loaded: facebook/opt-1.3b, 93b1932e-4b43-42cb-b82d-7228ef21810b +INFO 06-05 13:01:01 torch.py:160] restore state_dict takes 0.0004298686981201172 seconds +DEBUG 06-05 13:01:02 transformers.py:224] load model takes 0.9706132411956787 seconds +INFO 06-05 13:01:02 client.py:117] confirm_model_loaded: facebook/opt-1.3b, 93b1932e-4b43-42cb-b82d-7228ef21810b +INFO 06-05 13:01:06 client.py:125] Model loaded +Model loading time: 5.28s +tokenizer_config.json: 100%|██████████████████████████████| 685/685 [00:00<00:00, 6.68MB/s] +vocab.json: 100%|███████████████████████████████████████| 899k/899k [00:00<00:00, 4.05MB/s] +merges.txt: 100%|███████████████████████████████████████| 456k/456k [00:00<00:00, 3.05MB/s] +special_tokens_map.json: 100%|████████████████████████████| 441/441 [00:00<00:00, 4.10MB/s] +/usr/local/lib/python3.12/dist-packages/torch/nn/modules/linear.py:125: UserWarning: Failed validator: GCN_ARCH_NAME (Triggered internally at /app/pytorch/aten/src/ATen/hip/tunable/Tunable.cpp:366.) + return F.linear(input, self.weight, self.bias) +Hello, my dog is cute and I want to give him a good home. I have a lot of experience with dogs and I +``` + +Try to save and load a model in vLLM: + +``` bash +python3 examples/save_vllm_model.py --model-name "facebook/opt-125m" --storage-path "/models" +python3 examples/load_vllm_model.py --model-name "facebook/opt-125m" --storage-path "/models" +``` +Expected output: + +``` bash +INFO 06-05 13:02:51 [__init__.py:243] Automatically detected platform rocm. +INFO 06-05 13:02:52 [__init__.py:31] Available plugins for group vllm.general_plugins: +INFO 06-05 13:02:52 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver +INFO 06-05 13:02:52 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load. +INFO 06-05 13:03:00 [config.py:793] This model supports multiple tasks: {'reward', 'embed', 'generate', 'classify', 'score'}. Defaulting to 'generate'. +INFO 06-05 13:03:00 [arg_utils.py:1594] rocm is experimental on VLLM_USE_V1=1. Falling back to V0 Engine. +INFO 06-05 13:03:04 [config.py:1910] Disabled the custom all-reduce kernel because it is not supported on current platform. +INFO 06-05 13:03:04 [llm_engine.py:230] Initializing a V0 LLM engine (v0.9.0.1) with config: model='/models/facebook/opt-125m', speculative_config=None, tokenizer='/models/facebook/opt-125m', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config={}, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=2048, download_dir=None, load_format=LoadFormat.SERVERLESS_LLM, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, decoding_config=DecodingConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_backend=''), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None), seed=0, served_model_name=/models/facebook/opt-125m, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=None, chunked_prefill_enabled=False, use_async_output_proc=True, pooler_config=None, compilation_config={"compile_sizes": [], "inductor_compile_config": {"enable_auto_functionalized_v2": false}, "cudagraph_capture_sizes": [256, 248, 240, 232, 224, 216, 208, 200, 192, 184, 176, 168, 160, 152, 144, 136, 128, 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 4, 2, 1], "max_capture_size": 256}, use_cached_outputs=False, +INFO 06-05 13:03:04 [rocm.py:208] None is not supported in AMD GPUs. +INFO 06-05 13:03:04 [rocm.py:209] Using ROCmFlashAttention backend. +INFO 06-05 13:03:05 [parallel_state.py:1064] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0, EP rank 0 +INFO 06-05 13:03:05 [model_runner.py:1170] Starting to load model /models/facebook/opt-125m... +DEBUG 06-05 13:03:05 torch.py:137] allocate_cuda_memory takes 0.0004763603210449219 seconds +DEBUG 06-05 13:03:05 client.py:72] load_into_gpu: facebook/opt-125m/rank_0, e8e7d900-652d-4822-8992-ad22f734b9c8 +INFO 06-05 13:03:05 client.py:113] Model loaded: facebook/opt-125m/rank_0, e8e7d900-652d-4822-8992-ad22f734b9c8 +INFO 06-05 13:03:05 torch.py:160] restore state_dict takes 0.00021338462829589844 seconds +INFO 06-05 13:03:05 client.py:117] confirm_model_loaded: facebook/opt-125m/rank_0, e8e7d900-652d-4822-8992-ad22f734b9c8 +INFO 06-05 13:03:05 client.py:125] Model loaded +INFO 06-05 13:03:05 [model_runner.py:1202] Model loading took 0.2363 GiB and 0.711783 seconds +/app/third_party/vllm/vllm/model_executor/layers/utils.py:80: UserWarning: Failed validator: GCN_ARCH_NAME (Triggered internally at /app/pytorch/aten/src/ATen/hip/tunable/Tunable.cpp:366.) + return torch.nn.functional.linear(x, weight, bias) +INFO 06-05 13:03:17 [worker.py:303] Memory profiling takes 11.68 seconds +INFO 06-05 13:03:17 [worker.py:303] the current vLLM instance can use total_gpu_memory (23.98GiB) x gpu_memory_utilization (0.90) = 21.59GiB +INFO 06-05 13:03:17 [worker.py:303] model weights take 0.24GiB; non_torch_memory takes 0.53GiB; PyTorch activation peak memory takes 0.49GiB; the rest of the memory reserved for KV Cache is 20.33GiB. +INFO 06-05 13:03:17 [executor_base.py:112] # rocm blocks: 37011, # CPU blocks: 7281 +INFO 06-05 13:03:17 [executor_base.py:117] Maximum concurrency for 2048 tokens per request: 289.15x +INFO 06-05 13:03:18 [model_runner.py:1526] Capturing cudagraphs for decoding. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI. If out-of-memory error occurs during cudagraph capture, consider decreasing `gpu_memory_utilization` or switching to eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage. +Capturing CUDA graph shapes: 100%|█████████████████████████| 35/35 [00:09<00:00, 3.55it/s] +INFO 06-05 13:03:28 [model_runner.py:1684] Graph capturing finished in 10 secs, took 0.13 GiB +INFO 06-05 13:03:28 [llm_engine.py:428] init engine (profile, create kv cache, warmup model) took 22.81 seconds +Adding requests: 100%|█████████████████████████████████████| 4/4 [00:00<00:00, 2079.22it/s] +Processed prompts: 100%|█| 4/4 [00:00<00:00, 6.71it/s, est. speed input: 43.59 toks/s, out +Prompt: 'Hello, my name is', Generated text: ' Joel, my dad is my friend and we are in a relationship. I am' +Prompt: 'The president of the United States is', Generated text: ' speaking out against the release of some State Department documents which show the Russians were involved' +Prompt: 'The capital of France is', Generated text: ' a worldwide knowledge center. What better place to learn about the history and culture of' +Prompt: 'The future of AI is', Generated text: " here: it's the future of everything\nIf you want to test your minds" +[rank0]:[W605 13:03:30.532018298 ProcessGroupNCCL.cpp:1476] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/distributed.html#shutdown (function operator()) +``` + +## Build the wheel from source and install + +Currently, `pip install .` does not work with ROCm. We suggest you build `sllm-store` wheel and manually install it in your environment. + + + +If there's a customized PyTorch version installed, you may need to run the following command to modify the `torch` version in `requirements.txt`: + +```bash +python3 using_existing_torch.py +``` + +2. Build the wheel: + +```bash +python setup.py sdist bdist_wheel +``` + +## Known issues + +1. GPU memory leak in ROCm before version 6.2.0. + +This issue is due to an internal bug in ROCm. After the inference instance is completed, the GPU memory is still occupied and not released. For more information, please refer to [issue](https://github.com/ROCm/HIP/issues/3580). + diff --git a/versioned_sidebars/version-0.7.0-sidebars.json b/versioned_sidebars/version-0.7.0-sidebars.json new file mode 100644 index 0000000..182c8e5 --- /dev/null +++ b/versioned_sidebars/version-0.7.0-sidebars.json @@ -0,0 +1,72 @@ +{ + "tutorialSidebar": [ + "intro", + "getting_started", + { + "type": "category", + "label": "Features", + "items": [ + { + "type": "autogenerated", + "dirName": "features" + } + ] + }, + { + "type": "category", + "label": "Deployment", + "items": [ + { + "type": "autogenerated", + "dirName": "deployment" + } + ] + }, + { + "type": "category", + "label": "ServerlessLLM Store", + "items": [ + { + "type": "autogenerated", + "dirName": "store" + } + ] + }, + { + "type": "category", + "label": "Developer Guide", + "items": [ + { + "type": "autogenerated", + "dirName": "developer" + } + ] + }, + { + "type": "category", + "label": "Models", + "items": [ + { + "type": "autogenerated", + "dirName": "models" + } + ] + }, + { + "type": "category", + "label": "Community", + "items": [ + { + "type": "autogenerated", + "dirName": "community" + } + ] + } + ], + "apiSidebar": [ + { + "type": "autogenerated", + "dirName": "api" + } + ] +} diff --git a/versioned_sidebars/version-0.8.0-sidebars.json b/versioned_sidebars/version-0.8.0-sidebars.json new file mode 100644 index 0000000..182c8e5 --- /dev/null +++ b/versioned_sidebars/version-0.8.0-sidebars.json @@ -0,0 +1,72 @@ +{ + "tutorialSidebar": [ + "intro", + "getting_started", + { + "type": "category", + "label": "Features", + "items": [ + { + "type": "autogenerated", + "dirName": "features" + } + ] + }, + { + "type": "category", + "label": "Deployment", + "items": [ + { + "type": "autogenerated", + "dirName": "deployment" + } + ] + }, + { + "type": "category", + "label": "ServerlessLLM Store", + "items": [ + { + "type": "autogenerated", + "dirName": "store" + } + ] + }, + { + "type": "category", + "label": "Developer Guide", + "items": [ + { + "type": "autogenerated", + "dirName": "developer" + } + ] + }, + { + "type": "category", + "label": "Models", + "items": [ + { + "type": "autogenerated", + "dirName": "models" + } + ] + }, + { + "type": "category", + "label": "Community", + "items": [ + { + "type": "autogenerated", + "dirName": "community" + } + ] + } + ], + "apiSidebar": [ + { + "type": "autogenerated", + "dirName": "api" + } + ] +} diff --git a/versions.json b/versions.json new file mode 100644 index 0000000..25cdde7 --- /dev/null +++ b/versions.json @@ -0,0 +1 @@ +["0.8.0", "0.7.0"] From 35d4046c2430ce847f56ea3ea808ae0592820737 Mon Sep 17 00:00:00 2001 From: future-xy Date: Thu, 6 Nov 2025 19:19:16 +0000 Subject: [PATCH 3/3] fix: set default to stable --- docusaurus.config.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index bbcc7b1..642b378 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -98,14 +98,14 @@ const config = { }, items: [ { - type: 'docSidebar', - sidebarId: 'tutorialSidebar', + type: 'doc', + docId: 'intro', position: 'left', label: 'Documents', }, { - type: 'docSidebar', - sidebarId: 'apiSidebar', + type: 'doc', + docId: 'api/intro', position: 'left', label: 'API', },