From c71290c5336d469abea7b9d8ccae1d6ccb9aacb3 Mon Sep 17 00:00:00 2001 From: WangCong <543529648@qq.com> Date: Thu, 31 Jul 2025 15:08:16 +0800 Subject: [PATCH 1/8] change docs outline --- docs/source/developer/add_connector.md | 2 +- docs/source/developer/block_layout.md | 1 - docs/source/developer/index.md | 3 ++- docs/source/developer/nfs_connector.md | 1 + docs/source/developer/performance_benchmark.md | 1 + docs/source/getting-started/index.md | 1 - docs/source/getting-started/quick_start.md | 1 - 7 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 docs/source/developer/block_layout.md create mode 100644 docs/source/developer/nfs_connector.md create mode 100644 docs/source/developer/performance_benchmark.md delete mode 100644 docs/source/getting-started/quick_start.md diff --git a/docs/source/developer/add_connector.md b/docs/source/developer/add_connector.md index 37552e9f..8d456ede 100644 --- a/docs/source/developer/add_connector.md +++ b/docs/source/developer/add_connector.md @@ -1 +1 @@ -# Add New Connector +# How To Add New Connector diff --git a/docs/source/developer/block_layout.md b/docs/source/developer/block_layout.md deleted file mode 100644 index a7365ff1..00000000 --- a/docs/source/developer/block_layout.md +++ /dev/null @@ -1 +0,0 @@ -# Block Layout diff --git a/docs/source/developer/index.md b/docs/source/developer/index.md index d6069244..488d86cf 100644 --- a/docs/source/developer/index.md +++ b/docs/source/developer/index.md @@ -3,7 +3,8 @@ :::{toctree} :maxdepth: 2 architecture.md -block_layout.md add_connector.md +nfs_connector.md +performance_benchmark.md ::: diff --git a/docs/source/developer/nfs_connector.md b/docs/source/developer/nfs_connector.md new file mode 100644 index 00000000..629c2daa --- /dev/null +++ b/docs/source/developer/nfs_connector.md @@ -0,0 +1 @@ +# NFS Connector \ No newline at end of file diff --git a/docs/source/developer/performance_benchmark.md b/docs/source/developer/performance_benchmark.md new file mode 100644 index 00000000..927cc276 --- /dev/null +++ b/docs/source/developer/performance_benchmark.md @@ -0,0 +1 @@ +# Performance Benchmark \ No newline at end of file diff --git a/docs/source/getting-started/index.md b/docs/source/getting-started/index.md index f2f03c4c..e3f8e3e3 100644 --- a/docs/source/getting-started/index.md +++ b/docs/source/getting-started/index.md @@ -4,7 +4,6 @@ :maxdepth: 2 installation.md installation_npu.md -quick_start.md example/index.md ::: diff --git a/docs/source/getting-started/quick_start.md b/docs/source/getting-started/quick_start.md deleted file mode 100644 index 05cf8c1f..00000000 --- a/docs/source/getting-started/quick_start.md +++ /dev/null @@ -1 +0,0 @@ -# Quick Start From 11630f96f5bc955a4ca918329f5cc92f4d7a507b Mon Sep 17 00:00:00 2001 From: harrisonyhq Date: Thu, 31 Jul 2025 11:24:28 +0800 Subject: [PATCH 2/8] [Feature] Add Cmake build command in setup.py --- setup.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a0ec1bfc..b50ce60e 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,9 @@ import os - +import subprocess +import torch from distutils.core import setup from setuptools import find_packages +from setuptools.command.build_ext import build_ext ROOT_DIR = os.path.dirname(__file__) @@ -10,15 +12,79 @@ def get_path(*filepath) -> str: return os.path.join(ROOT_DIR, *filepath) +def _is_cuda() -> bool: + return torch.cuda.is_available() + + +def _is_npu() -> bool: + return hasattr(torch, 'npu') and torch.npu.is_available() + + +class BuildUCMExtension(build_ext): + """Build UCM Extensions Using Cmake""" + + def run(self): + package_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'unifiedcache')) + ucm_nfs_path = os.path.join(package_path, 'csrc', 'ucmnfsstore') + if not os.path.exists(ucm_nfs_path): + raise RuntimeError(f"Expected directory {ucm_nfs_path} does not exist") + + build_path = os.path.join(ucm_nfs_path, 'build') + if not os.path.exists(build_path): + os.makedirs(build_path) + + os.chdir(build_path) + if _is_npu(): + cmake_command = [ + 'cmake', + '-DDOWNLOAD_DEPENDENCE=ON', + '-DRUNTIME_ENVIRONMENT=ascend', + '..', + ucm_nfs_path + ] + elif _is_cuda(): + cmake_command = [ + 'cmake', + '-DDOWNLOAD_DEPENDENCE=ON', + '-DRUNTIME_ENVIRONMENT=cuda', + '..', + ucm_nfs_path + ] + else: + raise RuntimeError( + "No supported accelerator found. " + "Please ensure either CUDA or NPU is available." + ) + subprocess.check_call(cmake_command) + + make_command = ['make', '-j', '8'] + subprocess.check_call(make_command) + + output_lib_path = os.path.join(ucm_nfs_path, 'output', 'lib') + so_files = [f for f in os.listdir(output_lib_path) if f.endswith('.so')] + for so_file in so_files: + src = os.path.join(output_lib_path, so_file) + dest = os.path.join(package_path, 'ucm_connector', so_file) + os.rename(src, dest) + + os.chdir(os.path.dirname(__file__)) + super().run() + + +cmdclass = { + 'build_ext': BuildUCMExtension, +} + print("FOUND PACKAGES:", find_packages()) setup( - name="unified_cache", + name="unifiedcache", version="0.0.1", author="Unified Cache Team", description="Unified Cache Management", packages=find_packages(), ext_modules=[], + cmdclass=cmdclass, package_data={}, include_package_data=True, install_requires=[], From 6aa8309f3c9bda9b44d7643307884dfdcf4c0f0b Mon Sep 17 00:00:00 2001 From: flesher0813 <1208954694@qq.com> Date: Fri, 1 Aug 2025 10:50:16 +0800 Subject: [PATCH 3/8] [Fix bug] fix issue#25 issue#31 and issue#33 --- README.md | 2 +- VERSION | 1 + docker/Dockerfile | 7 ++-- docker/Dockerfile-NPU | 8 ++-- docs/source/getting-started/installation.md | 11 +++--- .../getting-started/installation_npu.md | 16 +++++--- setup.py | 37 ++++++++++++++++--- test/dump_and_load_on_dram.py | 23 ++++++++++++ test/dump_and_load_on_hbm.py | 23 ++++++++++++ test/test_uc_connector.py | 24 ++++++++++++ test/test_ucm_dram.py | 24 ++++++++++++ unifiedcache/integration/vllm/uc_connector.py | 26 +++++++++++++ unifiedcache/logger.py | 24 ++++++++++++ unifiedcache/ucm_connector/base.py | 24 ++++++++++++ unifiedcache/ucm_connector/factory.py | 28 ++++++++++++-- unifiedcache/ucm_connector/ucm_dram.py | 24 ++++++++++++ unifiedcache/ucm_connector/ucm_nfs_store.py | 24 ++++++++++++ 17 files changed, 296 insertions(+), 30 deletions(-) create mode 100644 VERSION diff --git a/README.md b/README.md index a14303ae..391ca4d3 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ --- *Latest News* 🔥 -- [2025/07/30] We are excited to announce the alpha release of Unified Cache Manager. +- [2025/08/01] We are excited to announce the alpha release of Unified Cache Manager. --- diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..67d8aeb0 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.1-release \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 06edc006..fe2c2b84 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,11 +14,10 @@ ENV VLLM_USE_PRECOMPILED=1 RUN VLLM_TARGET_DEVICE=cuda pip install -v -e /vllm-workspace/vllm --extra-index=https://download.pytorch.org/whl/nightly/cu128 # Install unified-cache-management -ARG UCM_REPO=https://github.com/ModelEngine-Group/unified-cache-management.git -ARG UCM_BRANCH=develop -RUN git clone --depth 1 $UCM_REPO --branch $UCM_BRANCH /vllm-workspace/unified-cache-management +COPY . /vllm-workspace/unified-cache-management -RUN pip install -v -e /vllm-workspace/unified-cache-management +RUN export PLATFORM="cuda" && \ + pip install -v -e /vllm-workspace/unified-cache-management # Apply patch for vLLM RUN cd /vllm-workspace/vllm \ diff --git a/docker/Dockerfile-NPU b/docker/Dockerfile-NPU index 4216e292..519d4253 100644 --- a/docker/Dockerfile-NPU +++ b/docker/Dockerfile-NPU @@ -4,11 +4,11 @@ FROM quay.io/ascend/vllm-ascend:v0.9.2rc1 WORKDIR /workspace # Install unified-cache-management -ARG UCM_REPO=https://github.com/ModelEngine-Group/unified-cache-management.git -ARG UCM_BRANCH=develop -RUN git clone --depth 1 $UCM_REPO --branch $UCM_BRANCH /vllm-workspace/unified-cache-management +COPY . /vllm-workspace/unified-cache-management -RUN pip install -v -e /vllm-workspace/unified-cache-management +RUN export PLATFORM="ascend" && \ + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ + pip install -v -e /vllm-workspace/unified-cache-management # Apply patch for vLLM RUN cd /vllm-workspace/vllm \ diff --git a/docs/source/getting-started/installation.md b/docs/source/getting-started/installation.md index 4adfc2e0..5a718059 100644 --- a/docs/source/getting-started/installation.md +++ b/docs/source/getting-started/installation.md @@ -35,7 +35,8 @@ Refer to [Set up using docker](https://docs.vllm.ai/en/latest/getting_started/in ### Build from source code Follow commands below to install unified-cache-management: ```bash -git clone --depth 1 --branch develop https://github.com/ModelEngine-Group/unified-cache-management.git +# Replace with the branch or tag name needed +git clone --depth 1 --branch https://github.com/ModelEngine-Group/unified-cache-management.git cd unified-cache-management pip install -v -e . cd .. @@ -44,10 +45,10 @@ cd .. ## Setup from docker Download the pre-built docker image provided or build unified-cache-management docker image by commands below: ```bash - # Build docker image using source code - git clone --depth 1 --branch develop https://github.com/ModelEngine-Group/unified-cache-management.git - cd unified-cache-management/docker - docker build -t ucm-vllm:latest -f ./Dockerfile ./ + # Build docker image using source code, replace with the branch or tag name needed + git clone --depth 1 --branch https://github.com/ModelEngine-Group/unified-cache-management.git + cd unified-cache-management + docker build -t ucm-vllm:latest -f ./docker/Dockerfile ./ ``` Then run your container using following command. You can add or remove Docker parameters as needed. ```bash diff --git a/docs/source/getting-started/installation_npu.md b/docs/source/getting-started/installation_npu.md index 6c4322c2..8087ef4f 100644 --- a/docs/source/getting-started/installation_npu.md +++ b/docs/source/getting-started/installation_npu.md @@ -44,7 +44,8 @@ Codes of vLLM and vLLM Ascend are placed in /vllm-workspace, you can refer to [v ### Build from source code Follow commands below to install unified-cache-management: ```bash -git clone --depth 1 --branch develop https://github.com/ModelEngine-Group/unified-cache-management.git +# Replace with the branch or tag name needed +git clone --depth 1 --branch https://github.com/ModelEngine-Group/unified-cache-management.git cd unified-cache-management pip install -v -e . cd .. @@ -53,15 +54,18 @@ cd .. ## Setup from docker Download the pre-built docker image provided or build unified-cache-management docker image by commands below: ```bash - # Build docker image using source code - git clone --depth 1 --branch develop https://github.com/ModelEngine-Group/unified-cache-management.git - cd unified-cache-management/docker - docker build -t ucm-vllm:latest -f ./Dockerfile-NPU ./ + # Build docker image using source code, replace with the branch or tag name needed + git clone --depth 1 --branch https://github.com/ModelEngine-Group/unified-cache-management.git + cd unified-cache-management + docker build -t ucm-vllm:latest -f ./docker/Dockerfile-NPU ./ ``` Then run your container using following command. You can add or remove Docker parameters as needed. ```bash -# Use `--ipc=host` to make sure the shared memory is large enough. +# Update DEVICE according to your device (/dev/davinci[0-7]) +export DEVICE=/dev/davinci7 +# Update the vllm-ascend image docker run --rm \ + --network=host \ --device $DEVICE \ --device /dev/davinci_manager \ --device /dev/devmm_svm \ diff --git a/setup.py b/setup.py index b50ce60e..8b0c2103 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,47 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import os import subprocess -import torch from distutils.core import setup +from pathlib import Path from setuptools import find_packages from setuptools.command.build_ext import build_ext ROOT_DIR = os.path.dirname(__file__) - +PLATFORM = os.getenv("PLATFORM") def get_path(*filepath) -> str: return os.path.join(ROOT_DIR, *filepath) def _is_cuda() -> bool: - return torch.cuda.is_available() + return PLATFORM == "cuda" def _is_npu() -> bool: - return hasattr(torch, 'npu') and torch.npu.is_available() + return PLATFORM == "ascend" class BuildUCMExtension(build_ext): @@ -76,10 +100,11 @@ def run(self): } print("FOUND PACKAGES:", find_packages()) - +__version__ = Path(__file__).with_name('VERSION').read_text().strip() +print("Current version:", __version__) setup( name="unifiedcache", - version="0.0.1", + version=__version__, author="Unified Cache Team", description="Unified Cache Management", packages=find_packages(), diff --git a/test/dump_and_load_on_dram.py b/test/dump_and_load_on_dram.py index 12ae52da..55af3c2b 100644 --- a/test/dump_and_load_on_dram.py +++ b/test/dump_and_load_on_dram.py @@ -1,3 +1,26 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# # -*- coding: utf-8 -*- from unifiedcache.csrc.ucmnfsstore.output.lib import ucmnfsstore as ucmstore import secrets diff --git a/test/dump_and_load_on_hbm.py b/test/dump_and_load_on_hbm.py index d59197d5..a002afa8 100644 --- a/test/dump_and_load_on_hbm.py +++ b/test/dump_and_load_on_hbm.py @@ -1,3 +1,26 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# # -*- coding: utf-8 -*- from unifiedcache.csrc.ucmnfsstore.output.lib import ucmnfsstore as ucmstore import secrets diff --git a/test/test_uc_connector.py b/test/test_uc_connector.py index bff47297..1fb81877 100644 --- a/test/test_uc_connector.py +++ b/test/test_uc_connector.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import random import secrets import unittest diff --git a/test/test_ucm_dram.py b/test/test_ucm_dram.py index 1454fc2e..bbb66848 100644 --- a/test/test_ucm_dram.py +++ b/test/test_ucm_dram.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import random import torch import unittest diff --git a/unifiedcache/integration/vllm/uc_connector.py b/unifiedcache/integration/vllm/uc_connector.py index 6d9becdd..45339c1e 100644 --- a/unifiedcache/integration/vllm/uc_connector.py +++ b/unifiedcache/integration/vllm/uc_connector.py @@ -1,3 +1,29 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Adapted from lmcache/lmcache/integration/vllm/vllm_v1_adapter.py +# + from dataclasses import dataclass, field from typing import TYPE_CHECKING, List, Optional, Any, Generator diff --git a/unifiedcache/logger.py b/unifiedcache/logger.py index 3c5e03ba..4eb0ff78 100644 --- a/unifiedcache/logger.py +++ b/unifiedcache/logger.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import logging import os diff --git a/unifiedcache/ucm_connector/base.py b/unifiedcache/ucm_connector/base.py index ec2c3748..e30ec049 100644 --- a/unifiedcache/ucm_connector/base.py +++ b/unifiedcache/ucm_connector/base.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + from abc import ABC, abstractmethod from typing import List, Dict diff --git a/unifiedcache/ucm_connector/factory.py b/unifiedcache/ucm_connector/factory.py index f7973676..9644511f 100644 --- a/unifiedcache/ucm_connector/factory.py +++ b/unifiedcache/ucm_connector/factory.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import importlib from typing import Callable @@ -39,10 +63,6 @@ def create_connector( return connector_cls(config) -UcmConnectorFactory.register_connector( - "UcmOceanStore", - "unifiedcache.ucm_connector.ucm_oceanstor", - "UcmOceanStore") UcmConnectorFactory.register_connector( "UcmDram", "unifiedcache.ucm_connector.ucm_dram", diff --git a/unifiedcache/ucm_connector/ucm_dram.py b/unifiedcache/ucm_connector/ucm_dram.py index de4c2f08..3fa740b4 100644 --- a/unifiedcache/ucm_connector/ucm_dram.py +++ b/unifiedcache/ucm_connector/ucm_dram.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import torch from dataclasses import dataclass from typing import List, Dict, Optional, Any diff --git a/unifiedcache/ucm_connector/ucm_nfs_store.py b/unifiedcache/ucm_connector/ucm_nfs_store.py index 0f2d8e2f..9d3040b3 100644 --- a/unifiedcache/ucm_connector/ucm_nfs_store.py +++ b/unifiedcache/ucm_connector/ucm_nfs_store.py @@ -1,3 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import torch # import ucmnfsstore from dataclasses import dataclass From 0f81cb564364f81d6839376c8ad5ad201b836188 Mon Sep 17 00:00:00 2001 From: harrisonyhq Date: Fri, 1 Aug 2025 11:37:16 +0800 Subject: [PATCH 4/8] [Fix][Docs] Make example runnable and add performance data (closes #37 #29 #42) (#41) * [Fix] Fix the example and make it runable without reconfiguration, close #29 * [Docs] Add performance data in documents, close #37 * [Fix] Fix [0.0.1-release] vllm server crashed during performance test #42 --- .../source/getting-started/example/dram_conn.md | 14 +++++++++++--- docs/source/images/dram_perform.png | Bin 0 -> 66013 bytes ...{vllm_kv_offload.py => offline_inference.py} | 9 +++++---- unifiedcache/integration/vllm/uc_connector.py | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 docs/source/images/dram_perform.png rename examples/{vllm_kv_offload.py => offline_inference.py} (93%) diff --git a/docs/source/getting-started/example/dram_conn.md b/docs/source/getting-started/example/dram_conn.md index 6739b637..3c06704a 100644 --- a/docs/source/getting-started/example/dram_conn.md +++ b/docs/source/getting-started/example/dram_conn.md @@ -2,6 +2,14 @@ This document provides a usage example and configuration guide for the **DRAM Connector**. This connector enables offloading of KV cache from GPU HBM to CPU DRAM, helping reduce memory pressure and support larger models or batch sizes. +## Performance + +Combining UCM with vLLM delivers 3–10× improvements in latency and GPU efficiency, especially for long-context LLM tasks. + +

+ UCM +

+ ## Features The DRAM connector supports the following functionalities: @@ -33,10 +41,10 @@ kv_connector_extra_config={"ucm_connector_name": "UcmDram", "ucm_connector_confi ### Offline Inference -To start **offline inference** with the DRAM connector，modify the script `examples/vllm_kv_offload.py` to include the `kv_connector_extra_config` for DRAM connector usage: +To start **offline inference** with the DRAM connector，modify the script `examples/offline_inference.py` to include the `kv_connector_extra_config` for DRAM connector usage: ```python -# In examples/vllm_kv_offload.py +# In examples/offline_inference.py ktc = KVTransferConfig( ... kv_connector_extra_config={"ucm_connector_name": "UcmDram", "ucm_connector_config":{"max_cache_size": 5368709120}} @@ -47,7 +55,7 @@ Then run the script as follows: ```bash cd examples/ -python vllm_kv_offload.py +python offline_inference.py ``` ### Online Inference diff --git a/docs/source/images/dram_perform.png b/docs/source/images/dram_perform.png new file mode 100644 index 0000000000000000000000000000000000000000..89a14f96e9db25b3cbe88fed5133eacf29d0531a GIT binary patch literal 66013 zcmeFYbyQr<*CvX)1PE@y-2(&<2^uUwa3@Ic1oy^*YY4&VBv^tITpM?HZ`|E!TBh^< zzW1BC>;8FX-T7zET21$%R-LNawdHyC*%k9vQ<)Ht1`h=Vg-}&RK?el|1A&5qnuLRe z{0~cnmkja?)k8=56-vbz-2w6j!(LuP9tEX32_I^OiM+@CsAA-Sf^;W?Nhzr4q-RxK_ff>8h=`TB)v^WFG^>b$0 zl<8-i8#c>m-@ddNj;u<3xq!QPw6=Gr{4Du(2!=}Inbg_d)+dH99%gqJM67!rz`Ld~ zFX|$PtmuC)m7Z!NpMP(nRczkgb1?tcMf&3@Jm%jf1ZBU8O`M+KM zw@Iw~xyE!enI#NzvvA@~&|M{l+yFX{$;n9tMa7hF-|So*ZIEp*=!lM7Nk~%kr&nm| z=GUCKkiX}i;$ZAg7bkh$p%T%+clP#DGc#jGi#kJ@nWrFuvW(>94c%fLqj^9$Of^E z8-FeV0XgUUVhAfUv&-nwjgGeVfDJhe>Hwp^I8iG^>-@ku1+5ncM{C`pqyw&sZ z>l(*3-gby1_h%`dvU0*!a;Xfl9kup*C_Y6wU#(bNbaZWONM+W|aJt}Gl&SQb5>g6V zyzAxG!+l;)(`JpcQGRhP4kmu#{wu4Pmp`cCnZdys5TTm%3cJy%v&Op$WyAWysU>`M zIUb4;tE(ax=f!4pw~i%w1%>T}(v_yfd2|0@|opYR%2ee3B6-&oW*F zu)`~=!(FJhl2}$-QCQULR8Uw#?|wd7bcp)!@L*$WYgJPhBriDnj7ltC#sY8!**ZS- zXE(@`D9{G|gr)^bxvDRyrH{xu%82uuEL4@f^2kn)!CbWD5=FVk1oXL%^{2W`kZ5WQJ zzg^YR&>$2Q6)llrW@aiV6y;{_lAG?ITJ$)0?up@H;UB*wPw6ph)dhO_?4^-WrmJ;K zbS>)ib5#|UXYA~KOP(P(c!;pFtUJ-!i*Rb0#+JJ36sW}6#YP{7YbzwsX&RO2k}ob= zh3A9oIyupX#Y9UQHvHlF-ij|Rhj&bIJ!iV=9W0&4%|H5wSysFJ=flKkl>tZ9gxHAl z$^yZ3LR+DDhNVxjgNA31X@&xI*Qe8_(BeZ%(*CiJ=EA0c8*@kW_jWJAwzT6>V2;No=hVBJW zC7)K^QjnzJkdSNu4>pGHEQ*=MQm;SI#(^I2-1eAEFSo9!zOGI}tVu|4Kk?xr675AL zF;ZLErXQe>q8WDdJm~^?DRM&zDA2?Y~n|_N>UP2 zn~9dN6uXH9a>XR<=o^saGzu&#fqC+hfn&1hi6LD#6(nGdeShg9m{{~{xWrviRQ`WL&U<3a(W zW07rjEteTkIChz7+ZS^6bg?va)zA_zGDz&%5BGU4N_9l=$M#lG5VJqK)Ua;P!k%a4e%oJvCMB@ zdtXOqQI(9*Eay#GD<-1NY~V%gs$J&iI4AI?wwJdg^ddZ(lMIBY4ppN27XPCyXc=| zTh=Tt?ND$LbiY^+m@d*V*>@eqg&T9bk9|O*K$q`DANa1WoE~Put{Zs)>=#?ruyW)? z6kdhMcbxTu!1;NoZkXVp2lBe=+zwsMwOF>YF#*?WZX`mH{>Si*zqdAiz3i%!_T3w8 z6?b7eIg{r^&LId@e-zl4NJII#)UL)(jSB@I1nlAv**#yB61dpUPdWz<>IzzKmI?gz z_W2TsCEU>m4(1>HIlIts6l9;Pvpu=>5hwVVABUZh)bq_7{|o=Na^VQMYw87buNOgS z8#4Af=wv{Vm=GD`Quf#giEcrpupQ=_B{&4@jy!(te1EK+xWt>E1Eix zUSs*B@5H-d9G|y4vCSQS>_n~8uAx2>Xo+GmPln6ti78^~LY^XMO@)Pvi#sYQ9`*j7 zLs=O~IwB-L1Kd&m^Hd?|a-A!MMY&kx?iby1jX%Jb4oq;UAIaPBLxG<;67Q=EJ!|Tt zRWNk#=ix{NOnE#1o4MM0rpQGJMe=+_Mvs+`o&M#1FkvfTf8+*e8qQ2;M;+vAmL z{7TRuYCLjB`)7uz&n`BH-L7C^g*~39T8J^47~X#X{|}zBZqSJn1w^r$9*0) zu4{N$IrBkC#w~dCCZVm=I_PC;oL2E?5#-^fMxdOeT?RDmqR|bwf|9B z8!MWqy`tEEhmNna!5VVF)=M-84vQ`>n3RzT?ZqSP9tQU&f0h|iEk*2I79WyEp)nvq z_>-)hkeT!+^2+iWBkZ3IF7&His*H}1D!*NZf2gw{!S&)ZIuU2?0Q+n1dLba6Zwts0y zZBZ2=BGAjW6%N5~z3EhzzUW|eZ9V|D3VBgi{1R;EVfl}6k*J>Fv&+4GQtZ|j9z+l9 z4ju=9U)i~~oe;sVj~Gel>%sp`Wd^Fr>$NjS85A8Ixw*Kw3W|!Re!Rxs%+D`d{f`16 z|1Omy$NlfS?TLRQ1#({hKX7P;XCmtyPo9^zTr@fTzPT@JQB?S!=pu&Gd`=*H?ET%^ z;8(@Z3C4IHd*>IA78eg3jDMb2{?g7yPLcgznorUzKcBPFBjw0hc_nsyqfc=O#}=XX z-(WM94#v@C2aVolz05a2BBpKa|Kk6o+Ri;}tYnO2pRJCUTO1ahW4D?od3u%JDT&dK z=zJ?%YewKBr-Sw{*ok?;US!)0Ix+Dfr3BXQF;B?HZ&)2At|mx{GCjem9p=-)2=7F1 z7k62z{d{U?thLr3V#Toi>wjXAc#Kf9qYF19Jh7-cu8|Cd8#!7Q_Q9nIyO9uLG}-uH zhEA!`beKqJ;^!wpO-(&lr1TEhrS-k`2>{-T^Gp8aM3RU-Z~a%-xo^pqSbYuG`Pj<4H!Zn=Z=p(B$Q+z%|nw?EC$b?=k9Y z^X4c>aJBp3SE6*U)VltOfL&@GQQ~e4p_eBasEm0xHAl**u}F@ph054I8i(`LrHA1D ze3$QS5NQfz(n$=xB?z35Ue@`#`fN=z6L%&}_c+PwBZwiveD-O#ATmg*mVvXyT$Wz> zI1wUaTh{Pt&&OWTKnJYAMi2W#G&Kc0WZb zytmC&GhY^I*)F4t$Th-_bX`6lrmD2qZT`M;zuf`ZcXhb>1_tc;dwhgw1$QPhR2SSZ z^dfS7NJ#{l!Xx5Bh`~j@zH@*iP{G6!H*>kl22cfG@rK%`152=x^K-oY(*z&SrXb@4Tb&?Dox)x~ z3d@r;k>*L8!_X2^$N4dr8D(TJD*yc#qy75h^FbO^V84Zi=8s`mpn(in5@?aDF{R^N z3)W<`x0oewel6d>&JSiL?F^h9$$g-2ISQMjNZGQp2wIXuStu8q~<*9=$asv~?tb9nP-FW+gdHDQ&DApH^CVl~< z6(|5Z{CM8_$)vPNRxVF*Q3-DbfBb#SkKU9?d}XD|$Vv?maH-A%*WZ!$hAibhGL8>u z3;(o^IS?ng|C5;<46&2`2XS&O(9IPIz9*eUILBuMc~F0jZA&f5E((na#qGIa z2o)*JuPcpy-igdznVV}WDMi-(PT4X84vFpjDVpiUT|{MQJaP$ssFWI1mA(%6U0z=J zH}mizY72us`nYFhr(D}Nk#@9*DS_gmG^&(Fr*Ui@f*G)o;w>vlT-&9~9FYTb%Q&BpTR2mQei6x2Wuc!j+Q0XmJ{|iiupU%tc8MvN1wIY zft?Gk9~@~Yb4U_HheVP{_yDTcOMRs8TpE(epXtm?RtiM)@u0`4yhmm(-X$m`3MpG- z{}jP~zGTqk&jimJQhHQih4vKOY}?C$zA#$&%I@WQ3FB5MXv$ovV**Gi=y3mLK}oZ$ z9}_z3!K0i2JzPDfUGYN}W@SyJ;q!d-8tR)BYT0m_J_CD%>dhjxJwm`<9lI})QBCI^ zRT5URcmFIGtwAyDP`BfsoM?Y8(nOZs)bl6z;X^u(;*}0FNE3|jl1tGioqms^eelSD zFM=LiGAGb`{rJ}!&bQ-_MlcaRs0WjH2-emfoBwFM(6Oeqe^N)46z#2_H^tiqulGpH zpK?bUp|Q5r<3Dm*tRRUz5MR zgl|<-*){&0VsP^Cu>94?QVZ365vMme`llEy&@1x*f1*f{rDiD`#X`>>1EV(+y4X}0 zZRV|oianl(t|v5If$v-ldkh#seWj_O?d%oipVpES)w*cPV$+K$^y3C~p^6qkBTki? zyTfknR-HwTrnqJY3!RD>WlT$_m}saguHv6Qefm%IO)@f&uX1)$n0Yqx>~=;yKI7bi zm4&;IrM*NajKK$n7iUhK_s7~=GEz54^3CZ9FYuB247SH7`(THfU1G-Sd7^?nPokg3 zB559^0X-ir_B%>-ySM9PM^nw+_s+=Tbm4h|vwq=AWRic&ja)4on_?Yz4kipERy>TF z;0#l4AE5--F$d;92uS z9eAtK{pM9`nr(o-A%^lRZmw`8!%j&u<@B&0DA^IMi+te-j zb1Uu7HQ3f<%Z&n`1wi`Yia0J^fB1R#ICs z97HG;&Qag7Kq)5txKtFZ8CV85qZtE1TrP-V8rA*M22SV>mYMWl<|DQ@4uYw*g4^Q5=%qLfZ%N3MIo^sQ^!sveSa|~H z=IKXL=mpNw)Fd7vx!yG)*ELvaq+bR3e)e z8Hm&szsipXgW`gPL%q1-?Y&S@sfFtIgd%yZVRV38;P1d8mg0bEE5bk6^va^7Rz5DEF;5 zl%R({(Ujv(eNUh)5_xV|0JRt49MT6+t3%^FiI$68SuliB!hXPOlw6H`IDi`z2?XR^B*fv5C!7u6c&u;w z_*`D>(BC;Z=e6AsrQ;{{PhSs!hAJDywgSyBJ@+c7(=U9t#5*B-ebWtajjF4yVTXs7 z>$-DZzT#*0tPJ}Xx^7C|l1+-|9v`6>z`bVOgU<9n*kT2LzZ zmgN255GB|v#kG~eGK|JUyNW6z4j-b89uZXOk+Id8#*=4>`@U-S7vHe=1IR!%7yy(p zSp$&^ag<(llN9WZZ*B}5yV-8LyV(|&=49enK|E`#({LOx-L7W30{gT{e=H{WyeH^R z@Y80wdaEsS1Ni0%P{Zh@VuHSO+HLUnh%;d(OYJis04?}#QgQC88G#y4Ko9?{A)e6$HC`K`q0cxT2nRhQ@&-7^L++|<) ziB|hlyMcp6#KD0gckpq^*e8~>)4*SDAlQlRJCkg-BW$9k$UF%>pc_NY^F1E{2Su~a1J8b+5U=3_^=woN3$E=2F82#eUDzA zkv!qg55&0^7KnGHuk=el15Q^>d{3(%eK%kZ@2JH{$x>A{RC{b^1h=l{{njUo7aNb( z5i_U&!Y2Io&Bfw8tM@6(ODmT1SrRK=kLBt)zC35+hORCYxDt`w@h6(5`(?SnA5yANrn%`#0Bbgl;+!cD>k6bfWmSKpCfDpO1i`o5iHs{RK(5}w9Nlk>!^;)yf?zfgGlOZrM)=1{|csy!zPOBep~xzZgZ_!DvGXIfA>zC?lD5k&)6u{F$C1H_HpU z@Y}8YpTDR=3hre@F#CwNR#NM)dcuj_mwQ3Gg*OQTzw6uXIK~jm;nMd#-kOp=RM@~j zW?k}d;%Prv>VYStXlTrxOh<~+m^iY1%+OU^F&g-2eqgDv~oirXwVQ%#4*UwHM zy~FTexx!Kkl56FCjpZT6`%SWt;YQ~iQP0x!g#cbM_8f(?2#T@#J(Kacd9!hXzN;>5 z@d?CtV!L;juVLBie}?<$KYHMY^sL3_3WlcobIgoC6dRNOF=m6}Hx55smO9wv-jbcR zoyoGV`tvlb2&cr5xoymuoNc#1p6(2ZU<<>J1h~qbAnCJ$$E?LRZyWvi$WOo5QHOl5 z3`=`pgI-gDHkQE9ND<}hYHt!;Rc`}=-{`bpD=wo3$!oNi_GkyW5TUAuo5OyM&VEFa zTq|lL|E>>NR^krPN7nv0%^$jRZ~GEXRuwF9?sqRvOi=hCUaMD}i5@uOtq+3Xp_}%A zZgfJAx9^YEj{-&52)YJ(BDtFv!0d3BA&OVNFUUcu2r6>w!_1hmIYF zws1V@dGbwF=eQF<)v&|AUQ<~l@_qwvQ&=)J>&g2QX3{@&pKv{^%8D8TRvjI^qfVn5 z&1&+H4!3e6$CdgTI;iHXN;TRhV!k8}(8i~ST_6F?i9Q1tR_U;#ZAxM{)Op{$;8RB;qa#*kzr#bS7rWB@bRu60lx_ww)Q`1=u4%lv z({zM9{V+ei`zk1zs+qvJ#8MZ)PS z5XblxZqE-axhZ8YPz{~f{2rf zgZ)OcHeczNnQD|TV;sjz!c4Ke)vDV~S%0!!(?sxifuR$q4cVJ%akMuvIBZCj6a0gX z3kLTOQkzkGpL(s_u7^weE_;~@H**YjBYN)?HTJl$w@fZr)6K=v`w&r-6*4P$ZEVh^ z0J3Y9`@BcZvF}G!9!;n9*na1o%Z;aBCa={Q#z5@?f27w+jmcw?CLX7GtfFZr`NVQ{ zb20D^M(Q}^eR&a=9n>KbyBR!?jgb5}CXXv8)I9BO`&|mw>q(rAJ_)uxbJchYhc#*g0^E%9MtmvKQ5+8Jf z1+7=gw~2BZ?b#ma%Q7!m4YrSs$b%K|^&iV~ddn%-2Hwo~w%+ztzBnc&NfQkyD!fky z$5Zu}H=)z&7lEtou(Zrm!iQ_Vs%|OKH7->;>J2!oB41eufB_F&sz$Gk}-22XEhmo_nF3;P8l>u;_JlD`iHT{SNg{^?JtTEQoTzHDqdcq zb#--AN>r5#Gp~xQ`t_<~GA>-h}w zjdSS{fe$*gK3o+&KI+R28x?C=kLFe8k39;=wR#KX^1%wcs3PUxyutH++!&W}X<2eh zZn6kc_nc>PHhGbmE^9J0Fs;eKCYuX1^Pv8%eNwDci;1Lm?T2q-xrWRp-vYzr_hsFm zR@~#@_49})x5$JN4Zdyn7WT0SAamRE`~G{DoY)#Xbd@Z4PU_mea~H2tT6liN(YONw zlgQXfSBuzHC4z)k16zX&EDs1=X}lPS_xx0-WK#C>?%C|GRr=bTB!IHl%C7!W#7R!l z0%aLpb=U-Y0y|e0-W$qpGf6n%r|QPZ{ki#Nux&#L*;1H)rnncDOeEn}eldrTOKv)? zYZG<nSR19ksm#zTlo;y3+ALKh@!87xusEiJ{(deZZZTj;aqYg*zm90NaryjPV+ zyC~0_gk??(r3D>dkQq!}bGKo0+V+7XKlPEh!Uetf#*Z74H`fsJsQrThDZ2h<26B}y z5XOw=TWXq0(7xRPdOnpof_>q|qp{leU&gab#($RFN+SVw-@gcOXff?yp6?!=O`jb* zHJ&8x37utQ63Cpq`;rqS>wVQN$7w9_`R-dc5b!AxneBT^7c)f>rbvt9Y({}fY0jS5 zZg1VgYDC**K6_jKb;s>PW25k~W-|dT%@n0!As8Bix(AK29@|Oi5@`Inm~QJH4Lve?#Yl;=AcB*T4PJ5Nr|T{^G#93zx|Z zi2&VX{zGt{!RDHrTlThJ&twF>NpBwj0wBeZ^^XDulEteROsN89{aO(7{+!4-7RD>r zqoIs|xK9ayP;%XhI(5a3_prDowAk2+$+F?Pp}w!f}4HiZan@Nn`&K&9v2ipNFD_Ik`{OI z&VZ!n*@su^QJ#`S&W@tRf+AaPRQA%S@jIUrY8<$imv7eCJRyr5f^b{BBE;ddIF>n{ z+*@UchVQq1kf#0OR)X>x{d4GhEP%+^IxZ23;HiI-nj|brf3lH-eI-`PJue=4A1a{vQK?18~F zfJO)Ac~j}IQChJlz&YIXs}Hiv*@BRl z*uEMcMQRKp-Z>Bs!tI*_;i6%;;?9=5ojNplA)}l#bOwhIyTY#0JT={?Jnnt@vp!zR z8@yf~CV(t7JFv9Gsu>wn&{hSuM%g@?u4dx9Cz4ua8H@&aH-hL%M02nd^}2Dx8JNkdKf(0ElRdURL8`vzWr8+$SrsXyoA z1dW*@H|XtoEy@4)0WuoZw3CPUQ?rEqLeLTK0kW(ZrCM|p#bMCwf+Se;Q+z{F{T3E!)>|U` zxvQ8hgpg}ml~?ch(onJUl?d2wapJzKaH^gy1AAcnRe`SJ*%Sg!GsP}FRX5>`9)n&D z7g0XDM*1I@^4(>wj-rGGd0XJ_yX%RYzJ(%<4l};&d^hWoR-_*3ZT)HJL~@p`o=YdD z56(i36`MO840|!@m0gNLeRFFinW?-ukxo62!|E*70WJV%Y!%6lH2 zT4VX{qp!qVE33W9aCs8;CCHLcfx@BeK@Bp5`fi>Susw@(ckDjzvU|l!m`;d*DwyU2uZ@D9Igl(< z9X3nUO($xBEz#zTu9odvP2xny?G}~j!F@1r;p=u}_fJJSqHgIllyNA(2OY$vr zpuE+tO>v9IdMOvQ#YQZ6FBWWUAwsmFl4LNn2Pg+VSrvuL@n!~aaEQOZ^m2MZ}X{kO)`DYR}dYzeNLagqC@AtON#ik4P? zoHyH+l7uws>X+@1?TQk+ZfI`;B%(Fv!c>G}orhgkcq_zs-5f>-Xm(&*-xLSQxQ(l{ zy&i_0{iQTFE7FVp^14Iu+5CHJ)*sEFPv^yE>X~v7QJo=C>A^l6B4uu2I(91$*lQ?M z0rd8Fi!I=ONP6(-&5eO>t6%q%oH3@A^G()fbdL&zsDRN?X+*VTn6^ygMjtqpKb9C2r)pJCU?zu~iE1>2CI z16;y>+H*jIdo{ka04!0em|V2t6Y>?neQ~xSt|#!`L({>RtVHj}FFi_V$Mt3J(Rg+B zl%k^!!H(jxQL`@zE^K@YE*0GUBmF%2Jy(^J44hAm#qy6=>Myvx@^%=b#$xR`wqvQ| zp#h2*lNeiO4$15CRpZGuy&v@oo+Z_kh6gCkql3UT8@f?@|w zh5xz^7DMj^w>>n$++JE8gEIMo~X(JpZ-GytwJHA!bo z^c_=GR4ge|OTd)MS17X5soEqd=XASB77l;~GX`^~se``OKigk^8f8@tu#(bEiTXzvl- z&D0<_YX*41*ORhphYV3lc>B&fOE!I}HWidn->V)cn_4AFykr{kgdp>Kqke%z6Qi z@6wsVqcw=SkCee!<}EhQ=QYa1B2Y~|w??72Jw#u5gTJZX1)X&t!PS;;yz9vT(k{7T@m5t;Z_3{*$)pp@C!S zhmFvVdWB`ZAMTKMQH4N}V)Z z%U&N8ul`v+87*PtGdj=r>X!RnpR+vWGXKP-dQGz!Nom;5+zz-GhM!fOajvN^9^{oH!UxM;u_^ZlvaAHLLrr+2UiDi}^$>)4_hA8tZI^?Z+wU>?rq zip`pnN{=6xpyY}A4aS)?x6QOY3!Br}o0&Ok&r{Ll(qIqz{KwOPWAj8pWM!MZTTn@9 z*_ubF7v15swtHo8+>t;_o`FXxjW{d8c2@t|`lOTAhguru?TbyWwnkS?-BJd~g7AqZ z|0C!z(@kb&KPK=+Q;ae0y7{QdyhYzl<*v0KHEuqK@WYAJ3-N&QU%X&E28mCc;uR66 z;=%nTQGF?m2>dtp=M7f&=RFiu&~tYR#QOm&r7#sWy2^86Xqovny+!aNy&{|ZAFQO! zFN_-BJuXQS=S0Gt4aI<6^vju|Fj~nRU#@uzZ;G+k>H`b}SX?|jThG!HNQOGjTVj^0 zpL9!)usAV|38FXZK5nQUQ`~~pJRY1%eh&$JKpwbYFSs6J=awB?61{$?7rOd^taT-e zyB+CnP=1Hqw{p)d0Q>aZAwlyt+xIFg1?s{S?+?XpzyIv@u!`>De}$$5ciCWqJT8b` zF;&23$#@;1?E8YdO6EZaN%3os?180zyCc&XwOGyY1KoDel?~}FKy}gCl@@?I?iLPD zZQoj!BYIo@M^Gd6{@&Nx+8SAHjPv`P7zH58^KEBSU7dw5pY~ud;9Go(SpK;{Am%ME zMBBtj4o&dGEB_={&m;Rbp6Q_#TW8~jFE=eKPgYZ8!ymt-xhE6nEq@adZNM3@T#vAN zR9Zc$H0m$4>CSn0`rLy%tMPa;{=}?*=wNtp16%Ui9vZVs%*)x%$D3CjN!l^#$7xR> z3#Auw2y&r^qhhRxmYKKV1q5R2=t@m;l?T&nO4L7vE3!KU35V8>Y`sO2x}=g#X?+2{ z0yLE&=}=UOgX7HdK5Gk&l1lM}pAG))+_6{sHMLb}D$G|AsN2VP_cNXMC9#7hZC|_a z4xOwKd7gEx6=DF{8gYHJNl!+hM7YLEBe$7IoTACfOXfCBY7H$JJC6}M&^a+O#@O|i z1VZBJ4}Kh+d%2j%CVEVP$*#?@uln504ad*jb`(w?fo7 zKA4yP`L!Y#^?)flx_?&WeBvIwV6vq|R0!Pp%jC+F@er|M{F|TC!8Z10SmI!W5A^8U zTaxV2x7$Bb4e1)vs#;jXup;; z86tZC_<&We>-C;eo4v3Kd6oNKW5Z&52_&2OQt-pXZv%ESfBN7|f|QT$tzDo5e9-2i zxa^Oi)ULDB3>Cm5MG^1H=!2YO?mo*4&y6lEU%z1bRQRO_N>S_O)0~eadj!+=Z+Jcf zHYy7&$rK;5X_QcCh_EF#=ulz}YgwUEVP7T#k;#WMdtUo{zjXIoRrF5Jl*!Ic1Uo>< zD>WrQ`gt!ky~a2RvN*0G0(MK7z1Zeu@V6YpR5Va7h$MUrhP#(58UKC!8Q*B{9%JAe zBdMG@m2DN#8IC>p&F{B2s@oaAtWK8jf_;{Qps;y4=Dz127t|>NrGm!a-0;nNO&Izs zl3#(`>}$>+jh+_U3IDX9bP`XAo?w!SD;RskqQ?wG6@yoWy$s0d)>&%PCq#5au9C1W zrM&zq(?L8fxA%DuY^&x#4)G#$$lQa&5cP_QuHRQKYy2 z(*C-tzsRGv642e@%GFcowfE&p;OA>kjQAl6iD-UwNJ=XJ1xeuS#&pnR*+{3_YaQ?H z7q+M+q~pfT{ln7be4`BV5CSz<-X6Dp_Q`a8EX9h6=@}8e{w8Q02|QwNwRI*x0Hf(N z2{;mtM`zN3+n;^FyLFW2>&tp35?ie`gD+OLO_|+8{&q1!>yV!1NXfUcpE07Gka*iu z<|lG*wV_i&=U{p@JiPxjz-EIngpWJXJP2F33Ly5W#BYkolYSwo@h`4z^KzZzGoAdE zq*CB?BY{9U^mf~dS}~PwH}t0ibxXNg3s|d@j2k z6WLj!$Fpdz<8;Ew%$J;;rBGieTdt4XS8J0we%sx$&pmxT)qa!j@&cis?4ZeR$U3jl zWKn0!{9f6TH#~+i09$0KWlBj}HwcBt4lsI|<~QGluoMJ6lO_*h>Sj>IxNS7PbaeV(S=QC5BKyKY-lFndYKu z5jU>0K;bk=kP=EP9X`79Er*1Q1*fr*d3S4O6SB^()EdVG3_08Pd}E%tBg+0uRF;9qFKo#EUfRIF4p*W}EjLLY4aI=7Wts; zS}LnbAIBJ(USvB-RNG&zCDrCA zAw3^C8#ynqCDt-IQagl2lU~KKxk7@iw$5f77&KL&np!4>Iy5w-qN&*@n0Tx}nLQP= z(Tyd4!~d<^un+nWXV~U1YFj7kPb`;UVFts8cuxxGnh5=;u5oKXHY9@dZ)Kxy`V#i^ zZxfmppR_ISN5Rl6M*&5}wCe#`&+p9ct^bxMKI&UK4NcA!?-wNI2s-23E?Ix8G#zV2 z4wBmv`^8awFj zZhyVNN|MYpH~YfHvpN{r`pWxkEkT4oGw!YTNZgp9~$xj^cBu6>TTtt2CszEJ?X zCnoNc&Y!R!omnQUCs~Wxs%xrnrZLFE`Lg2<7TCxQYIov^ZQj^YO%`4%^vxocp5_LV2jk4_WPjVT|l~;kr83&cll>H5FZo)l9$AXRn41 z?{w|BZ&W!Yg0|#Y5KELuN&6Q+t#{_`V-yt9SZHgZiJDe5&N?Mk``{Rt0_e@AI6qwW z7?7}ff~!e;3p<()=4q$bG~3}X3clc)y^x5f%?esj3tP{VK+~9Z24q`xm+^J;f+&+dfJ{Yz5Qo4P^4coxZTbeIWHCm&K>vhzpuuj%^l(cmwW%Rd*|c)nqmR1)`R zyUyqj*tXb+ZF0u-wm+^qonuGwCU<44R-?40XON5$xJ^w~;E z43W9wxnazZPp`n8t6(ng6C}{l)y>w^T5{h8pCaEGaV74}q-HS;Ry*mUCKBZyB!AD| zC`X>PT&mQI6aQnYs~r@CN7HC+mg6_}Bq!=2kaKFzSopG1uM1S(8U6rikQ*pR^81f}x_}8V|PC=Y+|AW1^ zj*9B<`oBStkdl%HMF9b60ckF&; zdj5F+e9l_1hT+VqIeUNh*`K{%Z-OAr)!U$HLp^6(nq>vu#pkf6 zvL8$x6ej6Rm4{mB$KzLZO;_1l6BvsJ zm(o|o305&N5hZeybndP>xs7_>qt)f%n(;)JbNtxZ)R`-xX8q?EuDd@R8E^z#@hLiP z$Vsh$ihk>EvADq-+B)Fe5?J*+5O^KO{4eo%=cpg%Q@N9q;is%kSbS4zs7PMXyxF0_*MjyqEioFb3MJJE9YTrR_Jm?hg`|7R26p7zDP;(XA`(DfkZR(2 zS><;=u^Q$;Y%nyYSB0(>>PLPHA%3lMq6_nysop+{^I6p2_B}@( zd8Go5_m6jN%? zau79<`-;Q*R*Q9;VyIqNnkDG=tybH^T!F8I_uu9+o7z^8l~kxwZsf=uStvI7k4f2h>^r%Sb+Y@jXjWTk9teiBD@kgKzgDKq(OSFDlCYZHyz&)7@VgjX z!ldAeZRG(_1(9)rkGyQ(ego=ET=naDi9`0V{29p3JmlKYPs+@V6F z{rvn(&1A9c2%aqR6t69W%qz0CYTI?IEtZ)D;H2J~$DNdAy1bTAv!-RE4SdYI#6a!E>=U`2oS>x>MyD0?nFA96`B@3Zm4h?aXy@5Iyl1PdxOehP_)O?vTH` zzd5fd1m+zlVu>nkZc&t?@n+%r1$r+(Dt1e6)Z2*jeAs??CCYRCsI*nO)OUvkOfH1@ z7XpQ9PrS$sha0^GXQeh+?jzN7-O}oo9uqhNx^KRei;8p<>aoy( zc@Wbc>4@R-xVi0eTjTt-(Thwc2SnW~2bf#cJNw6>2f_3jGAHgiI=nvxReoRD$70%F zbb3eS{a7h`lhOMe#n|~xzc<@)Ja)i{9S!mZK9THmYhlyrmWoLKU4)a7HwUI1u#y|>+f#_AyF#Wyh*qeR%t9J_N zDJdSJ1LM=N1SgkjNzL;$O!uzK-iCA)J`FLrYRr-8CAYk`BDEUNUo{?v;M|yzS5l zpQz4SWHXms&a;q-@!Q@s$4og7c(TZTwxJ5x75g);Vx!wiF^@Vu0l%Jb>OK$B&R{_Ban_Cf_aa^xq7{FG3A^9dazBO{>4nf!Cf z*%xb*z3t^neS5)%hN9WeUxeixpFVZn|Jq__*zp(zzF+Z4xsZ$$q76h*Xbu2 z#~AjWpP%>j_w#0M?mCyI4hz9xtTZ#_h+It+zQVq_QX|KC_URn1HwDQ%dY&28No}Q< zH+AoxYSy#Jnf}ui_r~0h{3)2btv9_yU&59N_VUJWgpn^GU1wx|P9=Ntzu|!11LdeN z3z%)E=fN{l)Tuo0^y)$2IePHW!71=G=;1lRTc)mbA(kg*ijUpD{`S*)GLB2{&uXSS z9hG)d<(DaFH@&f$RJhY@3aw`NcXRC!M*8{L+3(?D!AuR68bfq=Jxrd<)$ao)s|6RP ztgkRW(8gTbsdog2wp_@6`91O<&2mA`H$)ffJ8Vzf-6CtmBFDBm*mFIq(msy7)bs7s z-=-7VQxMlEt|^i%LPPN*ARzb=V}G-J3|MMy=|enp%gS)1Ee$n=cIYEh*iXy#ZBTtu zB&qtSF#+JEU78V>JV1mwgp?1$Xvq3qBl0Qk8o7Tq^&JQ=dgqRBzPSm0lHUm;nE@dp zB6T`cx!58S!LLZ!4LcvTNHlq7^RoUNZeq53{ zNAdDGvhK)Aifv*z$97 z8YTSyS=ua_{o&ZyfXf z5;^KuecLQXdq2l#lxDwpR*p89;5-EZoknc!-J3+55JAGD;>|3Qd6o1x?-Io+s`BJ% zK+X|0=-wBUvl4_4mKyv4eN|j`M>#^<5=3bkD+(KLBPbUAV4h>7tyjR8b~>uDkoK6) zj?2>uxX>bI=2s0eT-w;|vqsvEoL)@IEO^oEV+IXp(OwZ(EpQdYBChv3?PdeR8ag-G z-reT?t`~<96ZI+!Z!RVem}y>Rp)TUs(bd)#qJR{OM#KCw!>hlRmQUrCVch6X`XH_# zm6dL0CC81g!i64kYMdi!z(hg`knOx0rR5lX~rH~3ZB*&r5d+`@%o@&``$;++L2H_-yx~{OjLD)AFuq&> z9zo9NzCT_$NrNnh_Jol;cHACv_hldZq;K#L+2@z(k9cyi^rO?MPa)Nknx$8onmWEg zA9zjPtexMU+&^MzwjCSxIai(LIZ$)fZf3#Hwm|LGHJmAKV=rq<(>nQd-(?>E$avF; z?d8QKSGDSi{KUtp@+tZazoN!-+@eR6bq=(8JoM@!P-pFOWg}gsrwJCJijFVsm#LfC z`i5wq7JQtYBDwhtu1?k@PRlM^-9zWzIg$l(vYoU6Cgc@J|jjlX2o=82tx z@*K;#WcYwQ!s+R$s00;^d^ws2u0i*p$q^<26Chd3Vk4bo^!w;YSqBfB{ay~o!~>A!#ory1n>TXAOWcXcUZVqdc2UOWF(=mR z{FH77mcF*7NPEiVwWQ$DR87icNwok_FF?&Q#j^i-ur^K-fVG;=(Lao(c9o^s`*jJ{ z_=9Epu!HKojo7T~;T3){3z2D5Ehe0yj%ic3PDrkyGt!Z~ao2G@hoer=mB{ld+{rQo zn04)YW_Xfzf|CnzVj0NlJ*^v1sBt2Mph64Fs#k)l6MFGiBEVDSHTjVre|bF@1DB#} z!K-?d)fR3xo;k(4ouFcQIZM6B!&$zbDVMYd=7P-Qdc6jmk23emIc@=GWZOex@_ufZ zfB(IKrw+TF8{YPJS2q41A<*^q>e&j4c6UpbiK2r7WH9ubz7;)^ zJMHKFtE$FdNiP*OcGsiFv757K#W9?I*qJA(J^O?iS?zon?T=7yV9a**Ya%BDAbD=0#EY18>~TAo#} z8bT&>nnHHDQi?#f^d_l}@kV?x*jz$V0(O#N=C_8BQ)Zm;?K1s|nJ7^b#1}*Al3@Ja z(W26xgEfEZZ-? zj5oz_`m9a#@;m9p4#7ZM+GW?dZagAZ@anD(sX2@fys0MW5 z^GJ^}^9ePT6OE4E+kw@?oK82d=Sf=y0dwq3H+jcnbIK59SO7c=RhPC%fiFLAn!3p& zHg)AE?c4_5~e+) zbl1b9)A8cVyX1SY9|oEN>KBjx(2oMKOwZx8(vyJcd3xz0n2$tqiCFbip){RsAcnnIXxhfs@_uwzKlp) z#7}54;4Z=_yfQD+HD&8J+AiU%*$E5kpVst=Ix z>>*RWRCcl(mIZ6Ob<=AYV>X_b*F|@yH+*FI#aL@N9W2n}VQ`OAb?cXnUmBGw5=H(% zf`eJl{bj2DBTkA?2c}!X^`Qug0x?K->(Sg4K$HR_?)+>pPG{RpYEXr}3bK|5Xe z0dgUGZ^W^nv@7!<(Ey*tEyJT~*9$>Q@>H-Mo+A&yWwU!Hm#Xr6A*YEn0OW42m{oR- z0r#MNxXyj`EqljVGg4MF5Y&XBTJGgcZYa*>$yR4|YOv-8czEga7gSNNfv`5RCL%zE z|8#0DaGw!W4jWxb%~`r5Q-6$3bWvg19}vafs}BP4O$}~@MGa$w29xaS+HEUd!TF zqZCK(Q;P-<xVX{72%xerx!XFqPmB1FGIHr`q zm3hCAhM)^&$gXAnD6WBtu7Tq0fM4!fQ)H!q{Wx_We_;rt`ec5d8@jfE(K>d}e0)G!J~#PgO?xI~7DZ zEN1d;jPd*=W2v8iNafY5LDu}zf{fzgVp~rJmb~J6Warc}^(+Fhjhhgs*#;L8t~!|Q z2S{^15q^(DhIwyK(LEQ9?)jK>Liw!RnJSA(Pk5ro^`i5=B}XDKLGW~5x?)~)lU-gk zGK$bVRK3=X+ng)@bg21||4M_oj=#dcM63q$aNMO7(qguN7}{Tb5>y?jN^)E0xGBQ* zrJJoo^^=B z#QsHkdTIH6wvj!)m~p#_rU{4mmfPSXd)=c_=VZ?2oWaI&WKEd5_Q6J1Mq&ijJtyXY zkA|^=dfSMMm+bY2Z5T^T7SQiL_ClJ0|DCSd@LG-0#+Did>lq7g|8Nt0?C-b8b5`bX zF5RkL4rIp~^}d!(G*iY8srTeFmSuw)Deo=iNN?#R49{z&b#C>vKVh96C+=IrdEmvb z;Sa($<2mXnu4}D?2V&ScK+5g_fN&&zTSQk>RTa0hJHw6ee!JWdIkd?vPaG>}VR6vP zL>oAB^R3d|1eeP59qkux^aOeF6gw2p#_Xts1c!}*C5aDJx&C^a@~j>3l_3A1o$?Jx4^E9aDev1&t7MPK*-}!yDGDJ$w%y@ZE)ooB8VC1!Wq zob*|*rPxR_nDO=ku_V_+sjk<2AmELpFn2a9Q3(T_^NaO|Nuw!k>R!xDR@c^P|JLn3g+NWEIhM0MZ&{&ZqW-@bdrA^$4{i`#*k#=zZ>O7Wsr-`eEn z#&NzS5IYK?9n16j)F*rj@~5sosaSrUFAgBD^as(7p(g;(W)%}jO{*A42cu2qNjwpv zoQ?xMdcA}ykPrVED%uV_F&MWj%Pc2!H@^NS?tQGsAnP#2*-nC1d;TZ*n5w-5av;_$ z{wT%xpX=DAL{KKkLYXV37ODTF4F1j9`xgU#jGC54lRGN8DX@}ZsP7-VUdG?mBryoy zrS_SZLE1IT_y@8<0)1w3{r^A|U&;R2jwDAS%q+B!{9zBiIP8hD(CM8`EqL!#S_3fY zfJO56--8&_aw7VqK>A7g{feZ)jCX6-2Q2e{K^1@KC*+!HI6v6{z~^ys0YkYStp6Ji z3dno~`}6!)B%DZ|V8iW6j#?l8p_2HObQKrZKWt25aX!pvmQLYkOfz*%Pm>qtNWFX! zpDfCE$sRm2ZUfk0`dzG;K>cBoqt(^_jaPl8ShOMePZ`~-9aL;M2VSMG7rB7Ao&vN2 zU{WSv{AY5?8kp4j!>vVkTzvtv)Z_~?0;W+wSHYb+c=d}`3s%lm%&e~?^?P+1u(pRc zqi|t~h#skdxRN`WCkk%{NdkpML`15Fhk&^!DuDEbw+E!5{t$H~^D**H%8Z>~F-{$B z_>X1<*CH9a=n_@e{=cSVJ~eYHHrYzi;=SGr$jc0#lu=dOvpN&}M?2uREieQE^s>YU z$0pZ%zdqSS!3a42)l26^dn#S#MOhJ9VldkCKV-_N;M{-KJSZjbx@}5!IT#J9-%fLP zS!oZtoiAes_y_<28d&hC|FRO_$LLqBhlX~n+N?P6+VZB(aHa!@VPNrR)voyaBDk6( zZI4}=!Mx-T-ta$of~E?-j?MQ6uRU5>p>x3YZ~=JR-}NQ=lv3(anmd$Ik{U%sv$aQN zb^Zbxvr|u0558%e0_pojG!)+{B}GLaF7jU~-awSo|-Agwl(P-&r)05#T;) znp$73B(T6XS59CH*#mKj%SP7-i*}?;@8d%rl9@1%h10DLG@ZG<>)4UTFI{>yAWkW>2 zs@olTS-gZuhjZMC3SNx3_Lpfe@Bo?Di32=#KpJ3b5f^&m9=ybh(}{UJ8U-W?Vy38c z4QNqTA6tT}QFBf4$jD?I9I9(}0m|1O(wFF)3HpaHtv@do#6)jQ+-}bNrVFGF#I`^1 zokJc7bn8|hHaesu>x7v)M9TuS*Y#4{hOsJ%I~*i{I>77sU|4#)sL_R#6SGl!2dwjS>GG@||X`5@7 zFooe!x}SI*wQLK*t{)l9tNu9(!1Xo}R<7!{<0vOqRtMX&d;pKpC?_<2i>bjd{uLOv zUu)BUWB*}0#hzpo5>x|dGW@ehHW~eQf9ynl*8VGqh^i&*!WOcb8RMo{f5V{zs|^p~ zhT;9JD4Va1TRzztP$seMdHS%kpe$uW8FWob`@dTp^+YE$$%Btf>+@9n2`-w589P7x zhp=olK$1OkYxzUtii*;=G~1}3qT`|d53n4fu+VTy#f(y;x@LCj&9w}zW}V-nl_k26 z4%Ne%l3xZ=36sI*rCULM{!0OY#kmO}#Ne;2s!}QHNchV*CiV5wgM^LP;Q#0RgJ_4= zf8p{4eY+vbUmSjP7Hqp&UL2ayk&4}bZux)|Hp{LT#inOG9}5RUXR3_}$6FtxnS72O z){m#hXJN2D0IYiK(0Ok70dwBGHOGiE}?h;$prb~Lz93_FP!6pXLyZRF>r$k z{k6RS&ztR{x!>A#{M};Q0&T%ZY=3r@)(cq)bU%8{SO5T0$^)ktrGK-u z1ZcH^0K6A7D`Z{tEk0Rk|B&cU%eMC&a4tu=JY3|gwc;k4Cp_e|3Xtfw6tPiJo@9QN zfB?Pc(Rh9fAba&DO!(hA4;i44ojCx6Ck1m0>I?L+5p4rk%_G$<>9JZ>|BoXiav(Zi zfgBkhaKa6=mI*?qGcCYx0xnFC_4^{lD(2uC z7IGaE*c*cd-#wk=%fJRC)hz* zsg_s4cPsjJ^T{`qHDYVVb$rxjv89*y<}Clj9>L^78Hs2l^25B=Pr4bms%+QY;)}~x zJ22!sv12WC`?kkltYLLz9z#~{siu^h4H=)Ru!WP1&bz(&H|Tl6tIyzB&_K*gXW#yG+iuNUutt6(Niuq?!3a<=oE~;e+&8h&umY$zDcjm8H@uB zFlb;5?W^hNn?m^PU$%URB94&x*`WosLCx|u75m(#8Dxl172pFfx<0`8cvdUtKeT`z zs|kYe(2xNP?$qL;-w<|AG8@Lf7?Z>T;=@ai>A!w&i2H@`|?L;7Kn5%KBX} z;_wHW;AwM9OR5xkld2a)<7SXUuY^GW2*&^XyYQ4UZlBM+1ep{pgGmp!5t@xZgi z-fsg21-vgPHkJZ%CwzC8w0q0;a925w&6AaCnlv#rrQs|g2tk!-FAh#PXPGfY^NFP- ze($==g4?r-AaPSxf#MCt@p0nf_AyPW*J4hQ&jaEHro?lF)jw$C|L*BT0jwy2>1sN} ziCfo`CkA(0OJv*sQywzl4gguA{i9%$8v#+m;dZq1xJ4`uZ*ETnShg%+?OjQiwEJgm zUOT#nKZwxUejuUgrmtYXJI!fMo*%wzR~E#u1j>4WF+TIVqGLwvP^NyH|S2XBrb4y$gfb{GWY`CxE;Nu?1;|c=3-8i={IN76?1;nos)q7Lu)6Z|=tt;a!kem*~ zKax>+nX**M%kB|aIiF{wr+{LcWqJexTvOBz-iLWs>u2`d_cxFM=A0Nh@O9q1^1sRf zxm<(E{moBsgc=^;J8MjOXVdooNJuD|w&M(HDEHP^1+kq?D9jZq6My_>5vc!ov>^!yw{Y=jG5_n~rS|y{svK3Bpc`sMR@r8yvfW$8gn`#A%qwuY zJ2HxtsbP^_l($;QwKnhp^p3XkwlqDDtgnmCfFs z{MQ)kW{M8jupKJfYf$HA-eso6xEAH@F+YrJ_c%M?6#Bi0k*9@8)4&U0ZbiFJ-AOMc7t-CeZWI+&4h`E|m4;^zJdH zJfhK8H-%?puExhyB!W+Gk>V7fDIX_U??LbVZJ+4h**XXzy8kufmq;%6hB1BV0oRxK z-t$-xcuqTyiT|awUKjvicZkKxTR*vcO^m6esQ6J{J`(U#`ed-gRs+h$P3AyddpZ*w z*(|edrK8quCNKNx&I>t#8*tA-oDncX@FZ`2#gjTpU0J-mFdz0pHKEFJ$7j_>-C4%} zPM^eL>6ueg4UIv?! zmA@R`T=2Z!*E%l?+DUghIkxq(;yJe)nAbi;Y-45JXF!X$28UnFV;aaDh2qA1g@0l` z4ApO|tI8-QkL;!2`Ff7Jvavdn#fLnnUTc{2e6#v=z`pVMI`7S`dzmCUJh%1idDtUT zqRZk_Z!q^UB;u@9XjMUor*E!C*fa;MsjCHsnuvGgh+-rpc$GPtd}TktX2pivGe@-_ zB7(UekT~#zLjv<{s2~#}j!Q2gDI;-J5{@4ZAo}-c@aFl3^WtM=#RGEhT zBkr5Y>>!dKUMKkTq0J0a7NtG%R7!Q=}sTPXnR>3H?P(ZW}MoPKS0W_`RC4lDR_tT7CMT{5UG-#+)_rF@8yvJ&Pj zdR926dpJA5O1+=iiTU=%yjSZpt}RQA4^&M^q@`un-EVIpX3UlcM^}NG<{|lVczy64 zbnR-NrM<0}?0QlBaAn|T5iDQrWcmDkM&t0ZppciuAWPG%ApCKTzEc&R?_3X$A^XTW z0~54M{3sh6k#9Va>x2u#E#6FYx5}Bn1#M^IF1{T|9}-XibuJ$ziq?HKHvxh#$JOV+ zO6!U4))@mlaIlQHV;*1}N3Lh)R`uJ=?CZs;K=wjORQl#8_qKpeg7Sa>_v15(a^GIs zMG1%uZ19%uI|CAKl+8Hspj=(}S!Tw7cmaYle|CoWcCI=xwzYF7a|IaqHNuzzUp?NJ zGl5+(y;^NT>n4Y+LR6hcond z_Xg~`LmC5OYFbjg#42pKQIbTh;K!rY5(a(Gb|+q=tsn8l`ap!a%J7czez8-#BDr~J zzHcX9&aF_DiU>EvA?j@ASVtHF{fqVuhTHoL=bll9E~xszGN;yLT4Keoc3#8028_|qM12ELH;S;W;H%x z%tw)f)^tu;Awna<8#(+kX!a*J)Xam+Y2#yKH)xMzkv%9MW^8#VuCA6IN< zPTX5W>9V7%^M(@;ak`rZSP$eakWX*@va^j=><(@%(eGwk+_{B2eHM+Mr>BGuv>sm% zYpThqlqBoDM2I9FQ^qHo+xR{CC0icnX4zF$&k+t1A`OwwZwrb68{EjZ9NY1h`4SFZ_ro9Cz328I&78H0n? zxo;>BUsS)W;3T;`3M-9|+3+NNfFubsNINMXp03+v}=h+xDtca>jn|?Q#^fu=$V} zrOR=r|A*Hs*7nD7-GM@*fr7K71aSgSR4iI1eF(JE&ZU)9wfL)84xRTR!nneXeOB??MQK?~P{9?jK12-I$%iXDOi}Xvi zQO&-o30K6tjz`ks;!ko-=>(bQ<~R+izt!m813hcB>dt@kU!qb{!Yj6^)aS|?&$MM9 z?bsCr9;xlsEOM2>?;4$?=g)n{zuo6@Hs6g^HJ|nLT9o-59HeC*Va0nd&YuGUWJI^e zAYGGgRTjfuOhc&2_gd!rc)p|Nbf_`YE5X+>#Yd?jW_vNkS9g8q{SU-M ziYxNDfnK<0tOV?wff0joceaX-g0K3a|IP=$J{Ey)#$yjL^_M$cedfu}Aq(`|SxnW#z?KCgVaA{9FJ4c+& zWbRuo{D;EN6mlg4?SqBp_-h^4!0lxiCYS?VVpCqwZJV3+9kTR+BKFZcV|a;7u;2`*6drXQ&?Msnyi84vrL?v*R<|^ zKOtNn4K<%n$qZ`MqmX~DcRpUbR|oX|DxzN@U3;GJQPvV|;XBacwSag%F#n@x@QH73 zzF-YEE#EWM(=!a{*<5mZdDigDJT(@jD}PWRB(fW$b+dpjeXYtA!`5j>5=tp>g#m9a z*5m}6q`M_K3*C_GN?QkRIV{*808kr`(ED_cC^53ozP-uhVbl93Z+YxC`OaOhhk&~w z%(}|T1Z%d%EvjO9C~K~g!sG6gjQ{?C@tqDN%u2w0j7A`5^>|55xu08EaHRkw1fAe1 z+7YsQXKH&%^aK_<#D7)>I_-5YkozuF8mU8cr+)P&Oh%Gd)k;s5Q=8bypWS_dIIewry3fkH0t)=-Z1QeLgoCd$h8Tqr1KgZRu~Mu!>$^ z^my#!dpD~n?Q~4OmU&>I-jwhYe*UHme$1DNH-2KZ+@@*WIviS3b8)=^O^*-fX_3tUWRYS1H8 ze^4AB_t_K~3%j$rAeRJYjoAE+)f9oM)hTgjZRG^E6L;&n6IES}Ft1aPnY z3~7y;x;hCU@l~jD7-y!~o_nu22rLZ3qH4V&xAy!HgZ(JK;xDM)C3=X3D#tKz z=CZmBJ-g+u3W4(a__&v*`yMGRxm=;^xE*tt8Mv`pZ6K@}YETZN6h3pYKXb;Uo1zo7pah9+d8DQe4x#JIeq^{xhPn=Pfoo z84_VjhxP4zj)(txB5d$Rp8`pqQX)@4%Kmani_(3AO)!P)r~IS5!PjG}`l}^=b~%r@ zd6&}yx*LNfT5+%tMU?>|i{SaQd8E;c!*To|l5Ye3iTqa^pjp%7{@vI2RhE6u58H9m z^v+HOdSzA-R$t<_hSq)PmNJg})+HB@=lYeK!lu(fkzm2(Q0@Yv*Y}Q{tlr+qP#yR9 zJl*6$RRuhgRJ$iqr}vd82n(AJ$CvPvt7=z=K2k|=DajX0?LLr>M_7IFs|!hm?+Xy7 zTj@jr5NB$mqv=`T;wZaZ3?PyTWVB1e^4**YB#;QL8+ZO4V|99Pb+-xh+c5t%P6AVQ@h_ z#^h`&u$k<7*4mWV)VHy>{1|BuCGVBW1Fvrg3%q4y$?4OdgF{-}!9|7rWI`~Ww$)!Q zAB9hRT|NnmNsghtW=SyR;@p)r9PjQ;EOr$KDH}pLjO96uIMfZfayDaz^tiTK$So`1 zvxIPWX)-WSyC4d0(Eb)f9=3dh&1|>v1R-UrD-T~k0`0N8OZNvY(53z2V-54Nnk#1! zTQ+wb(ESO1=^X#{7lO6*n9hL2;Ahvg0D4<0`?1R})HDa2MHD}a>FDST-GPLX723}ym>o^S-%tYKpV?MC@ z-WBq?N%V4*n;3o~`1mmo7+iaQm1@ycOJJTNX;E1bKjDjcZ}z_Kaz|8UmE?+&ce4D* z!-{eV5vIjCzr($Q-8kp|R;%n>eS632K6Qb|;%$ayGuu6i&Gm^(xFD}@BaYrpFdwnk zwBi#?9fH)Ec5{$h%v@yM`7$J_D_H+@f%8u1E`a!LK_Oi;R4;lhXsm#OT;ghi5w$7U+p6C0M8gl_x1{)`~{HG&dBRv|T8$h~4Dw|>( zht9aOHAtMa6i6sEtY^JA9L6@>wWaW2=(RET5hkn3M!BIisk_r*&K&)B(jm>$iC?eGtO2!XJs6w^sPp^#hhL0IhWPi9M${rb*$Z-0+9u|`C?eYeVC2me!9 z8t%o>6iG`vxb^Y=`NS{|)hE6?tkCtydPE1z@11cSm|H`s&`l>Gk-yv5b+s>doev9# z_bKe)X7~0Ii&`vbhfCLE9T$mZA41F1e?&BAvUj)ztgi1Xlr7;hXZ1P=qIBF}=b78Q zZ1p-sJDv>c`V8i9vf-Y3G-T5~;)}@8M$JSO9Uazt`ODD6?RsyhK3zaG_*jL@t0C@L zo*Uj`@8Wckza%7(cJGzwxy{pym7eQlXflFF^RWfg^ zFwfK>gWS?3e=$v;*|B;YRAp@@N8_Rab|X$@`^hN0HjkVX5Ys(h`XRsU&}0#oW5eBC z(L@sIwP|5l=QnmkWdyEbu5ELtb4i)7@~U!pSlr;_fbPw`#6CSlX#mDDeX=wLE@1Vk zq;Gc6@y;|IL(QpXZtLH>Ln<%pZn8w{l)wyNe0s2u#hIx^-x@5KkMYW~CVa}^^g?~~ zch9J$x$8O$Xtf1rN>NKi<}MO+dbDN~JE?IZ67=&YcCs?c_TW~%L~ZW$tQps+;YWU3 z&!!*t?;;FcRW$sd0eEiW7j2sKv3B<-W~QLFd^3)fW5QP zqDuuFT2*>GLO+?GCQeIq)ocP5qE05S~P2c8fDslk-8Qyw9dIk6O#z zC{sO!w9h0U$Fz8X-UL-)yMv+o-*KZXrPYB~TKP~jy_3eyi?uJ=7^mO8-aMhV_^2%= zShS?u_5fiqT0Wv^KEImxXyC&qQ?y}^2sq!Wba6D-6q1Z?VH-6w;A5 z3%q)iQIBhaGvq{LT2!?t7al@yW*h*a`uyZX>Fidc8>u={SNk| zbtV~c=VUD7^4IAIZ(dc|_^tybSzp|c$n@`d61+t}3_6^%%UvaeXHcTi5@(?8rbOABcrx@%@<+A>#Fh@7i^@5!4+V#0gp z<6jGyUrTKoroz&+eHEoS8>}x0iEefl>uv^gi7K~GY->xkhso@hSJo|28-6%?!|zw9 zHex6J3g0xHFd*GsA%qc9@RkJma8PIrma9};txpoTIIs2sQjFwSN6pv5R>!lrG>!vPqkqapXH;lZuA{mp54Tjz+7Q}El8lKrW0;9A0>~7 zazDBVLglLUtWEmVt$Nf17oOu2$UekR?|SR5Rr-#@~L%I zR|kPVhvpP@9aLjL;aK#jJI^`&`R*D8zmv1ukv}#f`{o3)sIeH%&Vec}Is*>)GbRDjk}_CMq_aqJo%sPA!7eo^Azw|*U1 zi`)NNa{!$m@g`?;@4Mc1&TN4 z1~iC=b9xun5>OZ$@|^*^efdL}7~4vus2l2iRI4?2w8co_J*u?+PqL7|dUy=lMhr>OOb9gtYo8U@A?=tw`FAuiURF+sz7dcM5T|=j$$; zt;@JXulL*g{F->|`$XZe79RU=M0C*jkql+l9_&?72?vqMIGa*Tlnd-r#e(nIswBb10Uu_i)!=l2G3O2MWJ}0D01j#R+^n4mI6V78! zJ*`db6Uw_JAIa#%AR`(!w)24NOy!Up4nkpJ62}qdH{q32Hl-T9T_N~8D~?<6#$w59WHAAkRr z<$S@VsSIpX?qOY*PqEfb&*B_e0@Usqn2fA^9`2FU}@wjQX z+j$`dUHRxS7hnVR%tFhJb>4c6vxB4cU?262)h!vr8)R;3G1?`0_?)fQZML3D+f{zB zozt# zK({pGs<1fJHRXGp&Zyr>2`?0eZ}NS9IQ^;zfj3ZMtf*y`OsJt&qZs)*m{zcnPAm^> zw;@YE8E~6ln~HWZxoUbGc(u4gu{W@^*pxpN^nZAJ%cwYlZf!IP36h|}bpiwn4#8m{ z5Zpb%-6es+Ex1F1JHg%EgS$HegEI^+gAVe|d(L;(S@-w7>;9S5-P2uFUA=eJu4nIh z_6$QgmbH+ZijFV+85lJ zog|TaF8=W^V1BQ6qpuA%H_+oR($-=Ucu72hemI9w3Dw#6hRx9t=pGYZT(Z4_=zwY$ z;=Kgpx4qPs(hQJ5#Zv@et>h8-qFh^^NsZFO?!o7FA*t3B>YH3;(Xh17&FJr|k1`~j zFoNumVCbb4U|nPjDbmVe1b=FdbMC5kD{bgJAfSl3Z?gK{7?M5_8A z^X_Pv4GBWwDisvevbq2|?ZH3J$}}Ii=_uP#_LmlVAojWf?t8C}i?`x=08acxZ;?!r zY1pfx`9gq>XW*T71;tuzfS;W}KXOg2xVNzD{qtjXG1%0~Ht>Yg&1v!2X9)7B4AHNN zZaYQbb>5+QTz15Aq8M+kwYb{=43AgkjjicOLpuxM5yQJ}=s)vb_2!Xs+uXzweuut4gB4iWV(Lz`)QRF5 z2+)p{FaBzM#lX(a1~otH4^j58o&H2KC_EBn_#D-U73~*s33Vm`txu9r&BjN5H*{uf zYu*GinfLCk#q=46*Vk#*IG(<=snSv{S(y@;n%us6&B*T;Gx7=lSm<>#yz-5e>5-V# z;EUQB;5NXJ!fZ|gUl#plJMfJkw1<#c1np~1S;+?cPTWDB!r#z^KHEXDpM>m7b@xEa z5%=pGfcW!@bWFd!$%OW&ERNIjMppUv<}mzz3z|Gnbw7dZvDrJ&DPA~4 zZL(?VnjzXRSGBF$KT4KN;VHaoSlIBZY-U!@&&PDH4}SV~TFfsq5l%Om(;E6iCfWyI z*zgr%CtiL_L(oOiKZxJ|cuvVlFe{4p-J=QCBnrU}x1YJ(l=@bV8_$sursSKk)P+nj zkVRXs3s|}R+;z$p_?{GRF;E?CX6O@3k+KGXxd+4D{RW1(*A2tchT?FGy8D}(k4sfG ze3?R-m00%R{V(%sy0&e55Bwt*1@=GKL)Ho9?S}gFxN9ZAm&vW2w0T=%&7XHv-_5{duVE7?8_;T?2C%kKhO}!D`L$zSb#hYy}iBBVN|{Qsunnsp5zFnI=)AiB^!Q24Y~P(I@+(1a zxF>uch9?qaJ=xW_ZHm_7hee&n!@nn?+r%%_1U5k~S_>89JZ4&v%Q= z!1wOM;gkV(2EU|H5rhQGV7=VJL1%Sue98PWVylXmlE|qJGYXNhl6d0q2&ypbA>bdy zrl&_8mhIcy+ifT`co;%Rm|#=cnX)uhZ^KsPEyt597~Bk(p^(143^#u zN}ozhKDluUQ^)*kG|))2?G5t5MD>1H!As@{>7Eujt4%BJ)RDJ8XHuJ;%&nXMtqlq< zt<}Q@h)h%-d!1uo7pQ*&SUkWcSNWRVdIh{ZJWz;S9oE9#2H8L`T$yC0%$6T4iRyGj z-rh)&xScAr|KJvrKE%xa922&o>vrLGu0M+x&!3TXY|8d)G^%S@ zt?4POUMc@eU=V`6VwHR^f3(z|TBq3wKX@N?j5^x({1XJk?64#lxL0#U<|&8OwZNle zTqX_lnHH&BvzE$<2#jhxInOFL;V1i}=N&>zJ}j|kR2@1;QAS6k@>}*I0zN49wXgFy;hcG z7LyC?0;(o-xw_E3Lx%C{Z!C=ISef0#$C&jk&{u7s!BaUlFYN4pqf_B3MK>?LRV6v? z^$>d7Ytf*=2dg?W^o3?PgdT~?iJ0*ML*B2^rf^HE$u7HP{66cv^$@rp;46rJJKb=$ zHtwBy&jazNv|1ksZ5+T(Vawe6swCiUO-n9x6J{Znjs5_>Hz_Hbtt8w4P$=2K#R6VE z{xciM4J@Y(Z2>i8sT-csh;+R&FWu1C<5bFG9w6CJZp#XEKdx9LohxDGjsM=p%lk%< zzwz`#!+qmxG_rFZS24eWU$ozeRrX&vinm=G4T<1fZk=X*oa&&vr2jo2y15n8_Xkv+ zcrlrHc>XrxZ!gc@B-_wL;fxy8<6G?b*M)EvyX&K&(#!42KeXsvb2YG8_A~5*d@N(U zMY`(=JkV8Jk7XG!r%};O9Bh-CB)uS5tF!_|KOY)_;eVey1Q|7KEOzqsZkaY`WKZl- z7NNg$jCD#Q&S4S_!R*XT%cZpwR2h@NA2ewZy@~Boj|l|KbMNn>J7xjRxV;uc0n){k;XD^WxyZrurKJ5CYL*f1kN^ zFkg1vFaj*sHE#HXz#STx^z`yFZuoZ8+ttA_Z`iNr*lWUzPXhcc`gF(h9)YT^>hi`p zGraak3no9XsWPeak8L96MG9VW?!Zqfxg73#+P@VqyRI|Qv45oh z)(#`uf8`nX`EycxFOzpxoT5h8fPrEbqde<(QcDgm>t32QO(MgOP;n+fNA`sAi0-9K zJc;dMCSX4_J@!2jUPSA}t17Ef?Xk75NXnwl6VnMz((wJ&nZ6Y*C4gvni8WX&a5X+v zL)YGY*{ke|EMe#uRX2s=nz(Ct;_{rv zJ%}@)Auiry7k)tH@El~ipFFXfVMhe;ZYOw2>4m$%`n$CD?Wa=PjK&{%c1Gw%eM811 zEXlWvt@ib&$R0R6Li(843|pg;4TV~7_H9lxbE{$PFi{XimpfcqZx}cvUB<_UO{Z~X zmNm9DHtl1#;t_P6*{Wvf8l#VhrmOO$cZa1=!~M1U-RI|YLd7?(UD0h0hXXm$fOvx~ z2BzeU$j6OK_Zl6D8g6_&zS=SR>!?h#<*$bZ7`?1py zEJ>!|WXBhIfG4_})rH`yr#l?irJ`T27{`Sq6f45bAFuSU+EQh}W{e(LOj0iHh zbdbo-qU@IUuQgWRIHskmSvbQXAF6IXF@x^Ru-3+kFj`^#;!SsIL2DUQm+kuTW@@i* zhRlYn%6WOYN>)fcq6@;g7#ow*8;bY0?)_>WgB1Iehjq3DRG@8N$cp33H*4~_a0trD zhfnPy2A$oYV3-qyc+_qQ1YG zo*zG=v?4cxxd;gmu3!l5VZt$@jFx`OjM4Ao&x5JN?G#Vq2h#tDdZxV0Ih!wJXPhnk z^y=@C39qyg(QJ#QsCXde++1q6o^93~{qpmJFChztJDN%EEcMlJ1<@%#UoeGme z9&A*XBdI|nz(uXI)nW}f<-=b{Zl2fI*DoR3ON}1y%oFDcGDH+B-l%A??0LSONS?{j z1qjw(F3^`&l;zAE$ZLGXsh*TIuUkIq#SLoqJ@1X@SczboZ=h|LFx{>Zw@&man$hvy z^Nrcj-a6b3RbTZbzivqm`H0Q1MEt6^75@l}*Ia@rSS>5!HkYS8 zN}tr_#$In|oW5H7V+>81T8y;>nWK*R6OzW);GecF{mk1rC1pv4WanjHetbiAcEqd` z^KD2RXz@e&m4)Z^?1jy77LVIHe3c3i>gIygy@vQ9{gC;3-z5kHi|BsG2DMa$5z1Am zAX3Q!V1!f(Yr8aZrALXl&V+soyV#B4MdmG4|`@6{VUyg95Sdthb?q zKP*A~TJW!5L5uVf+PsLn7DHteQ!39C$be3xcT4c6q6rqdyCRuMwj$|lUl977b3-GQ z%hlUNBFB3vxcI4gm-6R1J$>IUo$btyt_a1P@`!(bIRXOVJC@L?MhX7oN4sww0sG|3 z!Exn*-Yy*Jt)r59jIEga)yw626>LSda?)K(21vJ%BvWx^&)svcN8dg{WMMJZ8pC|4sn7?t^P+C<@|V&~3n$`PEPv#olPw zn~PkR8^$i%eOctqXxpZpCVvH6XQ!w(;Y_doR=$*w&{R2G#wttuzPO}CA$bUh#rgIx z#AIO)V{!TPcSX{1de=r=KhBxWw<@wUMLP<)yMUv&lH>)&+^O{l(FeAN4xcazelcxu&!KY=I!^_0KUwsASjv%laD4 zcppLSQ7KmXwbw_|6vj8I_BOQX!$He-s6NtC ztAv3~v7#uba8v@K$fT_DER#~~F(PiFNJ4f1!0sy1`%3d|=Vr~rj{l}kN3Tt{U5!y? zg)|Js!)^v2biro192d*)*Fu$ElF^aczxsR@7)2$L=5>PL+>YNWVHFtNqeN?%z!1~V z)OtxK{P|DJlcy*AP@0_bTLnFWB2@A6r-Y0smEX>wdoqRTG1RNL$x+Qve`I2=L$7tK z_ZXwexU8(O5h7>qfpI_8iMbD!}^U?Cml_TSu64yT+hTzRL6^E zRyD-<&|$NbUYlGipDf_ao6Jky5q?BCcD|OrtR8>oo2gBtrAgHJXJBrCVZ zrLA8lVB6m9d+Oq-*t9|Kfruk{xyo;_X@&^bLMM=MAPEs+U+1ymkNw;4EWD214}w!= z(%Rl#(QZcTYVB4@deav$xG}-=V3I7{rB4hgH=6zJ5HVDfE(-o1~yCtHq){*|)?= z6%=$kj6Si!K;TgHM#UU#alt9uX4-T)1pgxGu~2cs@^9f7F&0qgg3^q*j7S1K9z4AH zbY|kZTkqavf`r9c5t=w`4RruTzetV#o#&v`HDJ8Q?!0FZS7tIx|NL?~#=D{YS5>1= z|L~=xyBA`sjL+E!tkvF%y;)L}^?_wd%wdnZbEP4vSReBy2DFm_3aS-sq_j?5PNrO)+x>v9}C#9H}paT3Ro{T=Z{w)w26_Rr}ua<{9)p(H<_?(@3f zAZf{I)}`dgd7v=IoX&y1<0SM*PF|UcIRU-Xaq4Xpq0g7~gggfsixQ>V8NziQ>bz> z1l@lh zw?T|6cS@B>b}=y0Q)joKt&^l^WE^&0{TR&S5mVP z4G_t@$0MH)kCH$LcyF;lfb?pOK3~3Mvg38s^ZD&%1IKIBEg9QDe`?D-TkE;&>}Rzu zMi2M7KSYS>@-)2kX?Bx&E$!y`@mPkQK9UdU>;DywJ!ll0`QoX2jpY4WXs-I8>}0E$ ziy~bClubp&w9rw*@zsi3ZsfGw{kdJl#ia2*>E!md^zY{@w(Ejysdcs#SA%2$r>)QK zXxz6Gz>KAa%<_QI*TJq^a@IsePI4GWbMf7QCg@maw2%8ru0>P<|ET&TZ*sldNhAeG zk^cYC)mqP7Q-?hHYS9)b7r#s4FDjJNjT7o=J;C(umdJiR#`dv}*azYoV}F7S;J^ur zbV=C_HH`%>WpuRTQ9G4ZS>pZuz3l*F=x8$gY>ot4xaKqU3aaZr4NH1XYl7p)_F^`{ zUdwbhrxu8muBcG1L|0CnU<^!lO4NAaUOVJ!{x+Te}xhm zzV)ufj7s$D2xJ>h%JHkt@N{?MoMj~hL`#z^q;1QxD&onnS%g!=dWEZ%2{0gf_tc$< z@$M(9?rptwXh~-hcqyB0{#1&V)>Dh)5XQ@pm#4qYy{~LlV=E8+G)@jZ^bA*<4o0s+(Ktma; zs!sS@d+AI^bXkmPvwWWk|83|L`HD$t!&HmcO$Z-3*oJwj#fAI(o89`W%U-+EJ@}$m z4Jy`7vhov>TsTUv^GAfD;c?E)Yv2_<$R5o7Lxtjtf%cHFOWHfh-Gm*g&40ss362~- zJwmETv^_9X`DedLPHsKA1o(Q#+*@&FaZu%{8x8}bIbUfy06cIYyUTWR%Zky7Z|}6Ip{{0$-Av?sg&#>R_uFApDGdto2Q=?P_Ij0n0JyVj5wBiQcE9+n?x3-s#!owUalv-o@I$qg~Au z+~lYm@Cs~P$+u~Gh6J``&7(;T4FTmu_<|jxU;Gu{br7kG2m(+b1}o`-_DE}~ql05{ z(L@zkf9}lwoAJ}=Xh{X1snuc@wv@8-=~8jK#(-uWp<$qC_m>rp+7)I(4Sk-R-pGF& zMwo)k^y$Q%kO-tb_76<#nAu}iGYw>H&f#2E^Gm!KVcz%g>9)w}ghV-}yS{F4q{wnD z5GPmLIou9jHXE2+qB12nL<$L%s>S-5c}P6Wpw0=}N}?SAvN=#MS?n+`Cyv{Y&O| zX~eZZn|^ndMv`ey@?WvRzV3+9h)}r44rFk9e}}hJ$(l%*T(d*~0otP<81-ow#zGc<6*Q(rFuUwu0awU} zIv$bN?PhLd>jP@lENeF{r!Ej>n#nC{j*?Y30~Wldu49vX6-D}DFMta=Y=$L+i;HOK z{Q=7mk)Ba-DDRWhin}gj)m+F@Bi6o{kf`IqLHX?+tfW=1v|refeawQ}{*~a=gvbP= zq>qnCr?2n6%G_e=6oTyRB4DMA$NdnzQRX%-QND-FJo}0Bd-qbO$nirOc3Vx_0g^-I zpII>P@hdAwJ5%Mn-MQ~}`yj~cH)uI5^a)C_O{GOgPIM4YLl7^s#jfIyAE6-wUH55n zNW=N|z^581;)+@;Bq-=si(2xbBT|wcq(tovOUAV`l@e$x_nt@tGs3fuw5&fMxe13i zy+^bf`EJHD~5Kjhjl&rrrX#y+l2gj z;n)5mvLptHEImAc@vYc(^(rd!7wq1c_ctkikSY@Jt_mGCLoz+Y{l=Bf`#Cz1B_dyC z(D700F|_tOYYHnb+*3sEJ0}?Wba8S0rRwHOksVUNkrx?*hwW-{l;KjRWzyeSSEf|W zdv#AE-FjmcjZ679ck>2y(Jbng?K`Xp&GV44se{t7N<|#$W7j#E2Kzo&?q0C^!=vNC zrY7}AIcTs!)#3q}MI`TT%tTub1XD+UD1x`(&%8-@NK?neg5ZcnphY3KDlhZZ6@5D# z)mvm|_Qi-v;e_26=Ea(=O}ok#djSdRhU z9oE1%0k%etlBUmWZXXBt7joPlASXzJw^W+`QLZ`_rK`W4yrS^4F7X8vOrcjvgiLj4QiH@-Om1>XOV?AF!XJ<#Q2#`<+{l|}eJ`K|E%aHYZgOR*9(8+8D8k!y-P5g1*Cv!{7 z?c?KUr6OlTI^+$jAYWhK+xz>O8dK>TO*SM25)!HYX~^=QXKi)J^C&1>n2^P9fz29p z!lR-Ubw$q*`e19n4ZZ*)V{wGjr0|`XHilf6kkx z=TBMCu7IFu8K`(!=xmk92N%$=RD(Ph3SOA1(OT2Vi=9 zMYhhnw}Ozz!?N~TIJ!jKWHr>nV#0-Z`DSuFMd~?@NFt)c$<1gGo0p2&Y$(5rquh?# zvCZ)7AP)(4F zk$?WK=iWlppHOo1x)M{SOnk;9o@C2Lp4Kx5Y%f{o0T~Ah1HhZ}C45@gF0?VaC2h9E z_V#v^Tcd5UEG_&p#`atLsEDt?fgY<>jo#*$CkG-{@5IIRCGjWtad8u!c=$qmmS5BF z3))DJc4aTmb;Nt!F!Q-$)z#_q_s?kp0YZR>H=R?;!+avem)5vlXaC!JLn z1Tac&Uv6TgWegc4d#C#e7S9m@JYM5_;QQIayNQ`Dm`v<@&^XSgn$_d{=h4Gk6RRdq z>dzF6q%IVhU~bE9Y^h4w?uGQxqUG+N7benIU)qF^Vex-0Cs&arb|qm}0pW);1CQvH zZyReq9Y-K$i0Wzg80v1f>#0wek;4|3Gf>x0G-2acx5C{%Mdo`cP8$tB{tMn%EQ(=v zDg-GzMO#*T{?*&Yimle#5-sldMjQKgKkKpdx1{iF!95Qx&Z$CfbkeU(NC_8%ZFkDo zNujp0YYIEuYUB(iUR9VMMetjTH#Lqe}RPR;LE>{|P5YT6W& z`QP7b)S5r(?M-A(CXE0;@>d$Cv-MrI4#gphM?jP!TKf#RA%=An(|t|2Wx=@X%zFls+O(Js$W2nBd(@h*`sy(!wA6x-ZJ7};pvR)!v z%I2(1Mg75*;OF8egr8+b)p(PNAEWZ-U7PR7>YO;^%3#3y$7!Jp0kx6LMo!B*cRO@W z{O6up<>X5UAZxGFf34**@pJw7$`l=&KQQr{U)1kiqR)G>|FrifytO%~5oT#X|PzdjKe3acL1NpKyB-M1m6~xrT;8NScWoEeJ>ixNJ_o# zEEfOKu?V}&CHPw60aGGhnN|F{YS&PKt6-mErV>7ExvH%f@9Ot{9?M$;{anIy zL~U3WHJDmeE)u?!mXjqFeP6UIp>)j}1999xbxwbPOe7 z;iH;T6_*zldeQRPpOTz@fTFB71+4PO*7a8%co!qW0F*X8Wa^q^-} z2vC31?^8eZtjh5Q6WPPmxhJ~Id4nQ^e%ql3;VYYV^2 z0Y7-4hauw$rK^AG{Y_?2cgb>VHJsFf(wc?yi~+3HJ}o_&9U0SU2-YC{Q{)so>FS6} zAj+Z1^x%7|(Riy2on_D3ZEQRt7fWYz<7{2T;1ZYq>Gj(urzI%LzD&m8fs>qDj>Gi; zx!3?k14`Ru?nCH~YUnurN??#?v+h+CAZos1<|?BB{>H>d0=6nGuT4r@#Mkb6?aKCh z_Fv|#oRlfoc`YN_pUsQotAFN^hz9kl`|d)FQUD>kmlw=@JNKkd;aB8MYu$9k8M{dD zcFD}Lmi%vL`DU8V67AXj)Q5MssmSoT^43e}T3<=$g4x@;HM4tjT=dWA)8Zvd$L)tn zAeHE&s8G2(9ZX&$~>R8$b6V6MWtIjeb=9kR! z%ZrbL!ib$?esPDovH1*0N7T*wb4oPnJP5WHsxnC2RE1z+k7}j@?i}5sopyAEKU_cV zug?RZ8DIf`Ym79wQ4{yQ?mykE%4iseeEnD|gDDHuR=dyo^79t^spYhXm1sZ}9^2V( zXd4G$W8RA3=rUXdCX<@eNL`(NTG+ zEIJ9~?!rUl%TL~zmHYTN0iBc>t`UCRCx-giLQm&Zr+)3}dbS@5O+!nL)|*n{bV1EO zE$_~9ygV2f{@gUN4mgKJnCXerwb@EfljLTMJRqEFP~GyRi)S5g)4E)^>2xjsa2Vqi zXjiIeC&?Z#8JcW){S9WUnBeh)6&+1pQG1Ii*W>Q$89)l>=>n8_1}Y-hqNs3@b9H(6 z>UeuC2(ae&z@p4M{5SnVFWcGRhSbCOC;Y4iYMOpwy?!3w zLvfNASA;bRovAjq*i`Ey=nzf(WYOnP3%ACA=Jo2ysXKW4 zkjzM)slJ-tMEEIK;r{))E!WWvkxI2O)ulZ3!JT2;j@8SH4@cc4QSWM=i-e=KPmX%? zm8m^7Ug_hs_k}@nR&R26_24r(gzZl|KA88zAho(#_hWzAF>toxz)EV=;WMM{+h3*L zY;JxTvZ`V??o!cY{fvlPa_4K4fq}=64*Sy&Im&eM07{vgn#f_^rX2Y$pvFpDMprg? z95}`h7Ae11$J@ocwj!bVWZo(Y#IRq?BjGvN!`fw4_K5MvX;=OptJ;Tpw52TD5$??v z?8Nrds{c^fWBr3Kmq_AK{i3_kjPSLkWwq-b|Gj8yNWehH?+d+;9e&sBAdEbj{j^>r zZE`B&Mz}{BnS%7&!coI?%8<^bJQb4Mg5TMb!8yA*kh+R;D&*kR&Qm0@A5XI3__x9D zManJY!p5kZ>4h97E1pgVFPwpg>vs&I5V&+){V-D+SMrbjOzG65J8msj_g`mwjm}z5 zcs%#oQK$j;>*K|$(Y>!nW(_tcfbq8pR#J4J-GB15`;?vlfCZm-I(BeGpg>ui zVx@9K*2CcgPP?I0tZL2cqkDYW4q6Yk0tz-tlm0_xr}e+t93njF^@~(SGwqLNmgVMq z%}>Iu13xWCmOs0Ez^R!3!}00mv(GGP(&wt(982-A^udgQ&@SaS^yfW&MD+X9IOYfH zf3BZXh7B&!|IixH@R$8MAgi&h$8R^yhFi27v_y$ag1I&U@`V z{}1=;A`z;2I)9q#VaR>%)P<+<;0l~$Yl-B)k;t66Ne`84R?38T$!(X-NjdnUHUxQO zh03=fW6La5IY}2R8#R7(1Z+8Gu3OC4{akcVDJXF2 zYnVy5JZ^(6bGrd;xibnUn@Mjby`$Vh)`rWSJ4kp+iRs??jSOWwbaWaUiHB)%7Wsq` z5NhCuXlOb%zUwXQH$=6SxFeTIojh%9+V zNMusU@oX<)P}wr#7*T_R7Lih;&lB}lF}o;NOwooU)LDv-q18o*_gK5Gcd%ran2b2F z!t1O57sX|(XT4=t9uW%3TTnxXLP_m0_gok$SU;pnFlIfCYP^E#8W~7Q26)O{0skmb zI5<00eQPoofN3AzScT4G>Q!U&u%cSo8+$#eMIBU3OzvuS>Tt?cztG!YO(K_246!Du zGwTXHkJQlW)oM6c?HspRZ?SbD<0~nj$JgX%l;}gT>Dl>i6CXPH8I&&S{f(7`yeLcA z`oW>1$c595lwPGpRgsfqE+0$Dt#T7Gp%pDV+yeGZ6>Q%iJ|K3`-(oMPs_HRZ1-ly* z_8wd@5^%mtaJ4;JhkllMA`(q?3f;YExI;nrs}1JL}QzeXFRp|;)HHS;Y3QOeqsq% zgU5~PFleAvznID8Q{=)96P zSMCJcNqld$bzpRDT!V;uPsr56s zmS!^Os5c4Z08@pff=N|h$EcrDv|n2)fQ$G@V^;Z(Tq8%o9Gd;nhO4b5jH~XSwDpud zomU;zA*feJK?0St{A>9LH+lipwQD8QAT?sGvU6^oKNEd7aqtI3H*>*$#x=Vk;Kq`0 zoDtHuE?R84)wijnJQ*l+KQ8|0O5XmoLf_lsXzLv=NGrd#OL$oqD zd&|%dlmP~OcOD|h1H@5Amy8!E`} z=`5CX(h?@6Q?u!OStPO#wP_`DNJreVTP! z-g!X8`lNg;YUjSFbRYWkcgz}JyJA!~AP!=r>-3)Pa*)$=-;6g}cMwT#?D|#C^!U(a z?%1=5TC4MHAUPpO>=8}3T3Y^v2|y_q$>5OdV?Wr`Del5F#a#b0#Y)951m`7NLgSSB zUsW@`^#T6V^{g1N`@8m#tNSUd5S&^+?xhYftpM zOTcEu=NqbR7|X-BDU09*Q&oFIwD-+C-$v^Uxe?RyxUDif?O8w1gYP%fmW{%-E>}u) zA)eT#1tCw-5YCh6qMsna_u{L<=5M~g*!R()%(>l}-~RfzNJ3VJDe@=&c}PP-B)hY1 z*gQxnvdcx;sp+gq8SH#u=w~-1{nWgNs}r_jyY7Z7>u5(SP!e>vEyWGGpdYC43Un0x z+K>xG6Jx<);7Nj^(45j_v5-k<@aTluP|%aj`SBwUG=pd=9ZeyJ>z%Z}l}y8C@X16n zHnALc>}9>~g?ST~yHdsK?eQle)0s7Z#?J$#zg$i&Ykv%z3A+*~8V)&9>Gu6mHr%YR zzhk?Ck~{mruvij~e4~q!-&|~^t#gkF$gJlh7Aqx4iuOAC?1HQtMfY1rt_wDaLG9Nco)fTGdjDE|Vd z%O#+9l;M3ES0EToq#wFoP4D8gUL?k$7(b&=QViYr4oyfHd&D>gjp=>CBt&t==#0^_jLZF`+XH0m2{r-I_aoK|D*S)TTk3{JDR5(*+n&X)nUoAD#$id@)G^a?MY zHofIflcMEus{U9E^JX9#g6n#mw!(8`q5M$MQ8(Lk?u>ZZz{Wam!C~R?5I!rJ-=Q%Y z^^}A8Fl<{sS)6BS7_K6!F>`wpNG$fe8UDiV{yqgULR0|VYh}ftbwbBAnWN8sq<8Xx zC4COzG}U{iiJI`48IfJr>F&UnkoCBi1zH)$G^dx+grPTGl5y|x3K~vUQSZNe@_nEc zHzW~XRel1nKbSB+7MXFI&XPwVu*uVu*gR88I*M$+3_HGvt>PD3X>a4d+fskn&POb~ z@WpiZ#*)67ps(CPp*r$nsLJ=r<9L%l`o#5oA#vdpHzC{0*bG zwivU|tYXfa30)txwWiBJ9AS(+IRQFo0h`&ZB6Il2QRNYhd*VetlIakJAGSw1;(Bmb z4qLwD{qXvR>nz$z6QY(iu@K>vN|M$ytL*#F4L{E5{5WY2zDg4{4&S)`s(j?wRAXuL z3Fn%8nBCykPk2c*+|9wTXQc#XmG9~1V!`xP(FALaOD(XJy~1rMnH?`O^DU zo_*81<3ZEk9-Ffv z6*VV)nv zS9kPy%P-O!kQtzc`?Cx2u1-8AUj3-HCh~6%d)+e}wa`^)_^1u_e25I^Wga2w$0#(l z+mG|CGP~x#&c;_`aWnI%ej08?dmRh8aW2jp7SZbdMlR z9sD-;XiD|0>pTe$!sSbRa`3&g!K{wzDz{@Y*a_(_Z-OTAOW1r5`6wD5CW z912?v3`eoXIt{ZnLe|inFPD&ep3KM4G~K)eSEmVu|p9 z7QeJ$;Co@edZaGpf0&Zyx_b;B2Y#PqWkN0GcdNWxU@0pIexW2F4bhu`DN({cpJ*#^ z?r{6*3CDKT1*%I*?wsh3;m5y|A*pa#o<-u+=@rd=$GX{i6OK?~mEW1s4bm(%e=?^Q z6I4Mhum-ml-jl;L(}_JpX9J_|g`~aMeI@u1ZY|_)szA+l0ywrXL*gmW(3VJOhO!)e z>GcU*<$kMun}mQ3{+aCVxi( z@bn_Gk<~felNO`DP;(RW|cg6@MdS;JgZk@&t z^D|+S4N-%5PnY2JXN)iS<-fl?sJmY8B8@Duj%8(abcPNczU6nDa6fMEN3Qpw&tbc} z0#?-4T&L`IC$gSwIx>RNb=1Rl-8-FQ;R4Uj25b7tug@Qja5(3RgvWnxc3Jv(vPnu7 z-Co=QYWTe|QLI*zZP|Zu>24narBCe(OKI z<~O`K-OYVf$6|A-b+Z+9W?FMqZ+{y0#csy(UzBIhoy3G4u6=X|=X-1n>S@jG88%tc zc^vfM-gSH{&v<`R&hTm|!-O8Kr1ji4AZ3^=#!oTl->!TeKZW~w-lbIwlE3;oJ%>4vUckk*rn~VPR13B=;4|-fIZpGJ#hk4$ zuB$8MwyK=$^?d|4H`L(#?s;q;l?{*ZwJPko+#-XRT8AZI`0}F(i9xK+pVCF#i_*dF z>FK1@d5Qxw`TLm`wr~1$)Vk>?Y4qAx(;U`U$v~`cN{!zc`gy=chP*JV_X_S%ELn|L zdwHSAB^LF@x`Q42qX-=KnTGU-XXugLP(Nd-I&QVJZI>tMbe6wr?}squFAYXK zzfR7L%8er6E6!4w*~mP#iz2LZafGIpN`vzfbu3E2OUEmat4!nMwWR>)b7^fMgj6mw z@r><5*H~1VkgfQYy1rF=6t9a6w?y;HXyKV*qMe?#`|c%V%= z`R#4Fr@C7*l2wb9M;kn}IB7O*S@Wt?+cHX@yl8r_KtNWi*0_m8y5_n%C7I@tHD=1Kdh2Q#%j1{iWjm<3+Z=!TT6gqcHozFWl_Z>{xh?h zs-M@#To*^F$1j5ggEe6#QiV@|;-sELMC|}wo-8p(l%>PK8^3HxAK;`QKPcO*RkYmn zF)gS)x248^X29M-_I-F{YGWH?od!}fbAPB_$=3I$1j636z^*La?$_(KYQ&nvl9urW_X-+_P1pO2G4Rc{kKO zMo9=$P2dXZ{C!yDt(!f=FDy$-H#C^b%;3@r!9^s8eN5gPx|Gin!$qW@A&)&xiODuU z+UFq2`{2jJt_xvfIl9{(Xc0$H_PKW1Hblrlr;^Eg?^lK$Bsh(Ke0iXU5mN zow%%rWSh7Bq@AKtQ6f`BoC?4g6r$hvOBC?^ly1Z4V&~%U))bLtn@YD*ucYqHtMo5I z125}#i>%$A&r%S@UQaJ7mv-6mT+uAsl(dvX$e23S-}KGq+eG7Q<7st&6L9!06`L)| zF54xQ{FbjnJh6C3h&i$CdNv2_^eOUDgqE)8s&HZr5a%^er7X)r64Suu3dY9AT(-ye z36+dmnVt%uD%BX)PSN&f#R4)dZsMev)G3;Nf{uUX^1zl^W_d9^B*Bbp#hxnsEcw(J+tS9b3>d4nJQ% z(ez_+UGY%kr-RLHw;0b*Ld3)i6W#u0vftRC)P~*ml^K&hSRj;N=lhPft*BWI!g8dc z#u{vaS?A=U&ln1>$doV}u;aDy7tWiDU;g}hEI60U#(K>5!*pce>pVdb*s)>vQb*@Lj|}5YJjg8pL+; z*kZJ5iOz14T?|d>y6{yr#rFHwJz|E18#>L)`HC#Z>xDn!eNU_3yD-dE*caIJlT9Wq zbd&0A8f(~d8~Arzq>v6^+@vO!t?Vr4zyX9zt?C{r?!DsK(Ny`3z(DyFXG0hrL!(1I zl_lZ4HECGqu;UUi@T?uV^7{O?#(0R(<&e1s+Wlf5FdcvPY;|Nre=Gey!^sMGe`XCZi=-|8IstTi=Dxzri^eFcJZej#Ls18w=w{%aD&)LUdXd?6=(<9)Nvl1N45hP-#DsYl=$b7xK9yXYm?QVrvZ zf7MBM+f}-VU8||9*@hDozmV6pznJ&r;hu%;U%kE~{m9rg?T8%5a|(zU0CIMEl)Roi zV3`)Qgj76odTh+JMrn=j< zL@S$%e701_{sQix1`v|r^lOqSNf)Mm|1R+A_Gn%FvFr8D1pP?RC6iP(JP6`0TxhH5 z##*71aWiS1tp|Ka;s2mvdo%xc-1UFJUCMtMz@!w5H1Saak8OQ0fH6&e1`w>ve#)EL z%NO$Ir?|WNUJc=qzCV6t4=wJ~Cph<24qy_KF|V9lpL&?`qH-DU?npGX7Jpk}ko2P0GUK#2H*qu*s_nzaPVXb~b2J&Rba8O}O~yWIkQ8 z9qM6EE3CiAex}<2_8xsbcX3iuQqnQJci|V9oqBLmdAUQKdbXg!Or32xT=O~6bI)$$ zdIn>*F8IXmI@#X7YnjoDZnXIUPZ3w<=tSL(ADQeNyivVGEO54rv!wP%HoCf=M9Hi~ zC2-99Q8vwtmopRk!V6Fn8j5|YAKrL>OS=r!^JPzpE$~&}c&|s*csS{2!C-lyT7iR- z49N1uk!_}}JOF~{s8R4c6D#}Dr-+-G?QWoa7M-l$Jlfy)jJ8HtC?k13y2sTi^w)yE||LKQP1svgH zy?xpf;UNdT?)@HEuz^oGV<58)NlmD9QyU zRBeugHsb@1b$d2MA(-PXN7TqBb?+p(1ezs@9+UW&8nduB>r&VDLitf6 z`67o63On;qp8DdNWEBxp2_7DDM0ELk{dOG z*6Mvgzh$;aokWj7{-r(%+#4zpc8>OiZEfO~!(@OsyfW0tq1Uj0*+1W8-L!;ov^w&O z>vnv`fmL%vk6*yAVzKIzi+9b0u6(5yKwUTcr7;`g4XVAJ!Q=frchW0) znW*%X-S?zp=0~sb3G1N4Fn_HK=8nkGc$H7;uk3X=+@w2Vb$3D&hZLX*GzBdfKK=ze5lr`zCreO{39#Xy? zbGW0Skh!IP*P4Djs(1_LnWUnI=Hx@sXoYUf2F2YcU}{A}L5{tK)~u%L?;TfBJj%L6 zz;39!(m}=XPLFWXe|dwX2%CBem^upNmj!Hav_iva`@mav`i}efV}ZgFmQm?dyU0FH zZde*8(YO_vaiEa>=K~pDFP)#8){Q1A!FYd`VPmtk zzd#LY?Xl*!BV4>t;Thu#)N<*&9k9|zLuxvBTA^?@)KZlChC|!94@jMhwHRgf6oq2~ z_)7XZGFg9j(Mo4`b?aG76#i$y&Oxn=Op^t+~je&Fd5QXjL@MH2er`sx+nV2^cVh=hWR=fH9yV}|%;JbbW_ zkYjW_$XcfcHHW)tm(?>gbgtkB^YB%)n`%yO@7u zJ#&#yc$-p~Z{B``HYs!JMTd?mr(Ugw)Sbw=>bCs}J&5oe? zG!#t5G7nTN$XOHM+b`44_;Q)rx`Dg(o4;w*!)zc~?0y5PE7@Q|lBOX1k{mi%fwVtN zvE%!e+FfB($>@INfG#(;^gmf6)73ywf+18EP(3~MSdX#;r^;-PC%9v(5~Z3VM!rBXIBqYD+3I6tgH7o6 zseoh1uW_U3j)W+Pd7&E&FG(Q}Ld$vt;^OhG=F1B0<>D_sWMI zEDxFZN?=I^`d8JiKjBQP2nW5G(NM1G6#8ynT`;hWQFPQW9;S{TZ_JQybJABF)GcgE zwr5kT&o1j|+tuPU)5eqS2g}^oq@k2R;G?EX`aQN0K)170G;~%BEWVvwJGKhc2wq7E z*D~4A!IZ;C3N1^v-+lMxX6|GwY9uz{nJrn5!q>GVJGQj#z9($!l1Am_`Mktd5m->i zFvl~*HJ6q6esui6Cfk&&8(94|r|k*^L+vNK#5_w?-&W#=52PHzDYHkl+@phvb{rbp zI$Hllb?P1@4T@faLqGbM1Env0iG@^bfs2X)__(`X^Bx%tHE&C;yx}q-&6z0J5a?(u zlqms^a(_%ul}I~g4Nut9Chn;}&2zc&7nF~s<;*1HeLw}Gi>KH> zDm$)OL?@Og4y$%3YG?`bDU{E;rzSgpV{ZY85?t4k==m-QJS=XAKFZXndlv&^~d^D_1kzF{FSI1Js0` zN(0gQr}#>iBUim2nht%mHD)oW7QRp}Z}~FEpWOZ9p(>$8kWHapko_p0o@Yl-BUJ*b zPXg5n4OC>KanW^;eQ{FXsRHz@SY_q)0c|p;<5$(rs|`inBneyOI|T0tN<@xFRgi+X#N|`%U99CV8}{TU+d%mHy`4OxTK8C~M76f!<&%NEyDd`sjz1-mhUZ>h3aXYFr`rvLUu?Bd) z$Vn0j5&q&s9|6YwtQ`m60m&q1H{t@^FZ>n7t*;(CI}GKJSY?BQ{B#jsk=av&S_O3? zdj;#gfyYU2-q97U&=hUbG`c@I^Zf$#!+dc)UI>iA(9P7PyL%YJG*1T@ zF!6?g5iNZ1dmNU8hzU$9%~s(@&8{=r&p0Xorf^0&TlMMn_s~;FnHxkifq*g(+^?5F zRY=QZ(+ZR3m?DkAF{9Ds+)$o=lI^Z8*Scgbvn+eb(fzVh=(!{%hIQe^B47yXbVB>u z^3gv7@vPS0PJSDpTvHw7Dgt?Q4YYF~KlPL2QQ^|6>4Qg3s_WnpqmGz16)0`@cCdC@ z(XPW{NmsX_KMzm>@=iKqvJ5^=0q~d05)a+~B$bU7dA=LRLXCPsoEsh(n*8{huXlAt zP<>nBb#Z^9&G`ir&bez$Y5GA(3zj~><+$@5RJ1BqYM%2HQ zszLq5?B8*;k2O7eaX7m>3s|refjJK?KQRt*g4QDt{JYg>sp$hJHc#)2)BKe@5}&KE z39CN7NC%vzmGSLR0a#4KO*++Sja9=j;fozpn>}2<)Q(#NbQeEL$3YtgFNLj#yy5hw7z?LhQ33!-<<3(HyF)45!vy3Z1Lk6(_ap? z?QcnpqbGlNkACQ9tqTQCO5SvQSVJmXoqpvt!^-#0c9@=S|CZn0gA_~9@#U11vy|mc zP>GeM<8be;?*YELP4MQQl-|urSD>gycF?X8AXZXQ7hs9? zKlU7Pb?g={(;;gs7;l8i9z)B`9K8_Gr6WSgDkzT{P|P(bTx70gAxU1bY%#Iywbsqq z$G^nlF~g~tb#3=a&04+*F_!55q~7kxJ09ZtS7DIC6J3 zeQ#Het5D%~FdTsk?kE>%R=8XarCsPF!B*{3$96DvbD9-<*&wFQ=*E%8=!uNJmZ29z zPLISBW-gY=k8dRUkc|7JqGeLg)lt5fh5i`*7K)$nQS3w9ug_E27G3D#N8tU{%!1AS zRf`faIW9%9X{)b52a6mW>@p9=3(D+j=oUBk{A75Z?n&~M6rAMvSruIwI=9>z<~Avc zb}pGi7SWhum0;AZTEm?^B`IF|BAwv`PqRExN?H0()qmKt$~i9Yl$pmiUIX8#f%|Sj z7Zvqg@=;csVpG+xAKX7y@Im;~3g<>PdFAjX4Lsv8zp#6Lgw3ISMmaV2$iy^wgQ^7uBoAh@Ub8h zoxcJ>3rDOgO)1kAJ$lR6j_gumLnXbO$6LGaM0Hc%uW910d@i$|WR{BnvWoX0djzYy z*s+y&O2K0nOGlNL=s|?xh5S9Ot_HatPUA!Pm9bqAAep8FgON&8Av#Xa#R9Mz!N>27 zyq_S`)wbq`Tel;LLVr;O1ySvzvc}dMF+JpRn;OvO;Opgo!{(t zA>d*j%h~U4jDGwBM^%$uP>*7hC{S5uWi$!#xP`u4 z>DQG|>PxAo17|do6;IkUuP?d%p6>Sk0#3Oae3t2F3i|42A!><{;kj!Zlt2DpspH=$ zuZdbADg~ud^5@=93}>|Y^C;uW6pI&M1Y$={q=&wk>=Wc%vhzj1=lXn0DyTJHml}oS zQMPI`oH|*SyKI=2jVE1h%0>Gzqlq}z_AD5__k00;@AusY)DYLNb-!rAoUDw_to*A; z_0)?^q)^o}Y_hoJVxdHs!PgdN&^&MHDpr%%qrZecttv(E1UTaHq7aL(aBj-zy4qW? zx0Z)k#D*{O+gy7H@`NFI?3nsI(C17nzhuV)P302RiZr(5#+l7G&M6`mVz$USuY}kE z?s4F3YR5$BG8h7BN(u9Ow05#JHX=JtZpQ=)Sv!&o_bjIlo*$ODI1%#mz7?m|o$XsY z==-CX@#vPjh9Vgy25*uSBir1vwxvC36idBm0uWPoN4P`?KFHDf@Fr1uS>K<&6{Sh@ z%HnbN=V55uH+$Y2Ndj2mx8`m0Yuo_r^cDM!7{PjV5`+I=gqXCeRgdb5rdi{!uCqN& z``xFvFaMoTbXxD0tsK{ao{tV7J3YzY!FGRCVNbTA4hgc&tTsOx#HL@CCCKX3+ zt3$9;RS}1eT4L7*6*{;uiFZ;!4s-g4%&+maT)H=F*U!_y-@K{(hIrl*m$E1c?i+B` zJA4>DJDB=y7jgfk<|-c=&g#xIVv@tm{wy`GVyCE+*sv`1Rzg=t7Jr6-uIdQ*&)KE; zMa{cwiJ^nZaPUJ15Qm4w1jv^;XGVFwC?)*rTvKxY=L`MpE%lpLw(&41Hl;&Kbl!JV zN}Q}d_@a!HcjR$|pUeudL@-({5^x)yfCS**xpecBNktVZ4C8M-vW~+tH92VMzZCg# zde3iQMCh;*LaKYt{`76houPf4L0hvJ*6498WY{DFmHX;0*JpNtnd~fi`Gt_%zkg5L za7l};Jz+ZOek{Z2$X?%tByjxcexp^gAMO^zZAi?A?zkG%KMwsnW21EaJ~Y13m~eCf z-XFF?JJkX~FRt06ZwbQ^?^p{vlSCXIhDSVr8lrvT@?XuQ&p_TkJhAzWeCyb1d>f|Yzf4LZ}$y>n3!n=eaY;t^E}Dp zFdEgx%UeA10RskYe1h0%`54Fzm2tSS$}jlHaWm_upH0NK7$mpOM9Vo9iL{yK=;?3! zIHvyAk4^tpWFIUG;6K6lOZ|lLbi9_p@=s)lv`@zT!+oO%W13H)1Rgc_;Vsi4Bt0Z> zznewiAETQ@fVt@Z{3|-!Qb#ILe%p=qLQY+xqda}=-v4VxxMMgW^F}muzxF1S%Y~Sw zG~g+th3jJ;`LyDmC@iTpULP<=KNcM!SJdnS!p}8}m1^=C`C2D|jq8JaY$|8e-Ax03 z%rA+2f&;SJ3ef!y)}p!DVoTQ_e>g9BSH&vIp(~f675Ewa6zHbsGbv8iRVJ~K^b!+V z<{)nPud}_vyxklJQh?}{`=jg^bRxV4A23n8pf+dc{25?Ibf|c~aDsQ^Elh|KJ%IF+ z)Vt&nY(QO}IbPsGv*Xg;`uOgsR35q+1>Pl`z@2~ReB19@@h5ZX=O~V5s?+_;XeQxP zk8#XLi;9cWt5E3vqgMG0$V2RNwPBIWz2_27G#Q=@Fw8bbOE!+Ji2OXaBkGyk(`yr- z@Noz_>NJU6O2419emb(%x>)6W!>O@j(G&IOb8}5@ByR6I20qcl&HQ?gEtp$J=!;o$ z4kU>%3Xvz(1TIs*`$5{1|KwKN^`z@3+3@uvh7JmzcYpN`K7MAIHPDXysG_@D$fWx4 zlvAh&bI{gAFP2lA{mGA!mHRb=?oPkXb;CyqM*5l4&VWDn0;%u7%dK)86?fs8EnATL zue}3h8GdCcO*C~Ok79q#Us-t$V##d2L;Mp8AlV(dUk9bL_J72d6KH#qiiD+v zl@E?sZTmfSxm?l9N7@|Ra3iKUD93Y_)@_>vcZ;94a3SIV53SCbw|8i)G^A2}ZZ=6OM%7yiVbgcLdkR7{1r zFs}8uXD^@2!VDf~s{)g&;^3Hlm@M))e>G{+l0IMYOCs&B>WrUE`3^pIH7g4$4b@K; zxJ`*jFsNex0%j>MZHN)C%KpU^d!S)VfYpzFb3)FNmZW11S-cCFAxk7{Z8D_cxKdIgZSkg&hJ$ivV0dAZ6;T#L>wZ<*- zK(1`@-wlH9uW(;zun@WCuw44%1OsnjJk>@2&mIq5Iik^gk~SpyzOrGLF%XTA@hD0B zaI0-Gjx3?iCpFGgA11PbeC%1t{xlqXN3GycxzRwJB^d`!*)pw6i`f2{Au`p7?dPK|~-EOT9s7U~xQ2F*yzC&J$1zxUu*cYyG% z$~g!RV4^o8HrOwQKQ^ukgNJ=-QhFSC*0u&Y3{S*!Zq6wKmzlmy^gufN5{SRqQ+**3 zYt>~ep>lTSAUZut|52Lqm{ZC-S?4_A^d0_N-@N)9fTw3Qg9b9?z6U7$VDDHI^dBK@_7q^bIH~h?8fU)Z! zmKy%P*POy~cS z@>mV*T}yHPJU;|Ec{)lK^kF~I3%m@$m-*!zBdl86;6uY3u3^5~-*oxL z;dM8bM;x(7-nfq6(geAu(pu3iJw z+Sr-PcRF$bJtq!bRBf}?8ob3hhkL;%epEK8!KLM@*;v4+Y-LAniCoXkI0iHTM(INd zT>P*6of`U7&OS%pR#=wURJLqDmGKyaC@qY{WSusu_H(+N&pQ&0k5KBu#F!jLM}Y>+ z1$;5q%T6dVuhYEuR@qmf@J(`wF%T5*K|Z!A(^YaO)YF(@7MAY(#ZT6>Ql?8IJ09LE zgc;~%flW=Vb{mXmd>@amJXUFc$OK{rsfN%#aS18aVM|lP+ykoU>>cDZB2cI@9X2q1 zb)7~iROhK}G7^-aW1Ad}F?WF+3n#@JA>CribzLM$^M2&Un;TW=2&$U3Mi=QPQLiHl zBn*&X^8BO`(!8Q_iOItmj>0UpjuTB-1!4^u51J&fuGyn2XJS1Xi<@8=vsf8cvhIju?J725`n zgBhcC(eY4>t^g!(h(}RfmwY*X{L}jcLA@ko`#>xPPI^KeNC6BAi5)Cv?#+w_-9jim z!scxskN`>pttGKqzh8Bn{94hpTc>J`XwHu(Ms`Br$booJU5Z93Oi!ig+*>D{O0yk- zq*OY9C8_u=DRo*A!BlPXKm&8|qMqFL9T&^D{snp33Yw`!HX0eZ`33X`PKABr$F#-^ z#EO(ul!eT)Q6Nq*Kr8l`inrxHlUJXOD^Nu_=;^7%%<%Kb%Y>&Nx)xD!4RPyQMb8}h z@v!Mqt~2OuEqK3Fm|EF5Pb6LwP)(YxxXB~{&Dd#?=}s2AXOHFz7Ng=Bl8zfimV%bI z>_I(Obklq4r=4$-k`oGI%2QFynTrFaCgp@#%gqNw5JI7o;-llmj|X2Y+>%-Car4@E zVvQPu;a{}KD{$?!On@p4t}*&}nFO2SKV@OFx^bVP&%B~?PsnnDu!gS+*9ED19A(V- zt+1XFjmvTfy=@(J9bz2!I`#n(J1V?>x=&bsN3Uw-^>}QKdx?^IM(%Pv74m%oklLa^ z4a7b#{`VfH`f*i{s~hFz0+6&1T{@yO@AK3IkK+)uej)f+-qWcv?(j zBFM~mke-QNzMF`h3;Cn8!^*JJg2ZBwNnvw5oR}2_flnO-Og@CE$VDlHXd30ktm|=( zrM*eA3)hMTzdkM60EPYyP2o*;$|-M``A@`h5H`jxU-D;TreQ`@ONe=!??NyXkOai1 zd=1s-l&im>lL0TuCQ1edJzT(YSM=m$Yf0HY=UszpHn` zqk}0?9|Hb_(!ngU4+FjEbnNezEQt5QHr!LtU7!*U+Nas^8=4-kUVaVIe~30v<`iYcL!Yp`(o>AJ0_B!kjis90q_WK3kK0+{6@m zEVXwNvr3V)!-&ih2B&ZwAw6GF7d7)Z3e4e&F2F-I1s{mIRP7=ngpP`)_K{ z3{GE6E41FCWHr%-uv(R~5bAK#E;_D}qME3(V^Oaq-27txt}c$C#(5$hEC2LjH zsp9MZaEpYOrX@p8=>9HAnaKs5Dl_Zvf zF7F9DU5k5+*Tk^~*GK#diqBL~wdS?7%~?;&UdWNgLQ$uk|4D;fEj=J?$E1RteO`Wn zg0^)1xWglLtk}^rF93ClGILtY3$*Ly)$5)6!)F=&i!=ZaTrwFx7VR)xh+MU;P6gVL z+>cG~Y*#WU243ylf&R&`a4}1L4}VXfo3tLEdPXt(?27j#fT7H*Og^}@4ofL9B45^n zjnfioCH4@@!-(($v$;G5HlfJ8mn_Z z++i)ZoiPL;v>TPy{zi2$=WvgZ^McVPf$jNmL34l^c?J)~Jh>ce-1>VrzQygoM`ny_04i4Y@~L>#;+}X%+uAT`?FK=HZW~&2GwU9A4X~@wn#OU-smc?IMZz`QV75I&#%=|gmndw*J z9pLSC4mqXcI?2heHfBx)qHs(dfi6t{^yU`6eLXS}0fb&sz(>nwyBTvY7S}2PNx`{0 zfRv7O^T+6|x^K9#)@J`rh1Qy%+FoA|v*^z*tD^lqeV85fujkg(cmIhoR_%Rv*7XuF z!G9if>(=cy6?qw0Ru(-3TNRz->a$Fze{K5jv9rKgga1DL-|CKY`ERBw{%@2X{|D&X l|GM}8dzs1qe;$5xb2~;Kf|(Z;r@RIHs3>U2m&?BT^dE5)t=a$p literal 0 HcmV?d00001 diff --git a/examples/vllm_kv_offload.py b/examples/offline_inference.py similarity index 93% rename from examples/vllm_kv_offload.py rename to examples/offline_inference.py index c9f50248..2f5c8500 100644 --- a/examples/vllm_kv_offload.py +++ b/examples/offline_inference.py @@ -2,7 +2,6 @@ import os import time from dataclasses import asdict - # Third Party from vllm import LLM, SamplingParams from vllm.config import KVTransferConfig @@ -23,7 +22,8 @@ def build_llm_with_uc(module_path: str, name: str, model: str): kv_connector=name, kv_connector_module_path=module_path, kv_role="kv_both", - kv_connector_extra_config={"ucm_connector_name": "UcmOceanStore", "ucm_connector_config": {"block_size": 128}} + kv_connector_extra_config={"ucm_connector_name": "UcmDram", + "ucm_connector_config": {"max_cache_size": 5368709120}} ) llm_args = EngineArgs( @@ -73,12 +73,13 @@ def main(): "century, the root sauses behind it, and a set of scientifically grounded, morally sound, and globally " "cooperative solutions that transcend culturak and national boundaries. Include both immediate actions " "and long-term strategies." - ] + ] sampling_params = SamplingParams(temperature=0, top_p=0.95, max_tokens=100) print_output(llm, prompts, sampling_params, "first") print_output(llm, prompts, sampling_params, "second") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/unifiedcache/integration/vllm/uc_connector.py b/unifiedcache/integration/vllm/uc_connector.py index 45339c1e..0afc6a21 100644 --- a/unifiedcache/integration/vllm/uc_connector.py +++ b/unifiedcache/integration/vllm/uc_connector.py @@ -477,7 +477,7 @@ def get_num_new_matched_tokens( # we need to recompute the last token. This if condition will be removed # once vLLM's scheduler provides a better solution in the future. if num_external_computed_tokens == request.num_tokens: - num_external_computed_tokens -= 1 + num_external_computed_tokens -= self.block_size self.load_paras[request.request_id] = LoadPara( vllm_cached_tokens=num_computed_tokens, storage_cached_tokens=num_external_computed_tokens, From 381c832dcbc1ce81db08b07e4ce97ff1603c4133 Mon Sep 17 00:00:00 2001 From: harrisonyhq Date: Fri, 1 Aug 2025 15:17:06 +0800 Subject: [PATCH 5/8] [Feat] Move kv_block_size to config (#43) * [Feat] Move kv_block_size to config * [Fix] Fix the python hash seed: must be random or integer in range[0:4294967295] --- .../getting-started/example/dram_conn.md | 19 +++++++++++++++---- examples/offline_inference.py | 6 ++++-- unifiedcache/ucm_connector/ucm_dram.py | 2 +- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/source/getting-started/example/dram_conn.md b/docs/source/getting-started/example/dram_conn.md index 3c06704a..5ffcf276 100644 --- a/docs/source/getting-started/example/dram_conn.md +++ b/docs/source/getting-started/example/dram_conn.md @@ -29,12 +29,15 @@ To use the DRAM connector, you need to configure the `connector_config` dictiona - `max_cache_size` *(optional)*: Specifies the maximum allowed DRAM memory usage (in **byte**) for caching in `kv_connector_extra_config["ucm_connector_config"]`. If not provided, it defaults to **5 GB**. +- `kv_block_size` *(optional)*: + Specifies the memory size (in bytes) of a single key or value cache block used in vLLM’s paged attention mechanism, which is calculated as : `block_size * head_size * total_num_kv_heads * element_size`. ### Example: ```python -kv_connector_extra_config={"ucm_connector_name": "UcmDram", "ucm_connector_config":{"max_cache_size": 5368709120}} # Allocate up to 8GB DRAM for KV cache +# KV Block size (in byte) is 262144 +kv_connector_extra_config={"ucm_connector_name": "UcmDram", "ucm_connector_config":{"max_cache_size": 5368709120, "kv_block_size": 262144}} ``` ## Launching Inference @@ -47,7 +50,7 @@ To start **offline inference** with the DRAM connector，modify the script `exam # In examples/offline_inference.py ktc = KVTransferConfig( ... - kv_connector_extra_config={"ucm_connector_name": "UcmDram", "ucm_connector_config":{"max_cache_size": 5368709120}} + kv_connector_extra_config={"ucm_connector_name": "UcmDram", "ucm_connector_config":{"max_cache_size": 5368709120, "kv_block_size": 262144}} ) ``` @@ -60,7 +63,14 @@ python offline_inference.py ### Online Inference -For **online inference** , vLLM with our connector can also be deployed as a server that implements the OpenAI API protocol. Run the following command to start the vLLM server with the Qwen/Qwen2.5-14B-Instruct model: +For **online inference** , vLLM with our connector can also be deployed as a server that implements the OpenAI API protocol. + +First, specify the python hash seed by: +```bash +export PYTHONHASHSEED=123456 +``` + +Run the following command to start the vLLM server with the Qwen/Qwen2.5-14B-Instruct model: ```bash vllm serve /home/models/Qwen2.5-14B-Instruct \ @@ -77,7 +87,8 @@ vllm serve /home/models/Qwen2.5-14B-Instruct \ "kv_connector_extra_config": { "ucm_connector_name": "UcmDram", "ucm_connector_config": { - "max_cache_size": 5368709120 + "max_cache_size": 5368709120, + "kv_block_size": 262144 } } }' diff --git a/examples/offline_inference.py b/examples/offline_inference.py index 2f5c8500..5c63195e 100644 --- a/examples/offline_inference.py +++ b/examples/offline_inference.py @@ -14,7 +14,7 @@ def setup_environment_variables(): os.environ["VLLM_USE_V1"] = "1" - + os.environ["PYTHONHASHSEED"] = "123456" @contextlib.contextmanager def build_llm_with_uc(module_path: str, name: str, model: str): @@ -23,7 +23,9 @@ def build_llm_with_uc(module_path: str, name: str, model: str): kv_connector_module_path=module_path, kv_role="kv_both", kv_connector_extra_config={"ucm_connector_name": "UcmDram", - "ucm_connector_config": {"max_cache_size": 5368709120}} + "ucm_connector_config": {"max_cache_size": 5368709120, + "kv_block_size": 262144} + } ) llm_args = EngineArgs( diff --git a/unifiedcache/ucm_connector/ucm_dram.py b/unifiedcache/ucm_connector/ucm_dram.py index 3fa740b4..49583bd6 100644 --- a/unifiedcache/ucm_connector/ucm_dram.py +++ b/unifiedcache/ucm_connector/ucm_dram.py @@ -60,7 +60,7 @@ def __init__(self, config: Dict): super().__init__(config) self.dram_cache: Dict[str, any] = {} self.max_cache_byte = int(config.get("max_cache_size", 5368709120)) - self.kv_block_size = config["kv_block_size"] + self.kv_block_size = int(config.get("kv_block_size", 262144)) self.max_block_num = self.max_cache_byte // self.kv_block_size if config["role"] == "scheduler": self.cached_blocks = set() From 8e18e09ba7244c03fa692d36aa055d2597cc27ac Mon Sep 17 00:00:00 2001 From: qyh111 Date: Fri, 1 Aug 2025 16:23:47 +0800 Subject: [PATCH 6/8] [feature][docs]finish nfs store and add docs (#44) * nfs store modify ucm_nfs_store * Add NFS Connector Doc And Performance * Modify param kv_cache_size * Modify pictures * modify docs * Add 8K DRAM Conn performance exceed limit --------- Co-authored-by: Celina --- .../getting-started/example/nfs_conn.md | 126 ++++++++++++++++++ docs/source/images/nfs_performance.png | Bin 0 -> 77550 bytes .../cc/api/ucmnfsstore/ucmnfsstore.cc | 4 +- .../cc/api/ucmnfsstore/ucmnfsstore.h | 3 +- .../cc/domain/device/aclrt_device.cc | 17 +-- .../cc/domain/device/aclrt_device.h | 6 +- .../cc/domain/device/cuda_device.cc | 47 +++---- .../cc/domain/device/cuda_device.h | 9 +- .../ucmnfsstore/cc/domain/device/device.cc | 10 +- .../ucmnfsstore/cc/domain/device/device.h | 8 +- .../cc/domain/device/ibuffered_device.cc | 12 +- .../cc/domain/device/ibuffered_device.h | 17 +-- .../ucmnfsstore/cc/domain/device/idevice.h | 6 +- .../cc/domain/space/space_manager.cc | 21 ++- .../cc/domain/tsf_task/configurator.h | 80 ----------- .../ucmnfsstore/cc/domain/tsf_task/tsf_task.h | 16 +-- .../cc/domain/tsf_task/tsf_task_manager.cc | 26 ++-- .../cc/domain/tsf_task/tsf_task_manager.h | 12 +- .../cc/domain/tsf_task/tsf_task_queue.cc | 33 +++-- .../cc/domain/tsf_task/tsf_task_queue.h | 10 +- .../cc/domain/tsf_task/tsf_task_runner.cc | 81 ++++++----- .../cc/domain/tsf_task/tsf_task_runner.h | 14 +- .../cc/domain/tsf_task/tsf_task_set.cc | 57 -------- .../cc/domain/tsf_task/tsf_task_set.h | 38 ++++-- .../cc/domain/tsf_task/tsf_task_waiter.h | 13 +- .../csrc/ucmnfsstore/cc/infra/file/file.cc | 3 +- .../csrc/ucmnfsstore/cc/infra/file/ifile.h | 18 +-- .../ucmnfsstore/cc/infra/file/posix_file.cc | 114 ++-------------- .../ucmnfsstore/cc/infra/file/posix_file.h | 29 ++-- .../ucmnfsstore/cc/infra/memory/memory.cc | 2 +- .../csrc/ucmnfsstore/cc/infra/memory/memory.h | 6 +- .../csrc/ucmnfsstore/cc/infra/status/status.h | 1 - .../csrc/ucmnfsstore/cc/infra/thread/latch.h | 17 +-- .../csrc/ucmnfsstore/cmake/cuda.cmake | 6 + .../csrc/ucmnfsstore/cmake/flags.cmake | 4 +- .../csrc/ucmnfsstore/cpy/ucmnfsstore.py.cc | 7 +- unifiedcache/integration/vllm/uc_connector.py | 3 - unifiedcache/ucm_connector/factory.py | 7 +- unifiedcache/ucm_connector/ucm_nfs_store.py | 52 +++----- 39 files changed, 410 insertions(+), 525 deletions(-) create mode 100644 docs/source/images/nfs_performance.png delete mode 100644 unifiedcache/csrc/ucmnfsstore/cc/domain/tsf_task/configurator.h delete mode 100644 unifiedcache/csrc/ucmnfsstore/cc/domain/tsf_task/tsf_task_set.cc diff --git a/docs/source/getting-started/example/nfs_conn.md b/docs/source/getting-started/example/nfs_conn.md index d43aae88..95da8f69 100644 --- a/docs/source/getting-started/example/nfs_conn.md +++ b/docs/source/getting-started/example/nfs_conn.md @@ -1,2 +1,128 @@ # NFS Connector +This document provides a usage example and configuration guide for the **NFS Connector**. This connector enables offloading of KV cache from GPU HBM to SSD or Local Disk, helping reduce memory pressure and support larger models or batch sizes. + +## Performance: DRAM Connector vs NFS Connector + +### Overview +When the total size of `kvcache` does not exceed the `max_cache_size` configured for the DRAM Connector, the DRAM Connector demonstrates superior performance. However, when the `kvcache` size exceeds `max_cache_size`, the DRAM Connector experiences significant performance degradation, at which point the NFS Connector becomes the better-performing option. + +

+ UCM +

+ +## Features + +The DRAM connector supports the following functionalities: + +- `dump`: Offload KV cache blocks from HBM to SSD or Local Disk. +- `load`: Load KV cache blocks from SSD or Local Disk back to HBM. +- `lookup`: Look up KV blocks stored in SSD or Local Disk by block hash. +- `wait`: Ensure that all dump or load operations have completed. +- `commit`: Mark cache operations as complete and ready for reuse. + +## Configuration + +To use the NFS connector, you need to configure the `connector_config` dictionary in your model's launch configuration. + +### Required Parameters + +- `storage_backends` *(required)*: + The `storage_backends` directory can either be a local folder or an NFS-mounted directory backed by an SSD driver +- `kv_block_size` *(required)*: + `kv_block_size` represents `block_size * head_size * total_num_kv_heads * element_size * num_layers * 2` + +### Example: + +```python +kv_connector_extra_config={"ucm_connector_name": "UcmNfsStore", "ucm_connector_config":{"storage_backends": "/mnt/test1", "kv_block_size": 33554432}} +``` + +## Launching Inference + +### Offline Inference + +To start **offline inference** with the NFS connector，modify the script `examples/offline_inference.py` to include the `kv_connector_extra_config` for NFS connector usage: + +```python +# In examples/offline_inference.py +ktc = KVTransferConfig( + ... + kv_connector_extra_config={"ucm_connector_name": "UcmNfsStore", "ucm_connector_config":{"storage_backends": "/mnt/test1", "kv_block_size": 33554432}} +) +``` + +Then run the script as follows: + +```bash +cd examples/ +export PYTHONHASHSEED=123456 +python offline_inference.py +``` + +### Online Inference + +For **online inference** , vLLM with our connector can also be deployed as a server that implements the OpenAI API protocol. Run the following command to start the vLLM server with the Qwen/Qwen2.5-14B-Instruct model: + +```bash +export PYTHONHASHSEED=123456 +vllm serve /home/models/Qwen2.5-14B-Instruct \ +--max-model-len 20000 \ +--tensor-parallel-size 2 \ +--gpu_memory_utilization 0.87 \ +--trust-remote-code \ +--port 7800 \ +--kv-transfer-config \ +'{ + "kv_connector": "UnifiedCacheConnectorV1", + "kv_connector_module_path": "unifiedcache.integration.vllm.uc_connector", + "kv_role": "kv_both", + "kv_connector_extra_config": { + "ucm_connector_name": "UcmNfsStore", + "ucm_connector_config": { + "storage_backends": "/mnt/test", + "kv_block_size": 33554432 + } + } +}' +``` + +If you see log as below: + +```bash +INFO: Started server process [1049932] +INFO: Waiting for application startup. +INFO: Application startup complete. +``` + +Congratulations, you have successfully started the vLLM server with NFS Connector! + +Afrer successfully started the vLLM server，You can interact with the API as following: + +```bash +curl http://localhost:7800/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "/home/models/Qwen2.5-14B-Instruct", + "prompt": "Shanghai is a", + "max_tokens": 7, + "temperature": 0 + }' +``` +To quickly experience the NFS Connector's effect: + +1. Start the service with: + `--no-enable-prefix-caching` +2. Send the same request (exceed 128 tokens) twice consecutively +3. Remember to enable prefix caching (do not add `--no-enable-prefix-caching`) in production environments. +### Log Message Structure +```plaintext +[UCMNFSSTORE] [I] Task(,,,) finished, elapsed