From 6ffcf60c1c32c28f71fb45fd6eaa7fb50657d076 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 25 Sep 2025 14:14:42 +0800 Subject: [PATCH 01/88] try to enable auto_scheme API --- auto_round/__init__.py | 3 +- auto_round/__main__.py | 9 ++++ auto_round/auto_schemes/__init__.py | 24 +++++++++ auto_round/autoround.py | 5 +- auto_round/compressors/base.py | 76 +++++++++++++++-------------- auto_round/data_type/register.py | 3 +- auto_round/schemes.py | 29 +++++++++-- 7 files changed, 103 insertions(+), 46 deletions(-) create mode 100644 auto_round/auto_schemes/__init__.py diff --git a/auto_round/__init__.py b/auto_round/__init__.py index 15bbc373d..1ce7d5e1e 100644 --- a/auto_round/__init__.py +++ b/auto_round/__init__.py @@ -13,11 +13,10 @@ # limitations under the License. from auto_round.autoround import AutoRound -# support for old api from auto_round.autoround import AutoRoundLLM, AutoRoundMLLM, AutoRoundAdam +from auto_round.schemes import QuantizationScheme, AutoScheme from auto_round.utils import LazyImport - def __getattr__(name): if name == "AutoHfQuantizer": from auto_round.inference.auto_quantizer import AutoHfQuantizer diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 78a8fc9d6..5c77c2f10 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -466,6 +466,13 @@ def tune(args): extra_config.tuning_config = tuning_config extra_config.scheme_config = scheme_config extra_config.mllm_config = mllm_config + layer_config = {} + from auto_round.auto_schemes.delta_loss import get_mixed_config_layer_config + best_path = get_mixed_config_layer_config(model_name,target_bits=6) + for item in best_path: + layer_config[item[0]] = {} + layer_config[item[0]]["bits"] = item[1] + layer_config[item[0]]["act_bits"] = item[1] autoround: BaseCompressor = AutoRound( model=model_name, @@ -484,6 +491,8 @@ def tune(args): not_use_best_mse=args.not_use_best_mse, enable_adam=args.adam, extra_config=extra_config, + layer_config=layer_config + ) model_name = args.model.rstrip("/") diff --git a/auto_round/auto_schemes/__init__.py b/auto_round/auto_schemes/__init__.py new file mode 100644 index 000000000..38d40e023 --- /dev/null +++ b/auto_round/auto_schemes/__init__.py @@ -0,0 +1,24 @@ +AUTO_SCHEMES_ALGS = {} + +def register_dtype(names): + """Class decorator to register a mixed precision algorithm to the registry. + + Decorator function used before a Pattern subclass. + + Args: + names: A string. Define the export type. + + Returns: + cls: The class of register. + """ + + def register(alg): + if isinstance(names, (tuple, list)): + for name in names: + AUTO_SCHEMES_ALGS[name] = alg + else: + AUTO_SCHEMES_ALGS[names] = alg + + return alg + + return register diff --git a/auto_round/autoround.py b/auto_round/autoround.py index 4074213a9..22d3dc29b 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -25,7 +25,7 @@ MLLMCompressor, ) from auto_round.logger import deprecated, logger -from auto_round.schemes import QuantizationScheme +from auto_round.schemes import QuantizationScheme, AutoScheme from auto_round.utils import is_mllm_model @@ -64,6 +64,7 @@ def __new__( model: Union[torch.nn.Module, str], tokenizer=None, scheme: Union[str, dict, QuantizationScheme] = "W4A16", + auto_scheme: AutoScheme = None, layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None, dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k", iters: int = 200, @@ -77,7 +78,6 @@ def __new__( seed: int = 42, # for adam enable_adam: bool = False, - # for MLLM extra_config: ExtraConfig = None, **kwargs, ) -> BaseCompressor: @@ -159,6 +159,7 @@ def __new__( model=model, tokenizer=tokenizer, scheme=scheme, + auto_scheme=auto_scheme, layer_config=layer_config, dataset=dataset, iters=iters, diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 01546034d..fa7ee0bf8 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -35,7 +35,7 @@ from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType from auto_round.logger import logger from auto_round.low_cpu_mem.utils import get_layers_before_block -from auto_round.schemes import QuantizationScheme, preset_name_to_scheme +from auto_round.schemes import QuantizationScheme, preset_name_to_scheme, AutoScheme from auto_round.sign_sgd import SignSGD from auto_round.special_model_handler import _handle_moe_model from auto_round.utils import ( @@ -130,6 +130,7 @@ def __init__( model: Union[torch.nn.Module, str], tokenizer=None, scheme: Union[str, dict, QuantizationScheme] = "W4A16", + auto_scheme: AutoScheme = None, layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None, dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k", iters: int = 200, @@ -204,7 +205,6 @@ def __init__( """ self.scheme = None self._parse_and_set_scheme(scheme, kwargs) - # Extra/legacy kwargs for backward compatibility # Major version releases may pack them with extra configuration options amp = kwargs.pop("amp", True) @@ -237,7 +237,7 @@ def __init__( logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.") if "CUBLAS_WORKSPACE_CONFIG" not in os.environ: os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" - # deprecated, default not to use torch.use_deterministic_algorithms + # Deprecated, default not to use torch.use_deterministic_algorithms if not disable_deterministic_algorithms or enable_deterministic_algorithms: if not disable_deterministic_algorithms: logger.warning( @@ -255,26 +255,14 @@ def __init__( if device_map is None: device_map = 0 - # Set device, must place after model loading - self._set_device(device_map) - - if (isinstance(device_map, dict) and device_map) or device_map == "auto": - self.device_map = device_map - elif isinstance(device_map, str) and "," in device_map: - device_map = device_map.replace(" ", "") # Remove any spaces - self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()] - self.device_map = "auto" - else: - self.device_map = None - self._set_device_map_in_blocks(self.device_map) # Model related self.quantized = False if isinstance(model, str): model, tokenizer, low_cpu_mem_usage = llm_load_model( model, - device="cpu", - low_cpu_mem_mode=low_cpu_mem_usage, # always load cpu first + device="cpu", # always load cpu first + low_cpu_mem_mode=low_cpu_mem_usage, ) elif tokenizer is None and iters > 0: raise ValueError("A tokenizer must be set for non-str model input") @@ -289,17 +277,23 @@ def __init__( self.tokenizer = tokenizer self.shared_cache_keys = get_shared_keys(self.model) - not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers) - if len(not_quantize_layer_names) > 0: - logger.info(f"{not_quantize_layer_names} will not be quantized.") - if layer_config is None: - layer_config = {} - for name in not_quantize_layer_names: - layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"} - self._parse_layer_config(layer_config) # must place after model init + self._parse_layer_config(layer_config, fp_layers) # must place after model init self.to_quant_block_names = to_quant_block_names + # Set device, must place after model loading + self._set_device(device_map) + + if (isinstance(device_map, dict) and device_map) or device_map == "auto": + self.device_map = device_map + elif isinstance(device_map, str) and "," in device_map: + device_map = device_map.replace(" ", "") # Remove any spaces + self.device_list = [int(dev) for dev in device_map.split(",") if dev.isdigit()] + self.device_map = "auto" + else: + self.device_map = None + self._set_device_map_in_blocks(self.device_map) + # Tuning hyperparameters self.seed = seed set_seed(self.seed) @@ -385,7 +379,7 @@ def __init__( import habana_frameworks.torch.core as htcore # pylint: disable=E0401 import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401] - def _set_device(self, device_map): + def _set_device(self, device_map:Union[str, torch.device, int,dict])->None: if hasattr(self, "device") and self.device is not None: return if isinstance(device_map, (str, torch.device, int)): @@ -409,8 +403,16 @@ def _set_device(self, device_map): else: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") - def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]]) -> None: + def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]], fp_layers) -> None: """Parse and set the layer-wise quantization configuration.""" + not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers) + if len(not_quantize_layer_names) > 0: + logger.info(f"{not_quantize_layer_names} will not be quantized.") + if layer_config is None: + layer_config = {} + for name in not_quantize_layer_names: + layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"} + # Some other quantization configs self.layer_config = {} if layer_config is None else layer_config scheme_keys = [f.name for f in fields(QuantizationScheme)] @@ -1709,7 +1711,7 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: # It is best to modify the model structure in the quantize function and check the format, # because it may cause the gguf format to not be exported normally. self.model = _handle_moe_model(self.model, formats=formats) - self.has_qlayer_outside_block = self._set_layerwise_config(self.layer_config) + self.has_qlayer_outside_block = self._set_layerwise_config(model, self.layer_config) if not hasattr(self, "formats"): logger.warning("this API is deprecated, please use `quantize_and_save` instead") else: @@ -1935,7 +1937,7 @@ def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None: del layer_input clear_memory(q_layer_input) - def _set_layerwise_config(self, layer_config: dict) -> bool: + def _set_layerwise_config(self, model:torch.nn.Module, layer_config: dict) -> bool: """ Sets the layer-wise configuration based on the provided `layer_config`. By default, only quantize layers in blocks. @@ -1950,14 +1952,14 @@ def _set_layerwise_config(self, layer_config: dict) -> bool: # Get the names of layers in quantization blocks supported_types = self.supported_types layers_in_blocks = get_layer_names_in_block( - self.model, supported_types, self.quant_block_list, self.inner_supported_types + model, supported_types, self.quant_block_list, self.inner_supported_types ) - ##process regex in layer_config + # Process regex in layer_config all_supported_layer_names = [] # List of configuration keys keys = get_quant_keys() - for n, m in self.model.named_modules(): + for n, m in model.named_modules(): # Delete previous configuration to avoid conflicts with prior tuning for key in keys: if hasattr(m, key): @@ -1981,7 +1983,7 @@ def _set_layerwise_config(self, layer_config: dict) -> bool: for match_name in matched_names: layer_config[match_name] = val else: - tmp_m = get_module(self.model, name) + tmp_m = get_module(model, name) if not isinstance(tmp_m, torch.nn.Embedding): # TODO not good code style raise ValueError(f"key {name} in layer_config is invalid, please have a double check") @@ -1989,17 +1991,17 @@ def _set_layerwise_config(self, layer_config: dict) -> bool: # Iterate through all modules in the model is_gguf = hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats) - for n, m in self.model.named_modules(): + for n, m in model.named_modules(): # Skip unsupported types if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types: - if n in self.layer_config: + if n in layer_config: if not isinstance(m, torch.nn.Embedding): logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.") - self.layer_config.pop(n) + layer_config.pop(n) continue if not is_gguf: if not check_to_quantized(layer_config[n]): - self.layer_config.pop(n) + layer_config.pop(n) continue else: continue diff --git a/auto_round/data_type/register.py b/auto_round/data_type/register.py index 12c4406a4..fca259ed6 100644 --- a/auto_round/data_type/register.py +++ b/auto_round/data_type/register.py @@ -22,8 +22,7 @@ def register_dtype(names): Decorator function used before a Pattern subclass. Args: - cls (class): The subclass of register. - name: A string. Define the export type. + names: A string. Define the export type. Returns: cls: The class of register. diff --git a/auto_round/schemes.py b/auto_round/schemes.py index a5c5975c9..7b6cf2f4d 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -14,9 +14,9 @@ import copy from copy import deepcopy from dataclasses import dataclass, fields -from typing import Generator, List, Optional +from typing import Optional, Iterable -__all__ = ["QuantizationScheme", "preset_name_to_scheme"] +__all__ = ["QuantizationScheme", "preset_name_to_scheme", "AutoScheme"] @dataclass @@ -38,7 +38,7 @@ def from_dict(cls, config: dict): return cls(**config) @classmethod - def get_attributes(cls: "QuantizationScheme") -> List[str]: + def get_attributes(cls: "QuantizationScheme") -> list[str]: return [field.name for field in fields(cls)] def __getitem__(self, key: str): @@ -180,6 +180,8 @@ def is_preset_scheme(name: str) -> bool: } ) + + # FP8 = asdict(QuantArgs.from_dict({ # "bits": 8, # "group_size": 128, @@ -201,6 +203,18 @@ def is_preset_scheme(name: str) -> bool: } ) +# For AutoScheme 16 bits options +BF16 = QuantizationScheme.from_dict( + { + "bits": 16, + "group_size": 0, + "data_type": "fp", + "act_bits": 16, + "act_data_type": "fp", + } +) + + PRESET_SCHEMES = { "W4A16": W4A16, "W2A16": W2A16, @@ -211,6 +225,7 @@ def is_preset_scheme(name: str) -> bool: "NVFP4": NVFP4, "FPW8A16": FPW8A16, "FP8_STATIC": FP8_STATIC, + "BF16": BF16, } from auto_round.export.export_to_gguf.config import GGUF_CONFIG @@ -220,3 +235,11 @@ def is_preset_scheme(name: str) -> bool: value.pop("embedding", None) value.pop("lm_head", None) PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value) + + +@dataclass +class AutoScheme: + options:Optional[Iterable[QuantizationScheme]] + target_bits:float + shared_layers:Optional[Iterable[Iterable[str]]]=None + method:str="naive_pre" \ No newline at end of file From 5d80825baa9643790b1ada4061d32818ec82bb04 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Sep 2025 06:15:51 +0000 Subject: [PATCH 02/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/__init__.py | 1 + auto_round/__main__.py | 6 +++--- auto_round/auto_schemes/__init__.py | 15 +++++++++++++++ auto_round/autoround.py | 2 +- auto_round/compressors/base.py | 9 ++++----- auto_round/schemes.py | 11 +++++------ 6 files changed, 29 insertions(+), 15 deletions(-) diff --git a/auto_round/__init__.py b/auto_round/__init__.py index 1ce7d5e1e..d7be4984c 100644 --- a/auto_round/__init__.py +++ b/auto_round/__init__.py @@ -17,6 +17,7 @@ from auto_round.schemes import QuantizationScheme, AutoScheme from auto_round.utils import LazyImport + def __getattr__(name): if name == "AutoHfQuantizer": from auto_round.inference.auto_quantizer import AutoHfQuantizer diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 5c77c2f10..a69359db8 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -468,7 +468,8 @@ def tune(args): extra_config.mllm_config = mllm_config layer_config = {} from auto_round.auto_schemes.delta_loss import get_mixed_config_layer_config - best_path = get_mixed_config_layer_config(model_name,target_bits=6) + + best_path = get_mixed_config_layer_config(model_name, target_bits=6) for item in best_path: layer_config[item[0]] = {} layer_config[item[0]]["bits"] = item[1] @@ -491,8 +492,7 @@ def tune(args): not_use_best_mse=args.not_use_best_mse, enable_adam=args.adam, extra_config=extra_config, - layer_config=layer_config - + layer_config=layer_config, ) model_name = args.model.rstrip("/") diff --git a/auto_round/auto_schemes/__init__.py b/auto_round/auto_schemes/__init__.py index 38d40e023..d3b055be2 100644 --- a/auto_round/auto_schemes/__init__.py +++ b/auto_round/auto_schemes/__init__.py @@ -1,5 +1,20 @@ +# Copyright (c) 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + AUTO_SCHEMES_ALGS = {} + def register_dtype(names): """Class decorator to register a mixed precision algorithm to the registry. diff --git a/auto_round/autoround.py b/auto_round/autoround.py index 22d3dc29b..ae1a37677 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -25,7 +25,7 @@ MLLMCompressor, ) from auto_round.logger import deprecated, logger -from auto_round.schemes import QuantizationScheme, AutoScheme +from auto_round.schemes import AutoScheme, QuantizationScheme from auto_round.utils import is_mllm_model diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index fa7ee0bf8..72ca17ddc 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -35,7 +35,7 @@ from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType from auto_round.logger import logger from auto_round.low_cpu_mem.utils import get_layers_before_block -from auto_round.schemes import QuantizationScheme, preset_name_to_scheme, AutoScheme +from auto_round.schemes import AutoScheme, QuantizationScheme, preset_name_to_scheme from auto_round.sign_sgd import SignSGD from auto_round.special_model_handler import _handle_moe_model from auto_round.utils import ( @@ -255,13 +255,12 @@ def __init__( if device_map is None: device_map = 0 - # Model related self.quantized = False if isinstance(model, str): model, tokenizer, low_cpu_mem_usage = llm_load_model( model, - device="cpu", # always load cpu first + device="cpu", # always load cpu first low_cpu_mem_mode=low_cpu_mem_usage, ) elif tokenizer is None and iters > 0: @@ -379,7 +378,7 @@ def __init__( import habana_frameworks.torch.core as htcore # pylint: disable=E0401 import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401] - def _set_device(self, device_map:Union[str, torch.device, int,dict])->None: + def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: if hasattr(self, "device") and self.device is not None: return if isinstance(device_map, (str, torch.device, int)): @@ -1937,7 +1936,7 @@ def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None: del layer_input clear_memory(q_layer_input) - def _set_layerwise_config(self, model:torch.nn.Module, layer_config: dict) -> bool: + def _set_layerwise_config(self, model: torch.nn.Module, layer_config: dict) -> bool: """ Sets the layer-wise configuration based on the provided `layer_config`. By default, only quantize layers in blocks. diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 7b6cf2f4d..af51a881e 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -14,7 +14,7 @@ import copy from copy import deepcopy from dataclasses import dataclass, fields -from typing import Optional, Iterable +from typing import Iterable, Optional __all__ = ["QuantizationScheme", "preset_name_to_scheme", "AutoScheme"] @@ -181,7 +181,6 @@ def is_preset_scheme(name: str) -> bool: ) - # FP8 = asdict(QuantArgs.from_dict({ # "bits": 8, # "group_size": 128, @@ -239,7 +238,7 @@ def is_preset_scheme(name: str) -> bool: @dataclass class AutoScheme: - options:Optional[Iterable[QuantizationScheme]] - target_bits:float - shared_layers:Optional[Iterable[Iterable[str]]]=None - method:str="naive_pre" \ No newline at end of file + options: Optional[Iterable[QuantizationScheme]] + target_bits: float + shared_layers: Optional[Iterable[Iterable[str]]] = None + method: str = "naive_pre" From a4ef4950ad11a523e6c2679384c00b5b4ceadaf6 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 25 Sep 2025 14:19:25 +0800 Subject: [PATCH 03/88] update a little --- auto_round/__main__.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index a69359db8..adafd095e 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -466,14 +466,6 @@ def tune(args): extra_config.tuning_config = tuning_config extra_config.scheme_config = scheme_config extra_config.mllm_config = mllm_config - layer_config = {} - from auto_round.auto_schemes.delta_loss import get_mixed_config_layer_config - - best_path = get_mixed_config_layer_config(model_name, target_bits=6) - for item in best_path: - layer_config[item[0]] = {} - layer_config[item[0]]["bits"] = item[1] - layer_config[item[0]]["act_bits"] = item[1] autoround: BaseCompressor = AutoRound( model=model_name, @@ -491,8 +483,7 @@ def tune(args): fp_layers=args.fp_layers, not_use_best_mse=args.not_use_best_mse, enable_adam=args.adam, - extra_config=extra_config, - layer_config=layer_config, + extra_config=extra_config ) model_name = args.model.rstrip("/") From 4173c3eb7626b8509896d49c30958029ced1864e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Sep 2025 06:20:01 +0000 Subject: [PATCH 04/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index adafd095e..78a8fc9d6 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -483,7 +483,7 @@ def tune(args): fp_layers=args.fp_layers, not_use_best_mse=args.not_use_best_mse, enable_adam=args.adam, - extra_config=extra_config + extra_config=extra_config, ) model_name = args.model.rstrip("/") From 87e945407d7a09cb083a387be574681eee3a1ce0 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 25 Sep 2025 17:04:32 +0800 Subject: [PATCH 05/88] update a little --- auto_round/compressors/base.py | 5 ++--- auto_round/utils.py | 18 ------------------ 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 72ca17ddc..f08540207 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -75,7 +75,6 @@ get_lm_head_name, get_max_vram, get_module, - get_quant_keys, get_shared_keys, htcore, infer_bits_by_data_type, @@ -1710,7 +1709,7 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: # It is best to modify the model structure in the quantize function and check the format, # because it may cause the gguf format to not be exported normally. self.model = _handle_moe_model(self.model, formats=formats) - self.has_qlayer_outside_block = self._set_layerwise_config(model, self.layer_config) + self.has_qlayer_outside_block = self._set_layerwise_config(self.model, self.layer_config) if not hasattr(self, "formats"): logger.warning("this API is deprecated, please use `quantize_and_save` instead") else: @@ -1956,7 +1955,7 @@ def _set_layerwise_config(self, model: torch.nn.Module, layer_config: dict) -> b # Process regex in layer_config all_supported_layer_names = [] # List of configuration keys - keys = get_quant_keys() + keys = [f.name for f in fields(QuantizationScheme)] for n, m in model.named_modules(): # Delete previous configuration to avoid conflicts with prior tuning diff --git a/auto_round/utils.py b/auto_round/utils.py index 9af09758e..131a91db3 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2308,24 +2308,6 @@ def convert_fp8_model_to_16b_model(model, dtype=torch.bfloat16): return model -def get_quant_keys(): - keys = [ - "bits", - "group_size", - "sym", - "data_type", - "scale_dtype", - "act_bits", - "act_group_size", - "act_sym", - "act_dynamic", - "act_data_type", - "super_bits", - "super_group_size", - ] - return keys - - def out_of_vram(error_msg): error_msg = str(error_msg) # CUDA From 242d1ee29eb4833f0f2aba2d322e0380f8712ea5 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 25 Sep 2025 17:28:48 +0800 Subject: [PATCH 06/88] try to refine parse layer config code --- auto_round/compressors/base.py | 179 +++++++++++++++++++++++++++++++++ auto_round/utils.py | 5 +- 2 files changed, 182 insertions(+), 2 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index f08540207..cbe95703b 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -401,6 +401,185 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: else: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") + + + # TODO gguf apply mixd bits, so the gguf scheme meanings in scheme and autoscheme are different + def _convert_value_layer_config_to_dict(self, + layer_config: dict[str, Union[str, dict, QuantizationScheme]]) -> dict: + + new_layer_config = {} if layer_config is None else layer_config + scheme_keys = [f.name for f in fields(QuantizationScheme)] + for key, item in new_layer_config.items(): + if isinstance(item, str): + item = asdict(preset_name_to_scheme(item.upper())) + new_layer_config[key] = item + elif isinstance(item, QuantizationScheme): + config = asdict(item) + tmp_keys = copy.deepcopy(list(config.keys())) + for tmp_key in tmp_keys: # Pop None value to be overridden + if config[tmp_key] is None: + config.pop(tmp_key) + elif isinstance(item, dict): + item_keys = item.keys() + if item_keys not in scheme_keys: + for item_key in item_keys: + if item_key not in scheme_keys: + raise ValueError( + f"the key {item_key} in layer_config for layer {key} is invalid," + f" only {scheme_keys} are supported" + ) + new_layer_config[key]["fixed_by_user"] = True + return new_layer_config + + def _expand_layer_config(self, model: torch.nn.Module, layer_config: dict[str, dict], fp_layers, quant_lm_head, + scheme, quant_block_list, supported_types, inner_supported_types): + """ + Sets the layer-wise configuration based on the provided `layer_config`. + By default, only quantize layers in blocks. + + Args: + layer_config (dict): The configuration dictionary for each layer containing various configuration options. + + Returns: + bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head), + otherwise returns False. + """ + + # set fp layers + not_quantize_layer_names = get_fp_layer_names(model, fp_layers) + # if len(not_quantize_layer_names) > 0: + # logger.info(f"{not_quantize_layer_names} will not be quantized.") + if layer_config is None: + layer_config = {} + for name in not_quantize_layer_names: + layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", + "act_data_type": "float", "fixed_by_user": True} + + # Get the names of layers in quantization blocks + layers_in_blocks = get_layer_names_in_block( + model, supported_types, quant_block_list, inner_supported_types + ) + # Process regex in layer_config + all_supported_layer_names = [] + # List of configuration keys + scheme_keys = (f.name for f in fields(QuantizationScheme)) + + for n, m in model.named_modules(): + # Delete previous configuration to avoid conflicts with prior tuning + for key in scheme_keys: + if hasattr(m, key): + delattr(m, key) + if type(m) not in supported_types and m.__class__.__name__ not in self.inner_supported_types: + continue + all_supported_layer_names.append(n) + + names_in_layer_config = list(layer_config.keys()) + for name in names_in_layer_config: + if name in all_supported_layer_names: + continue + matched_names = [] + for layer_name in all_supported_layer_names: + if re.search(re.compile(name), layer_name) is not None: + matched_names.append(layer_name) + if len(matched_names) > 0: + val = layer_config[name] + layer_config.pop(name) + for match_name in matched_names: + layer_config[match_name] = val + else: + tmp_m = get_module(model, name) + if type(tmp_m) != torch.nn.Embedding: # GGUF needs to quantize embedding layer + raise ValueError(f"key {name} in layer_config is invalid, please have a double check") + + has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head) + + # Iterate through all modules in the model + is_gguf = ("gguf" in scheme.lower() or + (hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats))) + for n, m in model.named_modules(): + # Skip unsupported types + if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types: + if n in layer_config: + if not isinstance(m, torch.nn.Embedding): + logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.") + layer_config.pop(n) + + if not is_gguf: # TODO the code here seems to could be deleted + if not check_to_quantized(layer_config[n]): + layer_config.pop(n) + + continue + + # If the layer is not in the config and is part of a quantization block, use default configuration + if n not in layer_config.keys() and n in layers_in_blocks: + layer_config[n] = {} + for key in scheme_keys: + layer_config[n][key] = getattr(self, key) + + # If the layer is partially configured, fill in missing values + elif n in layer_config.keys(): + if "data_type" in layer_config[n] and "bits" not in layer_config[n]: + tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"]) + if tmp_bits is not None and tmp_bits != self.bits: + logger.warning( + f"'data_type' do not match the specified 'bits' setting for {n}." + f" Resetting 'bits' to {tmp_bits}." + ) + layer_config[n]["bits"] = tmp_bits + if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]: + tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"]) + if tmp_bits is not None and tmp_bits != self.act_bits: + logger.warning( + f"'act_data_type' do not match the specified 'act_bits' setting for {n}." + f" Resetting 'act_bits' to {tmp_bits}." + ) + layer_config[n]["act_bits"] = tmp_bits + + for key in scheme_keys: + if key not in layer_config[n].keys(): + layer_config[n][key] = getattr(self, key) + layer_config[n]["fixed_by_user"] = True + + # If the layer is not in the config and not part of a quantization block, + # use default configuration and set specific values + else: + layer_config[n] = {} + for key in scheme_keys: + layer_config[n][key] = getattr(self, key) + layer_config[n]["bits"] = 16 + layer_config[n]["act_bits"] = 16 + + if n in layers_in_blocks: + layer_config[n]["in_blocks"] = True + else: + layer_config[n]["in_blocks"] = False + + # If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block + if ( + n not in layers_in_blocks + and check_to_quantized(layer_config[n]) + and not isinstance(m, torch.nn.Embedding) + ): + has_qlayer_outside_block = True + + in_features, out_features = get_layer_features(m) + if in_features <= layer_config[n]["group_size"]: + layer_config[n]["group_size"] = -1 + + # Apply the configuration to the corresponding layer in the model + for key in scheme_keys: + setattr(m, key, layer_config[n][key]) + + + # TODO self.quant_lm_head has not handleed yet + + need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding() + if need_to_quantize_lm_head: + has_qlayer_outside_block = True + + # Return whether there are quantized layers outside the blocks + return has_qlayer_outside_block + def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]], fp_layers) -> None: """Parse and set the layer-wise quantization configuration.""" not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers) diff --git a/auto_round/utils.py b/auto_round/utils.py index 131a91db3..bd3d1d2b5 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -21,6 +21,7 @@ import re import sys from collections import UserDict +from dataclasses import fields from enum import Enum from functools import lru_cache from pathlib import Path @@ -2278,8 +2279,8 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16): new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype) if layer.bias is not None: new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype)) - - keys = get_quant_keys() + ["tmp_name"] + scheme_keys = [f.name for f in fields(QuantizationScheme)] + keys = scheme_keys + ["tmp_name"] for key in keys: setattr(new_layer, key, getattr(layer, key, None)) From 4fc6b64a56d0fa811f0f210833f366acebe9c918 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Sep 2025 09:29:46 +0000 Subject: [PATCH 07/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/compressors/base.py | 61 ++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index cbe95703b..0cb65f332 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -401,11 +401,10 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: else: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") - - # TODO gguf apply mixd bits, so the gguf scheme meanings in scheme and autoscheme are different - def _convert_value_layer_config_to_dict(self, - layer_config: dict[str, Union[str, dict, QuantizationScheme]]) -> dict: + def _convert_value_layer_config_to_dict( + self, layer_config: dict[str, Union[str, dict, QuantizationScheme]] + ) -> dict: new_layer_config = {} if layer_config is None else layer_config scheme_keys = [f.name for f in fields(QuantizationScheme)] @@ -431,19 +430,28 @@ def _convert_value_layer_config_to_dict(self, new_layer_config[key]["fixed_by_user"] = True return new_layer_config - def _expand_layer_config(self, model: torch.nn.Module, layer_config: dict[str, dict], fp_layers, quant_lm_head, - scheme, quant_block_list, supported_types, inner_supported_types): + def _expand_layer_config( + self, + model: torch.nn.Module, + layer_config: dict[str, dict], + fp_layers, + quant_lm_head, + scheme, + quant_block_list, + supported_types, + inner_supported_types, + ): """ - Sets the layer-wise configuration based on the provided `layer_config`. - By default, only quantize layers in blocks. + Sets the layer-wise configuration based on the provided `layer_config`. + By default, only quantize layers in blocks. - Args: - layer_config (dict): The configuration dictionary for each layer containing various configuration options. + Args: + layer_config (dict): The configuration dictionary for each layer containing various configuration options. - Returns: - bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head), - otherwise returns False. - """ + Returns: + bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head), + otherwise returns False. + """ # set fp layers not_quantize_layer_names = get_fp_layer_names(model, fp_layers) @@ -452,13 +460,16 @@ def _expand_layer_config(self, model: torch.nn.Module, layer_config: dict[str, d if layer_config is None: layer_config = {} for name in not_quantize_layer_names: - layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", - "act_data_type": "float", "fixed_by_user": True} + layer_config[name] = { + "bits": 16, + "act_bits": 16, + "data_type": "float", + "act_data_type": "float", + "fixed_by_user": True, + } # Get the names of layers in quantization blocks - layers_in_blocks = get_layer_names_in_block( - model, supported_types, quant_block_list, inner_supported_types - ) + layers_in_blocks = get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types) # Process regex in layer_config all_supported_layer_names = [] # List of configuration keys @@ -494,8 +505,9 @@ def _expand_layer_config(self, model: torch.nn.Module, layer_config: dict[str, d has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head) # Iterate through all modules in the model - is_gguf = ("gguf" in scheme.lower() or - (hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats))) + is_gguf = "gguf" in scheme.lower() or ( + hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats) + ) for n, m in model.named_modules(): # Skip unsupported types if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types: @@ -556,9 +568,9 @@ def _expand_layer_config(self, model: torch.nn.Module, layer_config: dict[str, d # If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block if ( - n not in layers_in_blocks - and check_to_quantized(layer_config[n]) - and not isinstance(m, torch.nn.Embedding) + n not in layers_in_blocks + and check_to_quantized(layer_config[n]) + and not isinstance(m, torch.nn.Embedding) ): has_qlayer_outside_block = True @@ -570,7 +582,6 @@ def _expand_layer_config(self, model: torch.nn.Module, layer_config: dict[str, d for key in scheme_keys: setattr(m, key, layer_config[n][key]) - # TODO self.quant_lm_head has not handleed yet need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding() From 7f76db26d3f3c8fe51b777928eb6ca078b22c138 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 05:08:34 +0000 Subject: [PATCH 08/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/compressors/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index e67adfc15..ecf87a62d 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -296,7 +296,7 @@ def __init__( for name in not_quantize_layer_names: layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"} self._parse_layer_config(layer_config) # must place after model init - + self.to_quant_block_names = to_quant_block_names # Set device, must place after model loading From ae8837b0b2bb1e8ef8ae03085d4b6b728977e495 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 26 Sep 2025 13:29:59 +0800 Subject: [PATCH 09/88] fix --- auto_round/compressors/base.py | 2 +- auto_round/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 53667c090..aaac722ba 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -2145,7 +2145,7 @@ def _set_layerwise_config(self, model: torch.nn.Module, layer_config: dict) -> b # Process regex in layer_config all_supported_layer_names = [] # List of configuration keys - keys = [f.name for f in fields(QuantizationScheme)] + keys = (f.name for f in fields(QuantizationScheme)) + ("scale_dtype") for n, m in model.named_modules(): # Delete previous configuration to avoid conflicts with prior tuning diff --git a/auto_round/utils.py b/auto_round/utils.py index dedaf5c2c..a48751d3e 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2279,8 +2279,8 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16): new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype) if layer.bias is not None: new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype)) - scheme_keys = [f.name for f in fields(QuantizationScheme)] - keys = scheme_keys + ["tmp_name"] + scheme_keys = (f.name for f in fields(QuantizationScheme)) + keys = scheme_keys + ("tmp_name", "scale_dtype") for key in keys: setattr(new_layer, key, getattr(layer, key, None)) From 531224de42da3f9ed466bc30c15121ec5597e80f Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 26 Sep 2025 13:34:11 +0800 Subject: [PATCH 10/88] fix --- auto_round/compressors/base.py | 13 +++---------- auto_round/utils.py | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 71df8b89f..8c0b699a2 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -230,7 +230,7 @@ def __init__( self.mllm = kwargs.pop("mllm") if "mllm" in kwargs else False # Scale factor for RAM usage per parameter. self.mem_per_param_scale = kwargs.pop("mem_per_param_scale", None) - fp_layers = kwargs.pop("fp_layers", None) + fp_layers = kwargs.pop("fp_layers", "") if kwargs: logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.") @@ -288,14 +288,7 @@ def __init__( self.device_map = None self._set_device_map_in_blocks(self.device_map) - not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers) - if len(not_quantize_layer_names) > 0: - logger.info(f"{not_quantize_layer_names} will not be quantized.") - if layer_config is None: - layer_config = {} - for name in not_quantize_layer_names: - layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"} - self._parse_layer_config(layer_config) # must place after model init + self._parse_layer_config(layer_config, fp_layers) # Must place after model init self.to_quant_block_names = to_quant_block_names @@ -611,7 +604,7 @@ def _expand_layer_config( # Return whether there are quantized layers outside the blocks return has_qlayer_outside_block - def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]], fp_layers) -> None: + def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]], fp_layers:str) -> None: """Parse and set the layer-wise quantization configuration.""" not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers) if len(not_quantize_layer_names) > 0: diff --git a/auto_round/utils.py b/auto_round/utils.py index a48751d3e..1cb36f2fb 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -1046,7 +1046,7 @@ def can_pack_with_numba(): # pragma: no cover return True -def get_fp_layer_names(model, fp_layers): +def get_fp_layer_names(model:torch.nn.Module, fp_layers:str): """Identifies and returns layers in the model to exclude from quantization. This function processes a comma-separated list of fully precision (FP) layers, From c9fa4088aee6d24b05993ea2d3766105ab6793db Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 05:34:44 +0000 Subject: [PATCH 11/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/compressors/base.py | 4 +++- auto_round/utils.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 8c0b699a2..bc83e041a 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -604,7 +604,9 @@ def _expand_layer_config( # Return whether there are quantized layers outside the blocks return has_qlayer_outside_block - def _parse_layer_config(self, layer_config: dict[str, Union[str, dict, QuantizationScheme]], fp_layers:str) -> None: + def _parse_layer_config( + self, layer_config: dict[str, Union[str, dict, QuantizationScheme]], fp_layers: str + ) -> None: """Parse and set the layer-wise quantization configuration.""" not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers) if len(not_quantize_layer_names) > 0: diff --git a/auto_round/utils.py b/auto_round/utils.py index 1cb36f2fb..8525d7a88 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -1046,7 +1046,7 @@ def can_pack_with_numba(): # pragma: no cover return True -def get_fp_layer_names(model:torch.nn.Module, fp_layers:str): +def get_fp_layer_names(model: torch.nn.Module, fp_layers: str): """Identifies and returns layers in the model to exclude from quantization. This function processes a comma-separated list of fully precision (FP) layers, From 6453200001920d6c9a0402680aaf4507bc45924a Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 26 Sep 2025 13:41:32 +0800 Subject: [PATCH 12/88] fix --- auto_round/compressors/base.py | 2 +- auto_round/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 8c0b699a2..2512ef5e8 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -2158,7 +2158,7 @@ def _set_layerwise_config(self, model: torch.nn.Module, layer_config: dict) -> b # Process regex in layer_config all_supported_layer_names = [] # List of configuration keys - keys = (f.name for f in fields(QuantizationScheme)) + ("scale_dtype") + keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype") for n, m in model.named_modules(): # Delete previous configuration to avoid conflicts with prior tuning diff --git a/auto_round/utils.py b/auto_round/utils.py index 1cb36f2fb..9bbdbc161 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2280,7 +2280,7 @@ def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16): if layer.bias is not None: new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype)) scheme_keys = (f.name for f in fields(QuantizationScheme)) - keys = scheme_keys + ("tmp_name", "scale_dtype") + keys = tuple(scheme_keys) + ("tmp_name", "scale_dtype") for key in keys: setattr(new_layer, key, getattr(layer, key, None)) From 3811010768472c9da67b13213dc1d571d457a8cd Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 26 Sep 2025 14:07:19 +0800 Subject: [PATCH 13/88] tmp_change --- auto_round/compressors/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 48814853b..c7c3abe40 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -416,7 +416,7 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: # TODO gguf apply mixd bits, so the gguf scheme meanings in scheme and autoscheme are different def _convert_value_layer_config_to_dict( - self, layer_config: dict[str, Union[str, dict, QuantizationScheme]] + self, layer_config: dict[str, Union[str, dict, QuantizationScheme]],default_scheme:QuantizationScheme, ) -> dict: new_layer_config = {} if layer_config is None else layer_config @@ -441,6 +441,7 @@ def _convert_value_layer_config_to_dict( f" only {scheme_keys} are supported" ) new_layer_config[key]["fixed_by_user"] = True + return new_layer_config def _expand_layer_config( From 4de7b0879cba422eac13532ed716df816b06c6ba Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 26 Sep 2025 14:43:26 +0800 Subject: [PATCH 14/88] commit --- auto_round/compressors/base.py | 53 ++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index c7c3abe40..067cddeda 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -416,31 +416,48 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: # TODO gguf apply mixd bits, so the gguf scheme meanings in scheme and autoscheme are different def _convert_value_layer_config_to_dict( - self, layer_config: dict[str, Union[str, dict, QuantizationScheme]],default_scheme:QuantizationScheme, + self, + layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], + default_scheme: "QuantizationScheme", + use_auto_mixed_bit_in_gguf: bool = False, ) -> dict: + """ + Convert layer_config values (string, dict, QuantizationScheme) into a standardized dict format. + Adds 'fixed_by_user': True for each processed layer config. + """ + if layer_config is None: + return {} + + scheme_keys = {f.name for f in fields(QuantizationScheme)} + new_layer_config = copy.deepcopy(layer_config) - new_layer_config = {} if layer_config is None else layer_config - scheme_keys = [f.name for f in fields(QuantizationScheme)] for key, item in new_layer_config.items(): if isinstance(item, str): - item = asdict(preset_name_to_scheme(item.upper())) - new_layer_config[key] = item + # Convert preset name to scheme dict + config = asdict(preset_name_to_scheme(item.upper())) elif isinstance(item, QuantizationScheme): config = asdict(item) - tmp_keys = copy.deepcopy(list(config.keys())) - for tmp_key in tmp_keys: # Pop None value to be overridden - if config[tmp_key] is None: - config.pop(tmp_key) elif isinstance(item, dict): - item_keys = item.keys() - if item_keys not in scheme_keys: - for item_key in item_keys: - if item_key not in scheme_keys: - raise ValueError( - f"the key {item_key} in layer_config for layer {key} is invalid," - f" only {scheme_keys} are supported" - ) - new_layer_config[key]["fixed_by_user"] = True + # Validate dict keys + invalid_keys = set(item) - scheme_keys + if invalid_keys: + raise ValueError( + f"Invalid keys {invalid_keys} in layer_config for layer '{key}', " + f"only {scheme_keys} are supported." + ) + config = dict(item) + else: + raise TypeError( + f"Unsupported type for layer_config[{key}]: {type(item)}. " + f"Expected str, dict, or QuantizationScheme." + ) + + # Drop None values + config = {k: v for k, v in config.items() if v is not None} + + # Mark as user-fixed + config["fixed_by_user"] = True + new_layer_config[key] = config return new_layer_config From a9f0e444fff29ca12aafd6dd24bc7e8a933534c1 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 26 Sep 2025 14:44:44 +0800 Subject: [PATCH 15/88] commit --- auto_round/compressors/base.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 067cddeda..13ef303a0 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -446,11 +446,6 @@ def _convert_value_layer_config_to_dict( f"only {scheme_keys} are supported." ) config = dict(item) - else: - raise TypeError( - f"Unsupported type for layer_config[{key}]: {type(item)}. " - f"Expected str, dict, or QuantizationScheme." - ) # Drop None values config = {k: v for k, v in config.items() if v is not None} From 59a9f5df246da7d9676d6315435b6626da07e582 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 26 Sep 2025 16:39:17 +0800 Subject: [PATCH 16/88] update a little --- auto_round/compressors/base.py | 116 +++++++++++++++++++++++++++------ auto_round/schemes.py | 14 +++- 2 files changed, 109 insertions(+), 21 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 13ef303a0..c12aea15f 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -414,47 +414,125 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: else: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") - # TODO gguf apply mixd bits, so the gguf scheme meanings in scheme and autoscheme are different - def _convert_value_layer_config_to_dict( + def _prepare_layer_config( self, - layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], + model: torch.nn.Module, + orig_layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], default_scheme: "QuantizationScheme", - use_auto_mixed_bit_in_gguf: bool = False, + supported_types, + inner_supported_types, + fp_layers: str = "", + quant_lm_head: bool = False, ) -> dict: """ - Convert layer_config values (string, dict, QuantizationScheme) into a standardized dict format. - Adds 'fixed_by_user': True for each processed layer config. + Normalize and validate layer-specific quantization schemes, + expand regex-based configs, and merge with default scheme. """ - if layer_config is None: - return {} + from auto_round.schemes import is_gguf_scheme scheme_keys = {f.name for f in fields(QuantizationScheme)} - new_layer_config = copy.deepcopy(layer_config) + layer_config = copy.deepcopy(orig_layer_config) or {} + + # Mark layers that should stay in FP + not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers) + for name in not_quantize_layer_names: + layer_config[name] = { + "bits": 16, + "act_bits": 16, + "data_type": "float", + "act_data_type": "float", + } - for key, item in new_layer_config.items(): + def normalize_item(item, layer_name: str) -> dict: + """Convert a single config entry to dict and validate keys.""" if isinstance(item, str): - # Convert preset name to scheme dict config = asdict(preset_name_to_scheme(item.upper())) elif isinstance(item, QuantizationScheme): config = asdict(item) elif isinstance(item, dict): - # Validate dict keys invalid_keys = set(item) - scheme_keys if invalid_keys: raise ValueError( - f"Invalid keys {invalid_keys} in layer_config for layer '{key}', " + f"Invalid keys {invalid_keys} in layer_config for layer '{layer_name}', " f"only {scheme_keys} are supported." ) config = dict(item) - - # Drop None values + else: + raise TypeError( + f"Unsupported type for layer_config[{layer_name}]: {type(item)}. " + f"Expected str, dict, or QuantizationScheme." + ) + # Drop None values & mark as fixed config = {k: v for k, v in config.items() if v is not None} - - # Mark as user-fixed config["fixed_by_user"] = True - new_layer_config[key] = config + return config + + # Normalize configs + layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()} + + # Infer missing bits from data_type / act_data_type + for cfg in layer_config.values(): + if "data_type" in cfg and "bits" not in cfg: + if (tmp_bits := infer_bits_by_data_type(cfg["data_type"])) is not None: + cfg["bits"] = tmp_bits + if "act_data_type" in cfg and "act_bits" not in cfg: + if (tmp_bits := infer_bits_by_data_type(cfg["act_data_type"])) is not None: + cfg["act_bits"] = tmp_bits + + # Fill missing values from default scheme + default_dict = asdict(default_scheme) + for cfg in layer_config.values(): + for scheme_key in scheme_keys: + cfg.setdefault(scheme_key, default_dict.get(scheme_key)) + + # Special case for GGUF + is_gguf = is_gguf_scheme(default_scheme) + if is_gguf and torch.nn.Embedding not in supported_types: + supported_types = tuple(list(supported_types) + [torch.nn.Embedding]) + + # Collect all supported layer names + all_supported_layer_names = [] + for n, m in model.named_modules(): + # Clear old attributes to avoid conflicts + for key in scheme_keys: + if hasattr(m, key): + delattr(m, key) + if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types: + continue + all_supported_layer_names.append(n) + + # Expand regex configs (compile once, reuse) + for name in list(layer_config.keys()): + if name in all_supported_layer_names: + continue + regex = re.compile(name) + matched_names = [ln for ln in all_supported_layer_names if regex.search(ln)] + if matched_names: + val = layer_config.pop(name) + for match_name in matched_names: + layer_config[match_name] = val + else: + raise ValueError(f"Key '{name}' in layer_config is invalid, please double check.") + + # Enforce group_size = 32 constraint for INT weight-only quantization + if default_scheme.data_type == "int" and default_scheme.act_bits >= 16 and not is_gguf: + for n, m in model.named_modules(): + if type(m) in supported_types or m.__class__.__name__ in inner_supported_types: + if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0: + if n in layer_config: + layer_config[n]["bits"] = 16 + layer_config[n]["data_type"] = "fp" + logger.warning_once( + f"{n} will not be quantized because its shape is not divisible by 32. " + "It will be exported in FP16 instead." + ) + + # Handle lm_head + lm_head_name = get_lm_head_name(model) + if lm_head_name not in layer_config and (quant_lm_head or is_gguf): + layer_config[lm_head_name] = default_dict.copy() - return new_layer_config + return layer_config def _expand_layer_config( self, diff --git a/auto_round/schemes.py b/auto_round/schemes.py index af51a881e..9c12b61c0 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -14,9 +14,9 @@ import copy from copy import deepcopy from dataclasses import dataclass, fields -from typing import Iterable, Optional +from typing import Iterable, Optional, Union -__all__ = ["QuantizationScheme", "preset_name_to_scheme", "AutoScheme"] +__all__ = ["QuantizationScheme", "is_gguf_scheme", "preset_name_to_scheme", "AutoScheme"] @dataclass @@ -235,6 +235,16 @@ def is_preset_scheme(name: str) -> bool: value.pop("lm_head", None) PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value) +def is_gguf_scheme(scheme:Union[str, QuantizationScheme])->bool: + if isinstance(scheme,str) and scheme.upper().startswith("GGUF"): + return True + for key, val in PRESET_SCHEMES.items(): + if not key.upper().startswith("GGUF"): + continue + if val==scheme: + return True + return False + @dataclass class AutoScheme: From 1b7e911656995558c6ea900a09f97f705cadd089 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 08:39:58 +0000 Subject: [PATCH 17/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/compressors/base.py | 16 ++++++++-------- auto_round/schemes.py | 7 ++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index c12aea15f..e8bb2fad2 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -415,14 +415,14 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") def _prepare_layer_config( - self, - model: torch.nn.Module, - orig_layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], - default_scheme: "QuantizationScheme", - supported_types, - inner_supported_types, - fp_layers: str = "", - quant_lm_head: bool = False, + self, + model: torch.nn.Module, + orig_layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], + default_scheme: "QuantizationScheme", + supported_types, + inner_supported_types, + fp_layers: str = "", + quant_lm_head: bool = False, ) -> dict: """ Normalize and validate layer-specific quantization schemes, diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 9c12b61c0..97a3cdf02 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -235,13 +235,14 @@ def is_preset_scheme(name: str) -> bool: value.pop("lm_head", None) PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value) -def is_gguf_scheme(scheme:Union[str, QuantizationScheme])->bool: - if isinstance(scheme,str) and scheme.upper().startswith("GGUF"): + +def is_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> bool: + if isinstance(scheme, str) and scheme.upper().startswith("GGUF"): return True for key, val in PRESET_SCHEMES.items(): if not key.upper().startswith("GGUF"): continue - if val==scheme: + if val == scheme: return True return False From e0680493b54b9e64e0eb8d26219435c2c3f58170 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 26 Sep 2025 16:44:08 +0800 Subject: [PATCH 18/88] fix --- auto_round/compressors/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index c12aea15f..7538fc362 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -519,12 +519,13 @@ def normalize_item(item, layer_name: str) -> dict: for n, m in model.named_modules(): if type(m) in supported_types or m.__class__.__name__ in inner_supported_types: if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0: - if n in layer_config: - layer_config[n]["bits"] = 16 - layer_config[n]["data_type"] = "fp" + if n not in layer_config: + layer_config[n] = default_dict.copy() + layer_config[n]["bits"] = 16 + layer_config[n]["data_type"] = "fp" + layer_config[n]["fixed_by_user"] = True logger.warning_once( f"{n} will not be quantized because its shape is not divisible by 32. " - "It will be exported in FP16 instead." ) # Handle lm_head From 0357c0b94b7070da52c9492212be961fab69994e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 08:45:30 +0000 Subject: [PATCH 19/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/compressors/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index a1558d247..182cc435f 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -524,9 +524,7 @@ def normalize_item(item, layer_name: str) -> dict: layer_config[n]["bits"] = 16 layer_config[n]["data_type"] = "fp" layer_config[n]["fixed_by_user"] = True - logger.warning_once( - f"{n} will not be quantized because its shape is not divisible by 32. " - ) + logger.warning_once(f"{n} will not be quantized because its shape is not divisible by 32. ") # Handle lm_head lm_head_name = get_lm_head_name(model) From 602421c6ab2340476ece3a6959409b7d58be8320 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 26 Sep 2025 18:22:47 +0800 Subject: [PATCH 20/88] merge autoscheme to scheme --- auto_round/compressors/base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 8dbb0e3cc..0e21e00d5 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -128,8 +128,7 @@ def __init__( self, model: Union[torch.nn.Module, str], tokenizer=None, - scheme: Union[str, dict, QuantizationScheme] = "W4A16", - auto_scheme: AutoScheme = None, + scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16", layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None, dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k", iters: int = 200, @@ -2247,7 +2246,7 @@ def _set_layerwise_config(self, model: torch.nn.Module, layer_config: dict) -> b # Process regex in layer_config all_supported_layer_names = [] # List of configuration keys - keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype") + keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",) for n, m in model.named_modules(): # Delete previous configuration to avoid conflicts with prior tuning From 091c5ad0e9045dbcc693aa3163ca4ec476ddf44f Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 29 Sep 2025 14:31:43 +0800 Subject: [PATCH 21/88] refine layer_config code --- auto_round/__main__.py | 10 + auto_round/autoround.py | 4 +- auto_round/compressors/base.py | 635 +++++++++------------------------ auto_round/schemes.py | 8 +- auto_round/utils.py | 18 +- 5 files changed, 191 insertions(+), 484 deletions(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 07bc3f273..43f55a050 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -470,6 +470,15 @@ def tune(args): extra_config.scheme_config = scheme_config extra_config.mllm_config = mllm_config + layer_config = {} + # from auto_round.auto_schemes.haha import get_mixed_config_layer_config + # layer_config = {} + # best_path = get_mixed_config_layer_config(model_name, target_bits=3) + # for item in best_path: + # layer_config[item[0]] = {} + # layer_config[item[0]]["bits"] = item[1] + + autoround: BaseCompressor = AutoRound( model=model_name, scheme=scheme, @@ -486,6 +495,7 @@ def tune(args): not_use_best_mse=args.not_use_best_mse, enable_adam=args.adam, extra_config=extra_config, + layer_config=layer_config, ) model_name = args.model.rstrip("/") diff --git a/auto_round/autoround.py b/auto_round/autoround.py index ae1a37677..ccdca1f09 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -63,8 +63,7 @@ def __new__( cls, model: Union[torch.nn.Module, str], tokenizer=None, - scheme: Union[str, dict, QuantizationScheme] = "W4A16", - auto_scheme: AutoScheme = None, + scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16", layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None, dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k", iters: int = 200, @@ -159,7 +158,6 @@ def __new__( model=model, tokenizer=tokenizer, scheme=scheme, - auto_scheme=auto_scheme, layer_config=layer_config, dataset=dataset, iters=iters, diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 0e21e00d5..590c480e8 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -35,7 +35,7 @@ from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType from auto_round.logger import logger from auto_round.low_cpu_mem.utils import get_layers_before_block -from auto_round.schemes import AutoScheme, QuantizationScheme, preset_name_to_scheme +from auto_round.schemes import AutoScheme, QuantizationScheme, preset_name_to_scheme, get_gguf_scheme from auto_round.sign_sgd import SignSGD from auto_round.special_model_handler import _handle_moe_model from auto_round.utils import ( @@ -201,8 +201,7 @@ def __init__( ... # ... ... } """ - self.scheme = None - self._parse_and_set_scheme(scheme, kwargs) + self.scheme = self._parse_and_set_scheme(scheme, kwargs) # Extra/legacy kwargs for backward compatibility # Major version releases may pack them with extra configuration options amp = kwargs.pop("amp", True) @@ -229,7 +228,8 @@ def __init__( self.mllm = kwargs.pop("mllm") if "mllm" in kwargs else False # Scale factor for RAM usage per parameter. self.mem_per_param_scale = kwargs.pop("mem_per_param_scale", None) - fp_layers = kwargs.pop("fp_layers", "") + self.fp_layers = kwargs.pop("fp_layers", "") + self.layer_config = layer_config if kwargs: logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.") @@ -287,7 +287,7 @@ def __init__( self.device_map = None self._set_device_map_in_blocks(self.device_map) - self._parse_layer_config(layer_config, fp_layers) # Must place after model init + # self._parse_layer_config(layer_config, fp_layers) # Must place after model init self.to_quant_block_names = to_quant_block_names @@ -414,46 +414,46 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") def _prepare_layer_config( - self, - model: torch.nn.Module, - orig_layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], - default_scheme: "QuantizationScheme", - supported_types, - inner_supported_types, - fp_layers: str = "", - quant_lm_head: bool = False, - ) -> dict: + self, + model: torch.nn.Module, + layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], + default_scheme: "QuantizationScheme", + default_scale_dtype: torch.dtype | str, + supported_types: tuple, + inner_supported_types: tuple, + quant_block_list=None, + fp_layers: str = "", + quant_lm_head: bool = False, + enable_gguf_official_mixed: bool = True, + is_mllm: bool = False, + ) -> tuple[dict, bool]: """ - Normalize and validate layer-specific quantization schemes, - expand regex-based configs, and merge with default scheme. + Normalize, validate, and expand layer-specific quantization configs. + Returns (final_layer_config, has_quant_layer_outside_block) """ - from auto_round.schemes import is_gguf_scheme - scheme_keys = {f.name for f in fields(QuantizationScheme)} - layer_config = copy.deepcopy(orig_layer_config) or {} + from auto_round.schemes import get_gguf_scheme - # Mark layers that should stay in FP - not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers) - for name in not_quantize_layer_names: - layer_config[name] = { - "bits": 16, - "act_bits": 16, - "data_type": "float", - "act_data_type": "float", - } + # ---- helpers ------------------------------------------------- + def dispatch_layer_config(layer_config: dict[str, dict]) -> None: + """Assign scheme values as attributes to matched modules.""" + for layer_name, scheme in layer_config.items(): + module = get_module(model, layer_name) + for attr, value in scheme.items(): + setattr(module, attr, value) - def normalize_item(item, layer_name: str) -> dict: - """Convert a single config entry to dict and validate keys.""" + def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict: + """Convert config entry into dict and validate keys.""" if isinstance(item, str): config = asdict(preset_name_to_scheme(item.upper())) elif isinstance(item, QuantizationScheme): config = asdict(item) elif isinstance(item, dict): - invalid_keys = set(item) - scheme_keys - if invalid_keys: + invalid = set(item) - set(scheme_keys) + if invalid: raise ValueError( - f"Invalid keys {invalid_keys} in layer_config for layer '{layer_name}', " - f"only {scheme_keys} are supported." + f"Invalid keys {invalid} in layer_config for '{layer_name}'. " + f"Allowed keys: {scheme_keys}" ) config = dict(item) else: @@ -461,237 +461,135 @@ def normalize_item(item, layer_name: str) -> dict: f"Unsupported type for layer_config[{layer_name}]: {type(item)}. " f"Expected str, dict, or QuantizationScheme." ) - # Drop None values & mark as fixed + # Clean up config = {k: v for k, v in config.items() if v is not None} config["fixed_by_user"] = True return config - # Normalize configs + # ---- main logic ---------------------------------------------- + scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",) + layer_config = copy.deepcopy(layer_config) or {} + + # 1. fp_layers -> force 16 + for name in get_fp_layer_names(self.model, fp_layers): + layer_config[name] = { + "bits": 16, "act_bits": 16, + "data_type": "float", "act_data_type": "float" + } + + # 2. normalize layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()} - # Infer missing bits from data_type / act_data_type + # 3. infer missing bits for cfg in layer_config.values(): if "data_type" in cfg and "bits" not in cfg: - if (tmp_bits := infer_bits_by_data_type(cfg["data_type"])) is not None: - cfg["bits"] = tmp_bits + if (b := infer_bits_by_data_type(cfg["data_type"])) is not None: + cfg["bits"] = b if "act_data_type" in cfg and "act_bits" not in cfg: - if (tmp_bits := infer_bits_by_data_type(cfg["act_data_type"])) is not None: - cfg["act_bits"] = tmp_bits + if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None: + cfg["act_bits"] = b - # Fill missing values from default scheme + # 4. fill defaults default_dict = asdict(default_scheme) + default_dict["scale_dtype"] = default_scale_dtype for cfg in layer_config.values(): - for scheme_key in scheme_keys: - cfg.setdefault(scheme_key, default_dict.get(scheme_key)) + for key in scheme_keys: + cfg.setdefault(key, default_dict.get(key)) - # Special case for GGUF - is_gguf = is_gguf_scheme(default_scheme) - if is_gguf and torch.nn.Embedding not in supported_types: - supported_types = tuple(list(supported_types) + [torch.nn.Embedding]) + # 5. collect supported modules + gguf_name = get_gguf_scheme(default_scheme) + if gguf_name and torch.nn.Embedding not in supported_types: + supported_types = (*supported_types, torch.nn.Embedding) - # Collect all supported layer names - all_supported_layer_names = [] + all_layer_names, embedding_layer_names = [], [] for n, m in model.named_modules(): - # Clear old attributes to avoid conflicts + # cleanup stale attributes for key in scheme_keys: if hasattr(m, key): delattr(m, key) if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types: continue - all_supported_layer_names.append(n) + all_layer_names.append(n) + if isinstance(m, torch.nn.Embedding): + embedding_layer_names.append(n) - # Expand regex configs (compile once, reuse) + # 6. expand regex configs for name in list(layer_config.keys()): - if name in all_supported_layer_names: + if name in all_layer_names: continue regex = re.compile(name) - matched_names = [ln for ln in all_supported_layer_names if regex.search(ln)] - if matched_names: - val = layer_config.pop(name) - for match_name in matched_names: - layer_config[match_name] = val - else: - raise ValueError(f"Key '{name}' in layer_config is invalid, please double check.") - - # Enforce group_size = 32 constraint for INT weight-only quantization - if default_scheme.data_type == "int" and default_scheme.act_bits >= 16 and not is_gguf: - for n, m in model.named_modules(): - if type(m) in supported_types or m.__class__.__name__ in inner_supported_types: - if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0: - if n not in layer_config: - layer_config[n] = default_dict.copy() - layer_config[n]["bits"] = 16 - layer_config[n]["data_type"] = "fp" - layer_config[n]["fixed_by_user"] = True - logger.warning_once(f"{n} will not be quantized because its shape is not divisible by 32. ") - - # Handle lm_head + matched = [ln for ln in all_layer_names if regex.search(ln)] + if not matched: + raise ValueError(f"Invalid regex '{name}' in layer_config, no match found.") + val = layer_config.pop(name) + for match in matched: + layer_config[match] = val + + # 7. lm_head lm_head_name = get_lm_head_name(model) - if lm_head_name not in layer_config and (quant_lm_head or is_gguf): + tied_lm_head = False + if ( + hasattr(model, "config") + and model.config.tie_word_embeddings + and hasattr(model, "_tied_weights_keys") + ): + tied_keys =model._tied_weights_keys + if lm_head_name in tied_keys: + tied_lm_head=True + if quant_lm_head and tied_lm_head: + quant_lm_head=False + logger.warning("reset `quant_lm_head` to false as quantizing lm_head with tied weights has not been supported currently") + + if lm_head_name not in layer_config and quant_lm_head: layer_config[lm_head_name] = default_dict.copy() - return layer_config - - def _expand_layer_config( - self, - model: torch.nn.Module, - layer_config: dict[str, dict], - fp_layers, - quant_lm_head, - scheme, - quant_block_list, - supported_types, - inner_supported_types, - ): - """ - Sets the layer-wise configuration based on the provided `layer_config`. - By default, only quantize layers in blocks. - - Args: - layer_config (dict): The configuration dictionary for each layer containing various configuration options. - - Returns: - bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head), - otherwise returns False. - """ - - # set fp layers - not_quantize_layer_names = get_fp_layer_names(model, fp_layers) - # if len(not_quantize_layer_names) > 0: - # logger.info(f"{not_quantize_layer_names} will not be quantized.") - if layer_config is None: - layer_config = {} - for name in not_quantize_layer_names: - layer_config[name] = { - "bits": 16, - "act_bits": 16, - "data_type": "float", - "act_data_type": "float", - "fixed_by_user": True, - } - - # Get the names of layers in quantization blocks - layers_in_blocks = get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types) - # Process regex in layer_config - all_supported_layer_names = [] - # List of configuration keys - scheme_keys = (f.name for f in fields(QuantizationScheme)) - - for n, m in model.named_modules(): - # Delete previous configuration to avoid conflicts with prior tuning - for key in scheme_keys: - if hasattr(m, key): - delattr(m, key) - if type(m) not in supported_types and m.__class__.__name__ not in self.inner_supported_types: - continue - all_supported_layer_names.append(n) - - names_in_layer_config = list(layer_config.keys()) - for name in names_in_layer_config: - if name in all_supported_layer_names: - continue - matched_names = [] - for layer_name in all_supported_layer_names: - if re.search(re.compile(name), layer_name) is not None: - matched_names.append(layer_name) - if len(matched_names) > 0: - val = layer_config[name] - layer_config.pop(name) - for match_name in matched_names: - layer_config[match_name] = val - else: - tmp_m = get_module(model, name) - if type(tmp_m) != torch.nn.Embedding: # GGUF needs to quantize embedding layer - raise ValueError(f"key {name} in layer_config is invalid, please have a double check") - - has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head) - - # Iterate through all modules in the model - is_gguf = "gguf" in scheme.lower() or ( - hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats) - ) - for n, m in model.named_modules(): - # Skip unsupported types - if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types: - if n in layer_config: - if not isinstance(m, torch.nn.Embedding): - logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.") - layer_config.pop(n) - - if not is_gguf: # TODO the code here seems to could be deleted - if not check_to_quantized(layer_config[n]): - layer_config.pop(n) - - continue - - # If the layer is not in the config and is part of a quantization block, use default configuration - if n not in layer_config.keys() and n in layers_in_blocks: - layer_config[n] = {} - for key in scheme_keys: - layer_config[n][key] = getattr(self, key) - - # If the layer is partially configured, fill in missing values - elif n in layer_config.keys(): - if "data_type" in layer_config[n] and "bits" not in layer_config[n]: - tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"]) - if tmp_bits is not None and tmp_bits != self.bits: - logger.warning( - f"'data_type' do not match the specified 'bits' setting for {n}." - f" Resetting 'bits' to {tmp_bits}." - ) - layer_config[n]["bits"] = tmp_bits - if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]: - tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"]) - if tmp_bits is not None and tmp_bits != self.act_bits: - logger.warning( - f"'act_data_type' do not match the specified 'act_bits' setting for {n}." - f" Resetting 'act_bits' to {tmp_bits}." - ) - layer_config[n]["act_bits"] = tmp_bits - - for key in scheme_keys: - if key not in layer_config[n].keys(): - layer_config[n][key] = getattr(self, key) - layer_config[n]["fixed_by_user"] = True - - # If the layer is not in the config and not part of a quantization block, - # use default configuration and set specific values - else: - layer_config[n] = {} - for key in scheme_keys: - layer_config[n][key] = getattr(self, key) - layer_config[n]["bits"] = 16 - layer_config[n]["act_bits"] = 16 - - if n in layers_in_blocks: - layer_config[n]["in_blocks"] = True - else: - layer_config[n]["in_blocks"] = False - - # If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block - if ( - n not in layers_in_blocks - and check_to_quantized(layer_config[n]) - and not isinstance(m, torch.nn.Embedding) - ): + # 8. enforce shape divisibility for int weight-only + if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name: + for n, m in model.named_modules(): + if type(m) in supported_types or m.__class__.__name__ in inner_supported_types: + if m.weight.shape[0] % 32 or m.weight.shape[1] % 32: + layer_config.setdefault(n, default_dict.copy()) + layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True}) + logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).") + + # 9. block layers: mark as in_blocks=True + for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types): + cfg = layer_config.setdefault(name, default_dict.copy()) + cfg["in_blocks"] = True + + # ---- restore: ensure missing in_blocks are set to False and compute flag ---- + has_qlayer_outside_block = False + for cfg in layer_config.values(): + if "in_blocks" not in cfg: + cfg["in_blocks"] = False + # 如果 layer 不在 blocks 且需要量化,则标记存在 blocks 外的量化层 + if not cfg["in_blocks"] and check_to_quantized(cfg): has_qlayer_outside_block = True - in_features, out_features = get_layer_features(m) - if in_features <= layer_config[n]["group_size"]: - layer_config[n]["group_size"] = -1 + # 10. GGUF handling + if not gguf_name: + dispatch_layer_config(layer_config) + return layer_config, has_qlayer_outside_block - # Apply the configuration to the corresponding layer in the model - for key in scheme_keys: - setattr(m, key, layer_config[n][key]) + # embed + lm_head defaults for gguf + if lm_head_name not in layer_config and not tied_lm_head: + cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]] + cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype} + layer_config[lm_head_name] = cfg + has_qlayer_outside_block = True + for emd_name in embedding_layer_names: + cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]] + cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype} + layer_config[emd_name] = cfg - # TODO self.quant_lm_head has not handleed yet + if enable_gguf_official_mixed: + model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT + layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type) + + dispatch_layer_config(layer_config) + return layer_config, has_qlayer_outside_block - need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding() - if need_to_quantize_lm_head: - has_qlayer_outside_block = True - # Return whether there are quantized layers outside the blocks - return has_qlayer_outside_block def _parse_layer_config( self, layer_config: dict[str, Union[str, dict, QuantizationScheme]], fp_layers: str @@ -753,7 +651,7 @@ def _parse_layer_config( if key not in lm_head_layer_config: lm_head_layer_config[key] = getattr(self, key) - def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> None: + def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> QuantizationScheme: """Parse and set the quantization scheme.""" if isinstance(scheme, QuantizationScheme): scheme = asdict(scheme) @@ -761,7 +659,6 @@ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kw scheme = scheme elif isinstance(scheme, str): scheme = scheme.upper() - self.scheme = scheme scheme = asdict(preset_name_to_scheme(scheme)) scheme_keys = [f.name for f in fields(QuantizationScheme)] for key in scheme_keys: @@ -807,6 +704,9 @@ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kw if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3 self.act_data_type = supported_dtype break + for key in scheme_keys: + scheme[key] = getattr(self, key) + return QuantizationScheme.from_dict(scheme) def _adjust_torch_compile(self, enable_torch_compile: bool) -> None: """Sets the torch compile configuration for the tuning.""" @@ -1112,19 +1012,29 @@ def remove_duplicates(lst): formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",") formats = remove_duplicates(formats) # need the keep origin order - if isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf"): + gguf_format_name = get_gguf_scheme(self.scheme) + + if gguf_format_name: for i in range(len(formats)): - if formats[i] != "fake" and formats[i] != self.scheme.lower(): + if formats[i] != "fake" and formats[i] != gguf_format_name.lower(): logger.warning( - f"reset format {formats[i]} to {self.scheme.lower()} " - f"since scheme {self.scheme} can only be exported to format {self.scheme.lower()}" + f"reset format {formats[i]} to {gguf_format_name.lower()} " + f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()}" ) - formats[i] = self.scheme.lower() + formats[i] = gguf_format_name.lower() + _gguf_args_check(self, formats, model_type=ModelType.TEXT) if self.mllm: _gguf_args_check(self, formats, model_type=ModelType.MMPROJ) + for f in formats: + if f.startswith("gguf"): + self.scheme = preset_name_to_scheme(f) + break + + + for format_ in formats: if format_ not in SUPPORTED_FORMATS: logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}") @@ -1608,91 +1518,6 @@ def get_imatrix_hook(module, input, output): for hook in hooks: hook.remove() - def _check_need_to_quantize_lm_head_embedding(self) -> bool: - """Checks if LM head and embedding layers need quantization for GGUF format. - - This function inspects the current model's formats and determines whether - it needs to apply quantization settings to the embedding and LM head layers. - The function modifies `self.layer_config` in-place and updates the model modules. - - Returns: - bool: True if the LM head needs quantization, otherwise False. - - Raises: - NotImplementedError: If multiple non-fake GGUF formats are specified. - """ - gguf_scheme = False - if isinstance(self.scheme, str) and "gguf" in self.scheme.lower(): - gguf_scheme = True - - if not hasattr(self, "formats") and not gguf_scheme: - return False - - has_gguf: bool = gguf_scheme or any("gguf" in fmt for fmt in self.formats) - if not has_gguf: - return False - if hasattr(self, "formats"): - formats: list[str] = [fmt for fmt in self.formats if "fake" not in fmt] - if not (len(formats) == 1 and "gguf" in formats[0]): - raise NotImplementedError("Only one GGUF format can be set at a time.") - target_format: str = formats[0] - - else: - target_format = self.scheme.lower() - - tie_word_embeddings: bool = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True) - for name, module in self.model.named_modules(): - if isinstance(module, torch.nn.Embedding): - key: str = "lm_head" if tie_word_embeddings else "embedding" - config: dict[str, Any] = GGUF_INNER_CONFIG[GGUF_CONFIG[target_format][key]] - self._apply_config_to_layer(name, config, True) - - if not tie_word_embeddings: - lm_head_name: str = get_lm_head_name(self.model) - config: dict[str, Any] = GGUF_CONFIG[GGUF_CONFIG[target_format]["lm_head"]] - check_fixed_by_user = ( - self.layer_config[lm_head_name].get("fixed_by_user", False) - if lm_head_name in self.layer_config - else None - ) - self._apply_config_to_layer(lm_head_name, config, check_fixed_by_user=check_fixed_by_user) - return True - - return False - - def _apply_config_to_layer( - self, - layer_name: str, - config: dict[str, Any], - check_fixed_by_user: bool = False, - ) -> None: - """Applies GGUF quantization configuration to a given layer. - - Args: - layer_name (str): Name of the layer to configure. - config (dict[str, Any]): GGUF layer configuration. - check_fixed_by_user (bool): If True, preserve user-defined settings. - """ - act_bits: int = 16 - scale_dtype: Any = self.scale_dtype - keys: list[str] = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"] - - self.layer_config[layer_name] = self.layer_config.get(layer_name, {}) - - for key in keys: - if ( - key in self.layer_config[layer_name] - and check_fixed_by_user - # and self.layer_config[layer_name].get("fixed_by_user", False) - ): - continue - self.layer_config[layer_name][key] = config.get(key) - setattr(get_module(self.model, layer_name), key, config.get(key)) - - self.layer_config[layer_name]["act_bits"] = act_bits - self.layer_config[layer_name]["scale_dtype"] = scale_dtype - setattr(get_module(self.model, layer_name), "act_bits", act_bits) - setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype) def _quantize_layer_via_rtn(self, name: str) -> None: """Quantizes a layer using RTN (Round-To-Nearest) if available. @@ -1993,14 +1818,21 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: Returns: The quantized model and layer configurations. """ - for n, m in self.model.named_modules(): + for n, m in self.model.named_modules(): # TODO check if could removed m.tmp_name = n self._check_compatibility() formats = self.formats if hasattr(self, "formats") else None # It is best to modify the model structure in the quantize function and check the format, # because it may cause the gguf format to not be exported normally. self.model = _handle_moe_model(self.model, formats=formats) - self.has_qlayer_outside_block = self._set_layerwise_config(self.model, self.layer_config) + # self.has_qlayer_outside_block = self._set_layerwise_config(self.model, self.layer_config) + # TODO check scale_dtype + self.layer_config, self.has_qlayer_outside_block = ( + self._prepare_layer_config(self.model, self.layer_config,self.scheme, self.scale_dtype, + self.supported_types,self.inner_supported_types,self.quant_block_list, + self.fp_layers,self.quant_lm_head, + enable_gguf_official_mixed=True,is_mllm=self.mllm)) + if not hasattr(self, "formats"): logger.warning("this API is deprecated, please use `quantize_and_save` instead") else: @@ -2011,14 +1843,14 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: break if len(self.formats) == 1 and self.formats[0] == "fake": only_gguf = False - if only_gguf: - self.layer_config, gguf_format_config = get_layer_config_by_gguf_format( - self.layer_config, self.formats, self.model, model_type=ModelType.TEXT - ) - if self.mllm: - self.layer_config, gguf_format_config = get_layer_config_by_gguf_format( - self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ - ) + # if only_gguf: + # self.layer_config, gguf_format_config = get_layer_config_by_gguf_format( + # self.layer_config, self.formats, self.model, model_type=ModelType.TEXT + # ) + # if self.mllm: + # self.layer_config, gguf_format_config = get_layer_config_by_gguf_format( + # self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ + # ) # Determine if immediate packing is required formats = self.formats if ( @@ -2226,141 +2058,6 @@ def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None: del layer_input clear_memory(q_layer_input) - def _set_layerwise_config(self, model: torch.nn.Module, layer_config: dict) -> bool: - """ - Sets the layer-wise configuration based on the provided `layer_config`. - By default, only quantize layers in blocks. - - Args: - layer_config (dict): The configuration dictionary for each layer containing various configuration options. - - Returns: - bool: Returns True if there are quantized layers outside the blocks (e.g., lm-head), - otherwise returns False. - """ - # Get the names of layers in quantization blocks - supported_types = self.supported_types - layers_in_blocks = get_layer_names_in_block( - model, supported_types, self.quant_block_list, self.inner_supported_types - ) - # Process regex in layer_config - all_supported_layer_names = [] - # List of configuration keys - keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",) - - for n, m in model.named_modules(): - # Delete previous configuration to avoid conflicts with prior tuning - for key in keys: - if hasattr(m, key): - delattr(m, key) - - if not isinstance(m, supported_types) and m.__class__.__name__ not in self.inner_supported_types: - continue - all_supported_layer_names.append(n) - - names_in_layer_config = list(layer_config.keys()) - for name in names_in_layer_config: - if name in all_supported_layer_names: - continue - matched_names = [] - for layer_name in all_supported_layer_names: - if re.search(re.compile(name), layer_name) is not None: - matched_names.append(layer_name) - if len(matched_names) > 0: - val = layer_config[name] - layer_config.pop(name) - for match_name in matched_names: - layer_config[match_name] = val - else: - tmp_m = get_module(model, name) - if not isinstance(tmp_m, torch.nn.Embedding): # TODO not good code style - raise ValueError(f"key {name} in layer_config is invalid, please have a double check") - - has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head) - - # Iterate through all modules in the model - is_gguf = hasattr(self, "formats") and any("gguf" in format_ for format_ in self.formats) - for n, m in model.named_modules(): - # Skip unsupported types - if type(m) not in supported_types and m.__class__.__name__ not in self.inner_supported_types: - if n in self.layer_config: - if not isinstance(m, torch.nn.Embedding): - logger.warning(f"{n} is not supported, layer_config {n}: {layer_config[n]} will be ignored.") - layer_config.pop(n) - continue - if not is_gguf: - if not check_to_quantized(layer_config[n]): - layer_config.pop(n) - continue - else: - continue - - # If the layer is not in the config and is part of a quantization block, use default configuration - if n not in layer_config.keys() and n in layers_in_blocks: - layer_config[n] = {} - for key in keys: - layer_config[n][key] = getattr(self, key) - - # If the layer is partially configured, fill in missing values - elif n in layer_config.keys(): - if "data_type" in layer_config[n] and "bits" not in layer_config[n]: - tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"]) - if tmp_bits is not None and tmp_bits != self.bits: - logger.warning( - f"'data_type' do not match the specified 'bits' setting for {n}." - f" Resetting 'bits' to {tmp_bits}." - ) - layer_config[n]["bits"] = tmp_bits - if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]: - tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"]) - if tmp_bits is not None and tmp_bits != self.act_bits: - logger.warning( - f"'act_data_type' do not match the specified 'act_bits' setting for {n}." - f" Resetting 'act_bits' to {tmp_bits}." - ) - layer_config[n]["act_bits"] = tmp_bits - - for key in keys: - if key not in layer_config[n].keys(): - layer_config[n][key] = getattr(self, key) - layer_config[n]["fixed_by_user"] = True - - # If the layer is not in the config and not part of a quantization block, - # use default configuration and set specific values - else: - layer_config[n] = {} - for key in keys: - layer_config[n][key] = getattr(self, key) - layer_config[n]["bits"] = 16 - layer_config[n]["act_bits"] = 16 - - if n in layers_in_blocks: - layer_config[n]["in_blocks"] = True - else: - layer_config[n]["in_blocks"] = False - - # If the layer is outside a block and requires quantization, mark it as a quantized layer outside the block - if ( - n not in layers_in_blocks - and check_to_quantized(layer_config[n]) - and not isinstance(m, torch.nn.Embedding) - ): - has_qlayer_outside_block = True - - in_features, out_features = get_layer_features(m) - if in_features <= layer_config[n]["group_size"]: - layer_config[n]["group_size"] = -1 - - # Apply the configuration to the corresponding layer in the model - for key in keys: - setattr(m, key, layer_config[n][key]) - need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding() - if need_to_quantize_lm_head: - has_qlayer_outside_block = True - - # Return whether there are quantized layers outside the blocks - return has_qlayer_outside_block - @torch.no_grad() def _get_block_outputs( self, diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 97a3cdf02..f6ca0cc98 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -16,7 +16,7 @@ from dataclasses import dataclass, fields from typing import Iterable, Optional, Union -__all__ = ["QuantizationScheme", "is_gguf_scheme", "preset_name_to_scheme", "AutoScheme"] +__all__ = ["QuantizationScheme", "get_gguf_scheme", "preset_name_to_scheme", "AutoScheme"] @dataclass @@ -236,15 +236,15 @@ def is_preset_scheme(name: str) -> bool: PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value) -def is_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> bool: +def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> bool: if isinstance(scheme, str) and scheme.upper().startswith("GGUF"): return True for key, val in PRESET_SCHEMES.items(): if not key.upper().startswith("GGUF"): continue if val == scheme: - return True - return False + return key + return None @dataclass diff --git a/auto_round/utils.py b/auto_round/utils.py index 575b8e3e8..3d35d303e 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -766,8 +766,9 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs): def get_layer_names_in_block( - model, supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), quant_block_list=None, class_names=None -): + model:torch.nn.Module, supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), + quant_block_list:list=None, class_names:tuple=None +) -> list[str]: """Retrieves the names of layers within each block of the model. Returns: @@ -778,7 +779,7 @@ def get_layer_names_in_block( class_names = [] for n, m in model.named_modules(): if type(m) in supported_types or (class_names is not None and m.__class__.__name__ in class_names): - m.tmp_name = n + m.backup_name = n layers_in_block = [] if bool(quant_block_list): all_blocks = quant_block_list @@ -788,8 +789,9 @@ def get_layer_names_in_block( for block_name in block_names: block = get_module(model, block_name) for n, m in block.named_modules(): - if hasattr(m, "tmp_name"): - layers_in_block.append(m.tmp_name) + if hasattr(m, "backup_name"): + layers_in_block.append(m.backup_name) + delattr(m, "backup_name") return layers_in_block @@ -1840,9 +1842,9 @@ def _gguf_type_fallback(gguf_type): ##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129 -def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_type=ModelType.TEXT): - # TODO: support for other format later - target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None) +def get_layer_config_by_gguf_format(layer_config, target_gguf_format:str, model, model_type=ModelType.TEXT): + # # TODO: support for other format later + # target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None) import gguf # pylint: disable=E0401 From f027801f450d8036e4a98d09a4b6cc3826a7fb21 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 06:33:01 +0000 Subject: [PATCH 22/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/__main__.py | 1 - auto_round/compressors/base.py | 78 ++++++++++++++++------------------ auto_round/utils.py | 8 ++-- 3 files changed, 42 insertions(+), 45 deletions(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 43f55a050..97e3eb6ff 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -478,7 +478,6 @@ def tune(args): # layer_config[item[0]] = {} # layer_config[item[0]]["bits"] = item[1] - autoround: BaseCompressor = AutoRound( model=model_name, scheme=scheme, diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 6178623f0..c5928da60 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -35,7 +35,7 @@ from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType from auto_round.logger import logger from auto_round.low_cpu_mem.utils import get_layers_before_block -from auto_round.schemes import AutoScheme, QuantizationScheme, preset_name_to_scheme, get_gguf_scheme +from auto_round.schemes import AutoScheme, QuantizationScheme, get_gguf_scheme, preset_name_to_scheme from auto_round.sign_sgd import SignSGD from auto_round.special_model_handler import _handle_moe_model from auto_round.utils import ( @@ -414,18 +414,18 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") def _prepare_layer_config( - self, - model: torch.nn.Module, - layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], - default_scheme: "QuantizationScheme", - default_scale_dtype: torch.dtype | str, - supported_types: tuple, - inner_supported_types: tuple, - quant_block_list=None, - fp_layers: str = "", - quant_lm_head: bool = False, - enable_gguf_official_mixed: bool = True, - is_mllm: bool = False, + self, + model: torch.nn.Module, + layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], + default_scheme: "QuantizationScheme", + default_scale_dtype: torch.dtype | str, + supported_types: tuple, + inner_supported_types: tuple, + quant_block_list=None, + fp_layers: str = "", + quant_lm_head: bool = False, + enable_gguf_official_mixed: bool = True, + is_mllm: bool = False, ) -> tuple[dict, bool]: """ Normalize, validate, and expand layer-specific quantization configs. @@ -452,8 +452,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str invalid = set(item) - set(scheme_keys) if invalid: raise ValueError( - f"Invalid keys {invalid} in layer_config for '{layer_name}'. " - f"Allowed keys: {scheme_keys}" + f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}" ) config = dict(item) else: @@ -472,10 +471,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str # 1. fp_layers -> force 16 for name in get_fp_layer_names(self.model, fp_layers): - layer_config[name] = { - "bits": 16, "act_bits": 16, - "data_type": "float", "act_data_type": "float" - } + layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"} # 2. normalize layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()} @@ -528,17 +524,15 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str # 7. lm_head lm_head_name = get_lm_head_name(model) tied_lm_head = False - if ( - hasattr(model, "config") - and model.config.tie_word_embeddings - and hasattr(model, "_tied_weights_keys") - ): - tied_keys =model._tied_weights_keys + if hasattr(model, "config") and model.config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"): + tied_keys = model._tied_weights_keys if lm_head_name in tied_keys: - tied_lm_head=True + tied_lm_head = True if quant_lm_head and tied_lm_head: - quant_lm_head=False - logger.warning("reset `quant_lm_head` to false as quantizing lm_head with tied weights has not been supported currently") + quant_lm_head = False + logger.warning( + "reset `quant_lm_head` to false as quantizing lm_head with tied weights has not been supported currently" + ) if lm_head_name not in layer_config and quant_lm_head: layer_config[lm_head_name] = default_dict.copy() @@ -589,8 +583,6 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str dispatch_layer_config(layer_config) return layer_config, has_qlayer_outside_block - - def _parse_layer_config( self, layer_config: dict[str, Union[str, dict, QuantizationScheme]], fp_layers: str ) -> None: @@ -651,7 +643,7 @@ def _parse_layer_config( if key not in lm_head_layer_config: lm_head_layer_config[key] = getattr(self, key) - def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> QuantizationScheme: + def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> QuantizationScheme: """Parse and set the quantization scheme.""" if isinstance(scheme, QuantizationScheme): scheme = asdict(scheme) @@ -1023,7 +1015,6 @@ def remove_duplicates(lst): ) formats[i] = gguf_format_name.lower() - _gguf_args_check(self, formats, model_type=ModelType.TEXT) if self.mllm: _gguf_args_check(self, formats, model_type=ModelType.MMPROJ) @@ -1033,8 +1024,6 @@ def remove_duplicates(lst): self.scheme = preset_name_to_scheme(f) break - - for format_ in formats: if format_ not in SUPPORTED_FORMATS: logger.error(f"Unsupported format {format_}, please choose from {SUPPORTED_FORMATS}") @@ -1518,7 +1507,6 @@ def get_imatrix_hook(module, input, output): for hook in hooks: hook.remove() - def _quantize_layer_via_rtn(self, name: str) -> None: """Quantizes a layer using RTN (Round-To-Nearest) if available. @@ -1818,7 +1806,7 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: Returns: The quantized model and layer configurations. """ - for n, m in self.model.named_modules(): # TODO check if could removed + for n, m in self.model.named_modules(): # TODO check if could removed m.tmp_name = n self._check_compatibility() formats = self.formats if hasattr(self, "formats") else None @@ -1827,11 +1815,19 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: self.model = _handle_moe_model(self.model, formats=formats) # self.has_qlayer_outside_block = self._set_layerwise_config(self.model, self.layer_config) # TODO check scale_dtype - self.layer_config, self.has_qlayer_outside_block = ( - self._prepare_layer_config(self.model, self.layer_config,self.scheme, self.scale_dtype, - self.supported_types,self.inner_supported_types,self.quant_block_list, - self.fp_layers,self.quant_lm_head, - enable_gguf_official_mixed=True,is_mllm=self.mllm)) + self.layer_config, self.has_qlayer_outside_block = self._prepare_layer_config( + self.model, + self.layer_config, + self.scheme, + self.scale_dtype, + self.supported_types, + self.inner_supported_types, + self.quant_block_list, + self.fp_layers, + self.quant_lm_head, + enable_gguf_official_mixed=True, + is_mllm=self.mllm, + ) if not hasattr(self, "formats"): logger.warning("this API is deprecated, please use `quantize_and_save` instead") diff --git a/auto_round/utils.py b/auto_round/utils.py index 3d35d303e..cac86e397 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -766,8 +766,10 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs): def get_layer_names_in_block( - model:torch.nn.Module, supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), - quant_block_list:list=None, class_names:tuple=None + model: torch.nn.Module, + supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D), + quant_block_list: list = None, + class_names: tuple = None, ) -> list[str]: """Retrieves the names of layers within each block of the model. @@ -1842,7 +1844,7 @@ def _gguf_type_fallback(gguf_type): ##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129 -def get_layer_config_by_gguf_format(layer_config, target_gguf_format:str, model, model_type=ModelType.TEXT): +def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT): # # TODO: support for other format later # target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None) From c6b78c6ad7276a2ba207d506e6857ba282b507ea Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 29 Sep 2025 14:56:09 +0800 Subject: [PATCH 23/88] tiny change --- auto_round/compressors/base.py | 41 +++++++++++----------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index c5928da60..931f18197 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -413,10 +413,10 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: else: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") - def _prepare_layer_config( + def _set_layer_config( self, model: torch.nn.Module, - layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], + layer_config: dict[str, str | dict | "QuantizationScheme"], default_scheme: "QuantizationScheme", default_scale_dtype: torch.dtype | str, supported_types: tuple, @@ -523,15 +523,15 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str # 7. lm_head lm_head_name = get_lm_head_name(model) - tied_lm_head = False - if hasattr(model, "config") and model.config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"): - tied_keys = model._tied_weights_keys - if lm_head_name in tied_keys: - tied_lm_head = True - if quant_lm_head and tied_lm_head: + tie_word_embeddings = False + if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"): + tie_word_embeddings = model.config.tie_word_embeddings + + if quant_lm_head and tie_word_embeddings: quant_lm_head = False logger.warning( - "reset `quant_lm_head` to false as quantizing lm_head with tied weights has not been supported currently" + "reset `quant_lm_head` to false as quantizing " + "lm_head with tied weights has not been supported currently" ) if lm_head_name not in layer_config and quant_lm_head: @@ -566,7 +566,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str return layer_config, has_qlayer_outside_block # embed + lm_head defaults for gguf - if lm_head_name not in layer_config and not tied_lm_head: + if lm_head_name not in layer_config and not tie_word_embeddings: cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]] cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype} layer_config[lm_head_name] = cfg @@ -1813,9 +1813,9 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: # It is best to modify the model structure in the quantize function and check the format, # because it may cause the gguf format to not be exported normally. self.model = _handle_moe_model(self.model, formats=formats) - # self.has_qlayer_outside_block = self._set_layerwise_config(self.model, self.layer_config) + # TODO check scale_dtype - self.layer_config, self.has_qlayer_outside_block = self._prepare_layer_config( + self.layer_config, self.has_qlayer_outside_block = self._set_layer_config( self.model, self.layer_config, self.scheme, @@ -1832,21 +1832,6 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: if not hasattr(self, "formats"): logger.warning("this API is deprecated, please use `quantize_and_save` instead") else: - only_gguf = True - for format_ in self.formats: - if not ("gguf" in format_ or "fake" in format_): - only_gguf = False - break - if len(self.formats) == 1 and self.formats[0] == "fake": - only_gguf = False - # if only_gguf: - # self.layer_config, gguf_format_config = get_layer_config_by_gguf_format( - # self.layer_config, self.formats, self.model, model_type=ModelType.TEXT - # ) - # if self.mllm: - # self.layer_config, gguf_format_config = get_layer_config_by_gguf_format( - # self.layer_config, self.formats, self.model, model_type=ModelType.MMPROJ - # ) # Determine if immediate packing is required formats = self.formats if ( @@ -1958,7 +1943,7 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: cost_time = end_time - self.start_time logger.info(f"quantization tuning time {cost_time}") - ## dump a summary + # Dump a summary quantized_layers = [] unquantized_layers = [] for n, m in self.model.named_modules(): From 1b9f24e8fde62ffa7f0e0e8321b69ae3c6d6479e Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 29 Sep 2025 14:57:58 +0800 Subject: [PATCH 24/88] tiny fix --- auto_round/schemes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_round/schemes.py b/auto_round/schemes.py index f6ca0cc98..8dde95430 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -236,7 +236,7 @@ def is_preset_scheme(name: str) -> bool: PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value) -def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> bool: +def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: if isinstance(scheme, str) and scheme.upper().startswith("GGUF"): return True for key, val in PRESET_SCHEMES.items(): From 2c0075ae48c98d095ef68515af51f509c94af1be Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 29 Sep 2025 15:11:58 +0800 Subject: [PATCH 25/88] tmp change --- auto_round/__main__.py | 2 +- auto_round/compressors/base.py | 6 ++++-- auto_round/schemes.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 97e3eb6ff..a25fc5421 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -110,7 +110,7 @@ def __init__(self, *args, **kwargs): self.add_argument( "--scale_dtype", - default="fp16", + default=None, choices=["fp16", "float16", "bf16", "bfloat16", "fp32", "float32"], help="scale data type to use for quantization", ) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 931f18197..281efff46 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -202,6 +202,10 @@ def __init__( ... } """ self.scheme = self._parse_and_set_scheme(scheme, kwargs) + + gguf_scheme_name = get_gguf_scheme(self.scheme) + # GGUF uses fp32 scale dtype as default + scale_dtype = kwargs.pop("scale_dtype", "fp32") if gguf_scheme_name else kwargs.pop("scale_dtype", "fp16") # Extra/legacy kwargs for backward compatibility # Major version releases may pack them with extra configuration options amp = kwargs.pop("amp", True) @@ -214,7 +218,6 @@ def __init__( sampler = kwargs.pop("sampler", "rand") not_use_best_mse = kwargs.pop("not_use_best_mse", False) dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1) - scale_dtype = kwargs.pop("scale_dtype", "fp16") nblocks = kwargs.pop("nblocks", 1) low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False) to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None) @@ -287,7 +290,6 @@ def __init__( self.device_map = None self._set_device_map_in_blocks(self.device_map) - # self._parse_layer_config(layer_config, fp_layers) # Must place after model init self.to_quant_block_names = to_quant_block_names diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 8dde95430..c5513d79a 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -244,7 +244,7 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: continue if val == scheme: return key - return None + return "" @dataclass From 97198f07b6fe660e6fba8b63c224ddb7440441d1 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 29 Sep 2025 15:40:49 +0800 Subject: [PATCH 26/88] tmp change --- auto_round/auto_schemes/utils.py | 5 ++++ auto_round/compressors/base.py | 39 +++++++++--------------------- auto_round/schemes.py | 4 +-- test/test_cuda/test_auto_scheme.py | 33 +++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 29 deletions(-) create mode 100644 auto_round/auto_schemes/utils.py create mode 100644 test/test_cuda/test_auto_scheme.py diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py new file mode 100644 index 000000000..fdcd343e7 --- /dev/null +++ b/auto_round/auto_schemes/utils.py @@ -0,0 +1,5 @@ +def get_total_bits(model, layer_config): + pass + +def get_bits(layer): + pass diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 281efff46..8477b2ed7 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -201,7 +201,10 @@ def __init__( ... # ... ... } """ - self.scheme = self._parse_and_set_scheme(scheme, kwargs) + if isinstance(scheme, AutoScheme): #TODO AutoScheme could also be patched by group_size, etc + self.scheme = self._parse_and_set_scheme(scheme.options[0], kwargs) + else: + self.scheme = self._parse_and_set_scheme(scheme, kwargs) gguf_scheme_name = get_gguf_scheme(self.scheme) # GGUF uses fp32 scale dtype as default @@ -271,6 +274,12 @@ def __init__( self.tokenizer = tokenizer self.shared_cache_keys = get_shared_keys(self.model) + self.to_quant_block_names = to_quant_block_names + if not hasattr(self, "quant_block_list"): + all_blocks = get_block_names(model) + self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names) + + if device is not None: logger.warning("`device` is deprecated, please use `device_map` instead") @@ -290,9 +299,6 @@ def __init__( self.device_map = None self._set_device_map_in_blocks(self.device_map) - - self.to_quant_block_names = to_quant_block_names - # Set device, must place after model loading self._set_device(device_map) @@ -342,27 +348,6 @@ def __init__( if self.static_kv_dtype is not None: logger.warning("The static kv is experimental and currently has limited support.") - # Model related - self.quantized = False - if isinstance(model, str): - model, tokenizer, low_cpu_mem_usage = llm_load_model( - model, device=device, low_cpu_mem_mode=low_cpu_mem_usage - ) - elif tokenizer is None and iters > 0: - raise ValueError("A tokenizer must be set for non-str model input") - self.low_cpu_mem_usage = bool(low_cpu_mem_usage) - if unsupported_meta_device(model): - raise RuntimeError( - "AutoRound does not support parameters on meta device. " - "Please use more GPUs by setting `--device_map 0,1,2,3` or just place the model on CPU." - ) - self.model = model.eval() - self.tokenizer = tokenizer - self.shared_cache_keys = get_shared_keys(self.model) - if not hasattr(self, "quant_block_list"): - all_blocks = get_block_names(model) - self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names) - self.scale_dtype = convert_dtype_str2torch(scale_dtype) self._set_amp_dtype() self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device @@ -418,7 +403,7 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: def _set_layer_config( self, model: torch.nn.Module, - layer_config: dict[str, str | dict | "QuantizationScheme"], + layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], default_scheme: "QuantizationScheme", default_scale_dtype: torch.dtype | str, supported_types: tuple, @@ -558,7 +543,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str for cfg in layer_config.values(): if "in_blocks" not in cfg: cfg["in_blocks"] = False - # 如果 layer 不在 blocks 且需要量化,则标记存在 blocks 外的量化层 + # mark layer outside block if not cfg["in_blocks"] and check_to_quantized(cfg): has_qlayer_outside_block = True diff --git a/auto_round/schemes.py b/auto_round/schemes.py index c5513d79a..ee12607eb 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -249,7 +249,7 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: @dataclass class AutoScheme: - options: Optional[Iterable[QuantizationScheme]] + options: Optional[Iterable[QuantizationScheme|str]] target_bits: float shared_layers: Optional[Iterable[Iterable[str]]] = None - method: str = "naive_pre" + method: str = "default" diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py new file mode 100644 index 000000000..4fd2e9c8b --- /dev/null +++ b/test/test_cuda/test_auto_scheme.py @@ -0,0 +1,33 @@ +import copy +import re +import shutil +import sys +import unittest + +sys.path.insert(0, "../..") +import torch +import transformers +from lm_eval.utils import make_table # pylint: disable=E0401 +from transformers import AutoModelForCausalLM, AutoTokenizer + +from auto_round import AutoRound, AutoRoundConfig,AutoScheme +from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model +from auto_round.testing_utils import require_autogptq, require_greater_than_050, require_greater_than_051 + +class TestAutoScheme(unittest.TestCase): + @classmethod + def setUpClass(self): + self.save_dir = "./saved" + self.tasks = "lambada_openai" + + @classmethod + def tearDownClass(self): + shutil.rmtree("./saved", ignore_errors=True) + shutil.rmtree("runs", ignore_errors=True) + + + def test_auto_scheme(self): + model_name = "facebook/opt-125m" + scheme = AutoScheme(target_bits=3, options=("W2A16","W4A16","BF16")) + ar = AutoRound(model_name=model_name,scheme=scheme) + ar.quantize_and_save(self.save_dir) From 27b4b4da882966b06ee750fe93e5fe4db3694bde Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 07:43:18 +0000 Subject: [PATCH 27/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/utils.py | 16 ++++++++++++++++ auto_round/compressors/base.py | 3 +-- auto_round/schemes.py | 2 +- test/test_cuda/test_auto_scheme.py | 8 ++++---- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index fdcd343e7..e01da9913 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -1,5 +1,21 @@ +# Copyright (c) 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + def get_total_bits(model, layer_config): pass + def get_bits(layer): pass diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 8477b2ed7..f7cd773d8 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -201,7 +201,7 @@ def __init__( ... # ... ... } """ - if isinstance(scheme, AutoScheme): #TODO AutoScheme could also be patched by group_size, etc + if isinstance(scheme, AutoScheme): # TODO AutoScheme could also be patched by group_size, etc self.scheme = self._parse_and_set_scheme(scheme.options[0], kwargs) else: self.scheme = self._parse_and_set_scheme(scheme, kwargs) @@ -279,7 +279,6 @@ def __init__( all_blocks = get_block_names(model) self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names) - if device is not None: logger.warning("`device` is deprecated, please use `device_map` instead") diff --git a/auto_round/schemes.py b/auto_round/schemes.py index ee12607eb..38bed87e1 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -249,7 +249,7 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: @dataclass class AutoScheme: - options: Optional[Iterable[QuantizationScheme|str]] + options: Optional[Iterable[QuantizationScheme | str]] target_bits: float shared_layers: Optional[Iterable[Iterable[str]]] = None method: str = "default" diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 4fd2e9c8b..6376e92c2 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -10,10 +10,11 @@ from lm_eval.utils import make_table # pylint: disable=E0401 from transformers import AutoModelForCausalLM, AutoTokenizer -from auto_round import AutoRound, AutoRoundConfig,AutoScheme +from auto_round import AutoRound, AutoRoundConfig, AutoScheme from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model from auto_round.testing_utils import require_autogptq, require_greater_than_050, require_greater_than_051 + class TestAutoScheme(unittest.TestCase): @classmethod def setUpClass(self): @@ -25,9 +26,8 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) - def test_auto_scheme(self): model_name = "facebook/opt-125m" - scheme = AutoScheme(target_bits=3, options=("W2A16","W4A16","BF16")) - ar = AutoRound(model_name=model_name,scheme=scheme) + scheme = AutoScheme(target_bits=3, options=("W2A16", "W4A16", "BF16")) + ar = AutoRound(model_name=model_name, scheme=scheme) ar.quantize_and_save(self.save_dir) From 2d3095a05368e3c082861df49d0cff4c0b7855c2 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 29 Sep 2025 16:36:52 +0800 Subject: [PATCH 28/88] update --- auto_round/auto_schemes/gen_scheme.py | 19 ++ auto_round/compressors/base.py | 261 +++----------------------- auto_round/schemes.py | 2 +- auto_round/utils.py | 177 ++++++++++++++++- test/test_cuda/test_auto_scheme.py | 11 +- 5 files changed, 224 insertions(+), 246 deletions(-) create mode 100644 auto_round/auto_schemes/gen_scheme.py diff --git a/auto_round/auto_schemes/gen_scheme.py b/auto_round/auto_schemes/gen_scheme.py new file mode 100644 index 000000000..badf39742 --- /dev/null +++ b/auto_round/auto_schemes/gen_scheme.py @@ -0,0 +1,19 @@ +from typing import Union, Iterable + +import torch + +from auto_round import AutoScheme + + +class GenScheme: + def __init__(self, + auto_scheme: AutoScheme, + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme:dict[str, dict], + scale_dtype: str = "fp16", + dataset="pile-10k" + ): + pass + + diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index f7cd773d8..495cf3d04 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -93,7 +93,7 @@ set_module, to_device, to_dtype, - unsupported_meta_device, + unsupported_meta_device, set_layer_config, ) from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block @@ -236,6 +236,9 @@ def __init__( self.mem_per_param_scale = kwargs.pop("mem_per_param_scale", None) self.fp_layers = kwargs.pop("fp_layers", "") self.layer_config = layer_config + self.supported_types = SUPPORTED_LAYER_TYPES + self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES + self.scale_dtype = convert_dtype_str2torch(scale_dtype) if kwargs: logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.") @@ -285,6 +288,28 @@ def __init__( if device_map is None: device_map = 0 + if isinstance(scheme, AutoScheme): + if self.mllm: + logger.info("AutoScheme with MLLM is not supported yet.") + sys.exit(1) + layer_config,_ = set_layer_config(self.model, + self.layer_config, + self.scheme, + self.scale_dtype, + self.supported_types, + self.inner_supported_types, + self.quant_block_list, + self.fp_layers, + self.quant_lm_head, + enable_gguf_official_mixed=False, + is_mllm=self.mllm) + quant_layer_names = layer_config.keys() + fixed_layer_scheme = {k: v for k, v in layer_config.items() if v.get("fixed_by_user", False)} + # mainly using quant_layers and fixed by users + from auto_round.auto_schemes.gen_scheme import GenScheme + gen_scheme = GenScheme(scheme,self.model,quant_layer_names,fixed_layer_scheme, self.scale_dtype, self.dataset) + + # Set device, must place after model loading self._set_device(device_map) @@ -347,7 +372,6 @@ def __init__( if self.static_kv_dtype is not None: logger.warning("The static kv is experimental and currently has limited support.") - self.scale_dtype = convert_dtype_str2torch(scale_dtype) self._set_amp_dtype() self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device if self.act_bits <= 8 and self.amp_dtype == torch.float16: @@ -359,8 +383,6 @@ def __init__( logger.info(f"using {self.model.dtype} for quantization tuning") # Some helpers - self.supported_types = SUPPORTED_LAYER_TYPES - self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES if "hpu" in str(self.device): self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear") self.batch_dim = None @@ -399,235 +421,6 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: else: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") - def _set_layer_config( - self, - model: torch.nn.Module, - layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], - default_scheme: "QuantizationScheme", - default_scale_dtype: torch.dtype | str, - supported_types: tuple, - inner_supported_types: tuple, - quant_block_list=None, - fp_layers: str = "", - quant_lm_head: bool = False, - enable_gguf_official_mixed: bool = True, - is_mllm: bool = False, - ) -> tuple[dict, bool]: - """ - Normalize, validate, and expand layer-specific quantization configs. - Returns (final_layer_config, has_quant_layer_outside_block) - """ - - from auto_round.schemes import get_gguf_scheme - - # ---- helpers ------------------------------------------------- - def dispatch_layer_config(layer_config: dict[str, dict]) -> None: - """Assign scheme values as attributes to matched modules.""" - for layer_name, scheme in layer_config.items(): - module = get_module(model, layer_name) - for attr, value in scheme.items(): - setattr(module, attr, value) - - def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict: - """Convert config entry into dict and validate keys.""" - if isinstance(item, str): - config = asdict(preset_name_to_scheme(item.upper())) - elif isinstance(item, QuantizationScheme): - config = asdict(item) - elif isinstance(item, dict): - invalid = set(item) - set(scheme_keys) - if invalid: - raise ValueError( - f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}" - ) - config = dict(item) - else: - raise TypeError( - f"Unsupported type for layer_config[{layer_name}]: {type(item)}. " - f"Expected str, dict, or QuantizationScheme." - ) - # Clean up - config = {k: v for k, v in config.items() if v is not None} - config["fixed_by_user"] = True - return config - - # ---- main logic ---------------------------------------------- - scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",) - layer_config = copy.deepcopy(layer_config) or {} - - # 1. fp_layers -> force 16 - for name in get_fp_layer_names(self.model, fp_layers): - layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"} - - # 2. normalize - layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()} - - # 3. infer missing bits - for cfg in layer_config.values(): - if "data_type" in cfg and "bits" not in cfg: - if (b := infer_bits_by_data_type(cfg["data_type"])) is not None: - cfg["bits"] = b - if "act_data_type" in cfg and "act_bits" not in cfg: - if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None: - cfg["act_bits"] = b - - # 4. fill defaults - default_dict = asdict(default_scheme) - default_dict["scale_dtype"] = default_scale_dtype - for cfg in layer_config.values(): - for key in scheme_keys: - cfg.setdefault(key, default_dict.get(key)) - - # 5. collect supported modules - gguf_name = get_gguf_scheme(default_scheme) - if gguf_name and torch.nn.Embedding not in supported_types: - supported_types = (*supported_types, torch.nn.Embedding) - - all_layer_names, embedding_layer_names = [], [] - for n, m in model.named_modules(): - # cleanup stale attributes - for key in scheme_keys: - if hasattr(m, key): - delattr(m, key) - if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types: - continue - all_layer_names.append(n) - if isinstance(m, torch.nn.Embedding): - embedding_layer_names.append(n) - - # 6. expand regex configs - for name in list(layer_config.keys()): - if name in all_layer_names: - continue - regex = re.compile(name) - matched = [ln for ln in all_layer_names if regex.search(ln)] - if not matched: - raise ValueError(f"Invalid regex '{name}' in layer_config, no match found.") - val = layer_config.pop(name) - for match in matched: - layer_config[match] = val - - # 7. lm_head - lm_head_name = get_lm_head_name(model) - tie_word_embeddings = False - if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"): - tie_word_embeddings = model.config.tie_word_embeddings - - if quant_lm_head and tie_word_embeddings: - quant_lm_head = False - logger.warning( - "reset `quant_lm_head` to false as quantizing " - "lm_head with tied weights has not been supported currently" - ) - - if lm_head_name not in layer_config and quant_lm_head: - layer_config[lm_head_name] = default_dict.copy() - - # 8. enforce shape divisibility for int weight-only - if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name: - for n, m in model.named_modules(): - if type(m) in supported_types or m.__class__.__name__ in inner_supported_types: - if m.weight.shape[0] % 32 or m.weight.shape[1] % 32: - layer_config.setdefault(n, default_dict.copy()) - layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True}) - logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).") - - # 9. block layers: mark as in_blocks=True - for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types): - cfg = layer_config.setdefault(name, default_dict.copy()) - cfg["in_blocks"] = True - - # ---- restore: ensure missing in_blocks are set to False and compute flag ---- - has_qlayer_outside_block = False - for cfg in layer_config.values(): - if "in_blocks" not in cfg: - cfg["in_blocks"] = False - # mark layer outside block - if not cfg["in_blocks"] and check_to_quantized(cfg): - has_qlayer_outside_block = True - - # 10. GGUF handling - if not gguf_name: - dispatch_layer_config(layer_config) - return layer_config, has_qlayer_outside_block - - # embed + lm_head defaults for gguf - if lm_head_name not in layer_config and not tie_word_embeddings: - cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]] - cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype} - layer_config[lm_head_name] = cfg - has_qlayer_outside_block = True - for emd_name in embedding_layer_names: - cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]] - cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype} - layer_config[emd_name] = cfg - - if enable_gguf_official_mixed: - model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT - layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type) - - dispatch_layer_config(layer_config) - return layer_config, has_qlayer_outside_block - - def _parse_layer_config( - self, layer_config: dict[str, Union[str, dict, QuantizationScheme]], fp_layers: str - ) -> None: - """Parse and set the layer-wise quantization configuration.""" - not_quantize_layer_names = get_fp_layer_names(self.model, fp_layers) - if len(not_quantize_layer_names) > 0: - logger.info(f"{not_quantize_layer_names} will not be quantized.") - if layer_config is None: - layer_config = {} - for name in not_quantize_layer_names: - layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float"} - - # Some other quantization configs - self.layer_config = copy.deepcopy(layer_config) if layer_config is not None else {} - scheme_keys = {f.name for f in fields(QuantizationScheme)} - - for key, item in self.layer_config.items(): - if isinstance(item, str): - config = asdict(preset_name_to_scheme(item.upper())) - elif isinstance(item, QuantizationScheme): - config = asdict(item) - elif isinstance(item, dict): - invalid_keys = set(item) - scheme_keys - if invalid_keys: - raise ValueError( - f"Invalid keys {invalid_keys} in layer_config for layer '{key}', " - f"only {scheme_keys} are supported" - ) - config = dict(item) - - # Drop None values - config = {k: v for k, v in config.items() if v is not None} - self.layer_config[key] = config - - if not self.quant_lm_head or (isinstance(self.scheme, str) and self.scheme.lower().startswith("gguf")): - return - for n, _ in self.model.named_modules(): - lm_head_layer_name = n - - if ( - hasattr(self.model, "config") - and self.model.config.tie_word_embeddings - and hasattr(self.model, "_tied_weights_keys") - ): - tied_keys = self.model._tied_weights_keys - for item in tied_keys: - if lm_head_layer_name in item: # TODO extend to encoder-decoder layer, seq classification model - self.quant_lm_head = False - logger.warning( - "reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been " - "supported currently" - ) - break - - lm_head_layer_config = self.layer_config[lm_head_layer_name] if lm_head_layer_name in self.layer_config else {} - - for key in scheme_keys: - if key not in lm_head_layer_config: - lm_head_layer_config[key] = getattr(self, key) def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> QuantizationScheme: """Parse and set the quantization scheme.""" @@ -1801,7 +1594,7 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: self.model = _handle_moe_model(self.model, formats=formats) # TODO check scale_dtype - self.layer_config, self.has_qlayer_outside_block = self._set_layer_config( + self.layer_config, self.has_qlayer_outside_block = set_layer_config( self.model, self.layer_config, self.scheme, diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 38bed87e1..cf7d4d433 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -250,6 +250,6 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: @dataclass class AutoScheme: options: Optional[Iterable[QuantizationScheme | str]] - target_bits: float + avg_bits: float shared_layers: Optional[Iterable[Iterable[str]]] = None method: str = "default" diff --git a/auto_round/utils.py b/auto_round/utils.py index cac86e397..92e54a3ba 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -21,7 +21,7 @@ import re import sys from collections import UserDict -from dataclasses import fields +from dataclasses import fields, asdict from enum import Enum from functools import lru_cache from pathlib import Path @@ -35,7 +35,7 @@ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType from auto_round.logger import logger -from auto_round.schemes import QuantizationScheme +from auto_round.schemes import QuantizationScheme, preset_name_to_scheme SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings") @@ -2742,3 +2742,176 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module]): return True return False + + + +def set_layer_config( + model: torch.nn.Module, + layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], + default_scheme: "QuantizationScheme", + default_scale_dtype: torch.dtype | str, + supported_types: tuple, + inner_supported_types: tuple, + quant_block_list=None, + fp_layers: str = "", + quant_lm_head: bool = False, + enable_gguf_official_mixed: bool = True, + is_mllm: bool = False, +) -> tuple[dict, bool]: + """ + Normalize, validate, and expand layer-specific quantization configs. + Returns (final_layer_config, has_quant_layer_outside_block) + """ + + from auto_round.schemes import get_gguf_scheme + + # ---- helpers ------------------------------------------------- + def dispatch_layer_config(layer_config: dict[str, dict]) -> None: + """Assign scheme values as attributes to matched modules.""" + for layer_name, scheme in layer_config.items(): + module = get_module(model, layer_name) + for attr, value in scheme.items(): + setattr(module, attr, value) + + def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict: + """Convert config entry into dict and validate keys.""" + if isinstance(item, str): + config = asdict(preset_name_to_scheme(item.upper())) + elif isinstance(item, QuantizationScheme): + config = asdict(item) + elif isinstance(item, dict): + invalid = set(item) - set(scheme_keys) + if invalid: + raise ValueError( + f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}" + ) + config = dict(item) + else: + raise TypeError( + f"Unsupported type for layer_config[{layer_name}]: {type(item)}. " + f"Expected str, dict, or QuantizationScheme." + ) + # Clean up + config = {k: v for k, v in config.items() if v is not None} + config["fixed_by_user"] = True + return config + + # ---- main logic ---------------------------------------------- + scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",) + layer_config = copy.deepcopy(layer_config) or {} + + # 1. fp_layers -> force 16 + for name in get_fp_layer_names(model, fp_layers): + layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float","fixed_by_user":True} + + # 2. normalize + layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()} + + # 3. infer missing bits + for cfg in layer_config.values(): + if "data_type" in cfg and "bits" not in cfg: + if (b := infer_bits_by_data_type(cfg["data_type"])) is not None: + cfg["bits"] = b + if "act_data_type" in cfg and "act_bits" not in cfg: + if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None: + cfg["act_bits"] = b + + # 4. fill defaults + default_dict = asdict(default_scheme) + default_dict["scale_dtype"] = default_scale_dtype + for cfg in layer_config.values(): + for key in scheme_keys: + cfg.setdefault(key, default_dict.get(key)) + + # 5. collect supported modules + gguf_name = get_gguf_scheme(default_scheme) + if gguf_name and torch.nn.Embedding not in supported_types: + supported_types = (*supported_types, torch.nn.Embedding) + + all_layer_names, embedding_layer_names = [], [] + for n, m in model.named_modules(): + # cleanup stale attributes + for key in scheme_keys: + if hasattr(m, key): + delattr(m, key) + if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types: + continue + all_layer_names.append(n) + if isinstance(m, torch.nn.Embedding): + embedding_layer_names.append(n) + + # 6. expand regex configs + for name in list(layer_config.keys()): + if name in all_layer_names: + continue + regex = re.compile(name) + matched = [ln for ln in all_layer_names if regex.search(ln)] + if not matched: + raise ValueError(f"Invalid '{name}' in layer_config, no match found.") + val = layer_config.pop(name) + for match in matched: + layer_config[match] = val + + # 7. lm_head + lm_head_name = get_lm_head_name(model) + tie_word_embeddings = False + if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"): + tie_word_embeddings = model.config.tie_word_embeddings + + if quant_lm_head and tie_word_embeddings: + quant_lm_head = False + logger.warning( + "reset `quant_lm_head` to false as quantizing " + "lm_head with tied weights has not been supported currently" + ) + + if lm_head_name not in layer_config and quant_lm_head: + layer_config[lm_head_name] = default_dict.copy() + + # 8. enforce shape divisibility for int weight-only + if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name: + for n, m in model.named_modules(): + if type(m) in supported_types or m.__class__.__name__ in inner_supported_types: + if m.weight.shape[0] % 32 or m.weight.shape[1] % 32: + layer_config.setdefault(n, default_dict.copy()) + layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True}) + logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).") + + # 9. block layers: mark as in_blocks=True + for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types): + if name not in layer_config: + layer_config[name] = default_dict.copy() + layer_config[name]["fixed_by_user"]=False + layer_config[name]["in_blocks"] = True + + # ---- restore: ensure missing in_blocks are set to False and compute flag ---- + has_qlayer_outside_block = False + for cfg in layer_config.values(): + if "in_blocks" not in cfg: + cfg["in_blocks"] = False + # mark layer outside block + if not cfg["in_blocks"] and check_to_quantized(cfg): + has_qlayer_outside_block = True + + # 10. GGUF handling + if not gguf_name: + dispatch_layer_config(layer_config) + return layer_config, has_qlayer_outside_block + + # embed + lm_head defaults for gguf + if lm_head_name not in layer_config and not tie_word_embeddings: + cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]] + cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype} + layer_config[lm_head_name] = cfg + has_qlayer_outside_block = True + for emd_name in embedding_layer_names: + cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]] + cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype} + layer_config[emd_name] = cfg + + if enable_gguf_official_mixed: + model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT + layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type) + + dispatch_layer_config(layer_config) + return layer_config, has_qlayer_outside_block diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 6376e92c2..b4f5e6041 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -5,15 +5,8 @@ import unittest sys.path.insert(0, "../..") -import torch -import transformers -from lm_eval.utils import make_table # pylint: disable=E0401 -from transformers import AutoModelForCausalLM, AutoTokenizer from auto_round import AutoRound, AutoRoundConfig, AutoScheme -from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model -from auto_round.testing_utils import require_autogptq, require_greater_than_050, require_greater_than_051 - class TestAutoScheme(unittest.TestCase): @classmethod @@ -28,6 +21,6 @@ def tearDownClass(self): def test_auto_scheme(self): model_name = "facebook/opt-125m" - scheme = AutoScheme(target_bits=3, options=("W2A16", "W4A16", "BF16")) - ar = AutoRound(model_name=model_name, scheme=scheme) + scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=1, nsamples=1) ar.quantize_and_save(self.save_dir) From 35a298b0f30c57df5e5af1808d3330538371c237 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 08:37:38 +0000 Subject: [PATCH 29/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/gen_scheme.py | 35 ++++++++++++++++++--------- auto_round/compressors/base.py | 32 +++++++++++++----------- auto_round/utils.py | 16 +++++++----- test/test_cuda/test_auto_scheme.py | 1 + 4 files changed, 53 insertions(+), 31 deletions(-) diff --git a/auto_round/auto_schemes/gen_scheme.py b/auto_round/auto_schemes/gen_scheme.py index badf39742..ba6b0a679 100644 --- a/auto_round/auto_schemes/gen_scheme.py +++ b/auto_round/auto_schemes/gen_scheme.py @@ -1,4 +1,18 @@ -from typing import Union, Iterable +# Copyright (c) 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable, Union import torch @@ -6,14 +20,13 @@ class GenScheme: - def __init__(self, - auto_scheme: AutoScheme, - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme:dict[str, dict], - scale_dtype: str = "fp16", - dataset="pile-10k" - ): + def __init__( + self, + auto_scheme: AutoScheme, + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scale_dtype: str = "fp16", + dataset="pile-10k", + ): pass - - diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 495cf3d04..66c79274f 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -90,10 +90,11 @@ mv_module_from_gpu, reset_params, set_amax_for_all_moe_layers, + set_layer_config, set_module, to_device, to_dtype, - unsupported_meta_device, set_layer_config, + unsupported_meta_device, ) from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block @@ -292,23 +293,27 @@ def __init__( if self.mllm: logger.info("AutoScheme with MLLM is not supported yet.") sys.exit(1) - layer_config,_ = set_layer_config(self.model, - self.layer_config, - self.scheme, - self.scale_dtype, - self.supported_types, - self.inner_supported_types, - self.quant_block_list, - self.fp_layers, - self.quant_lm_head, - enable_gguf_official_mixed=False, - is_mllm=self.mllm) + layer_config, _ = set_layer_config( + self.model, + self.layer_config, + self.scheme, + self.scale_dtype, + self.supported_types, + self.inner_supported_types, + self.quant_block_list, + self.fp_layers, + self.quant_lm_head, + enable_gguf_official_mixed=False, + is_mllm=self.mllm, + ) quant_layer_names = layer_config.keys() fixed_layer_scheme = {k: v for k, v in layer_config.items() if v.get("fixed_by_user", False)} # mainly using quant_layers and fixed by users from auto_round.auto_schemes.gen_scheme import GenScheme - gen_scheme = GenScheme(scheme,self.model,quant_layer_names,fixed_layer_scheme, self.scale_dtype, self.dataset) + gen_scheme = GenScheme( + scheme, self.model, quant_layer_names, fixed_layer_scheme, self.scale_dtype, self.dataset + ) # Set device, must place after model loading self._set_device(device_map) @@ -421,7 +426,6 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: else: raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}") - def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> QuantizationScheme: """Parse and set the quantization scheme.""" if isinstance(scheme, QuantizationScheme): diff --git a/auto_round/utils.py b/auto_round/utils.py index 92e54a3ba..e865726b8 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -21,7 +21,7 @@ import re import sys from collections import UserDict -from dataclasses import fields, asdict +from dataclasses import asdict, fields from enum import Enum from functools import lru_cache from pathlib import Path @@ -2744,7 +2744,6 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module]): return False - def set_layer_config( model: torch.nn.Module, layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], @@ -2802,7 +2801,13 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str # 1. fp_layers -> force 16 for name in get_fp_layer_names(model, fp_layers): - layer_config[name] = {"bits": 16, "act_bits": 16, "data_type": "float", "act_data_type": "float","fixed_by_user":True} + layer_config[name] = { + "bits": 16, + "act_bits": 16, + "data_type": "float", + "act_data_type": "float", + "fixed_by_user": True, + } # 2. normalize layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()} @@ -2861,8 +2866,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str if quant_lm_head and tie_word_embeddings: quant_lm_head = False logger.warning( - "reset `quant_lm_head` to false as quantizing " - "lm_head with tied weights has not been supported currently" + "reset `quant_lm_head` to false as quantizing " "lm_head with tied weights has not been supported currently" ) if lm_head_name not in layer_config and quant_lm_head: @@ -2881,7 +2885,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types): if name not in layer_config: layer_config[name] = default_dict.copy() - layer_config[name]["fixed_by_user"]=False + layer_config[name]["fixed_by_user"] = False layer_config[name]["in_blocks"] = True # ---- restore: ensure missing in_blocks are set to False and compute flag ---- diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index b4f5e6041..b9fffdee9 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -8,6 +8,7 @@ from auto_round import AutoRound, AutoRoundConfig, AutoScheme + class TestAutoScheme(unittest.TestCase): @classmethod def setUpClass(self): From 4a594cd5e778b2c5897e3131d72e21c7e46ba74f Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 29 Sep 2025 20:37:42 +0800 Subject: [PATCH 30/88] fix --- auto_round/compressors/base.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 66c79274f..955971306 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -725,20 +725,20 @@ def _check_compatibility(self) -> None: " We are likely to release new algorithm for certain configurations in the future." ) - # Check group_size 32 for auto_round - if ( - self.data_type == "int" - and hasattr(self, "formats") - and any(key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq")) - ): - for n, m in self.model.named_modules(): - if type(m) in self.supported_types: - if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0: - self.layer_config[n] = {"bits": 16} - logger.info( - f"{n} will not be quantized due to its shape not being divisible by 32," - " resulting in an exporting issue to autogptq" - ) + # # Check group_size 32 for auto_round + # if ( + # self.data_type == "int" + # and hasattr(self, "formats") + # and any(key in fmt for fmt in self.formats for key in ("auto_round", "auto_gptq", "auto_awq")) + # ): + # for n, m in self.model.named_modules(): + # if type(m) in self.supported_types: + # if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0: + # self.layer_config[n] = {"bits": 16} + # logger.info( + # f"{n} will not be quantized due to its shape not being divisible by 32," + # " resulting in an exporting issue to autogptq" + # ) if ( self.seqlen is not None From dcd08d629cc3840efdc24b8c9af97af2edf71095 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Tue, 30 Sep 2025 14:02:23 +0800 Subject: [PATCH 31/88] fix uts, still one left --- .../export/export_to_autoround/export_to_nvfp_mxfp.py | 4 ++-- auto_round/schemes.py | 9 +++++++-- auto_round/utils.py | 5 +++++ test/test_cpu/test_autoround.py | 1 + 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py index c4a02f673..240a94899 100644 --- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py +++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py @@ -174,7 +174,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs): for n, m in model.named_modules(): if type(m) in SUPPORTED_LAYER_TYPES: layer = m - if layer.act_bits < 8 and not getattr(layer, "input_global_scale", None): + if hasattr(layer,"act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None): assert hasattr(layer, "act_max") from auto_round.data_type.nvfp import calculate_gparam @@ -198,7 +198,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs): for layer_name in layer_config: if ( not layer_config[layer_name]["in_blocks"] and layer_config[layer_name]["bits"] <= 8 - ): ##lm head ##TODO fix act and so on + ): ##lm head # TODO fix act and so on extra_config[layer_name] = {} extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"] extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"] diff --git a/auto_round/schemes.py b/auto_round/schemes.py index cf7d4d433..32be2fb52 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -238,11 +238,16 @@ def is_preset_scheme(name: str) -> bool: def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: if isinstance(scheme, str) and scheme.upper().startswith("GGUF"): - return True + return scheme for key, val in PRESET_SCHEMES.items(): if not key.upper().startswith("GGUF"): continue - if val == scheme: + equal = True + for scheme_key in val.keys(): + if val[scheme_key] is not None and val[scheme_key] != scheme.get(scheme_key, None): + equal = False + break + if equal: return key return "" diff --git a/auto_round/utils.py b/auto_round/utils.py index e865726b8..009d516d8 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2834,7 +2834,9 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str supported_types = (*supported_types, torch.nn.Embedding) all_layer_names, embedding_layer_names = [], [] + all_module_names = [] for n, m in model.named_modules(): + all_module_names.append(n) # cleanup stale attributes for key in scheme_keys: if hasattr(m, key): @@ -2849,6 +2851,9 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str for name in list(layer_config.keys()): if name in all_layer_names: continue + if name in all_module_names: + logger.warning_once(f"the type of `{name}` is not supported in your scheme, ignore it for now.") + continue regex = re.compile(name) matched = [ln for ln in all_layer_names if regex.search(ln)] if not matched: diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py index 9511f0cf8..aac524800 100644 --- a/test/test_cpu/test_autoround.py +++ b/test/test_cpu/test_autoround.py @@ -720,6 +720,7 @@ def test_invalid_layer_config(self): iters=1, layer_config=layer_config, ) + ar.quantize() def test_quant_lm_head(self): model_name = "/tf_dataset/auto_round/models/Qwen/Qwen3-8B" From 91722646068c4adcdbfefe058e3f486b58793a6f Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Tue, 30 Sep 2025 15:19:25 +0800 Subject: [PATCH 32/88] fix gguf issue --- auto_round/compressors/base.py | 6 ++++-- auto_round/schemes.py | 2 ++ auto_round/utils.py | 37 ++++++++++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 955971306..0731d0ba8 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -428,11 +428,13 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> QuantizationScheme: """Parse and set the quantization scheme.""" + res= "" if isinstance(scheme, QuantizationScheme): scheme = asdict(scheme) elif isinstance(scheme, dict): scheme = scheme elif isinstance(scheme, str): + res = scheme # gguf:q4_k_s and gguf_q4_k_m has the same dict scheme, but the result is different scheme = scheme.upper() scheme = asdict(preset_name_to_scheme(scheme)) scheme_keys = [f.name for f in fields(QuantizationScheme)] @@ -481,7 +483,7 @@ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kw break for key in scheme_keys: scheme[key] = getattr(self, key) - return QuantizationScheme.from_dict(scheme) + return res if res else QuantizationScheme.from_dict(scheme) def _adjust_torch_compile(self, enable_torch_compile: bool) -> None: """Sets the torch compile configuration for the tuning.""" @@ -804,7 +806,7 @@ def remove_duplicates(lst): for f in formats: if f.startswith("gguf"): - self.scheme = preset_name_to_scheme(f) + self.scheme = f.upper() break for format_ in formats: diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 32be2fb52..cde37a0c9 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -239,6 +239,8 @@ def is_preset_scheme(name: str) -> bool: def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: if isinstance(scheme, str) and scheme.upper().startswith("GGUF"): return scheme + if isinstance(scheme, str): + return "" for key, val in PRESET_SCHEMES.items(): if not key.upper().startswith("GGUF"): continue diff --git a/auto_round/utils.py b/auto_round/utils.py index 009d516d8..84390fa43 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -35,7 +35,7 @@ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType from auto_round.logger import logger -from auto_round.schemes import QuantizationScheme, preset_name_to_scheme +from auto_round.schemes import QuantizationScheme, preset_name_to_scheme, get_gguf_scheme SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings") @@ -1940,6 +1940,30 @@ def _set_config(config, target_config): ) new_type = new_type[:bits_index] + target_bits + new_type[bits_index + 1 :] else: + config_tmp = config.copy() + scheme_keys = [f.name for f in fields(QuantizationScheme)] + for key in config.keys(): + if key not in scheme_keys: + config_tmp.pop(key, None) + matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched + if not matched_scheme: + if config.get("super_group_size", None) is not None: + new_type = new_type[:bits_index] + str(config["bits"]) + "_k" + if config.get("super_group_size", None) is None or new_type not in GGUF_INNER_CONFIG: + if config.get("sym", True): + new_type = new_type[:bits_index] + str(config["bits"]) + "_0" + if new_type not in GGUF_INNER_CONFIG: + new_type = new_type[:bits_index] + str(config["bits"]) + "_1" + if not config.get("sym", True): + new_type = new_type[:bits_index] + str(config["bits"]) + "_1" + if new_type not in GGUF_INNER_CONFIG: + new_type = new_type[:bits_index] + str(config["bits"]) + "_0" + if new_type not in GGUF_INNER_CONFIG: + raise ValueError(f"the setting in layer_config {layer_name} " + f"could not match any supported gguf format, please have a check.") + else: + logger.warning_once(f"the setting in layer_config {layer_name} " + f"could not match any supported gguf format, reset to {new_type}") new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :] new_type = _search_gguf_type(new_type) if new_type is None: @@ -2747,7 +2771,7 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module]): def set_layer_config( model: torch.nn.Module, layer_config: dict[str, Union[str, dict, "QuantizationScheme"]], - default_scheme: "QuantizationScheme", + default_scheme: Union[str, "QuantizationScheme"], default_scale_dtype: torch.dtype | str, supported_types: tuple, inner_supported_types: tuple, @@ -2822,11 +2846,14 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str cfg["act_bits"] = b # 4. fill defaults - default_dict = asdict(default_scheme) + if isinstance(default_scheme,str): + default_dict = asdict(preset_name_to_scheme(default_scheme.upper())) + else: + default_dict = asdict(default_scheme) default_dict["scale_dtype"] = default_scale_dtype for cfg in layer_config.values(): for key in scheme_keys: - cfg.setdefault(key, default_dict.get(key)) + cfg.setdefault(key, default_dict.copy().get(key)) # 5. collect supported modules gguf_name = get_gguf_scheme(default_scheme) @@ -2914,6 +2941,8 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str layer_config[lm_head_name] = cfg has_qlayer_outside_block = True for emd_name in embedding_layer_names: + if emd_name in layer_config: + continue cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]] cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype} layer_config[emd_name] = cfg From f98092c6e6b4e53e9d4653ec21fe4e7fa69a0ee3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Sep 2025 07:25:15 +0000 Subject: [PATCH 33/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/compressors/base.py | 5 ++--- .../export_to_autoround/export_to_nvfp_mxfp.py | 2 +- auto_round/utils.py | 18 +++++++++++------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index f957b2a5c..f1036ecac 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -328,7 +328,6 @@ def __init__( self.device_map = None self._set_device_map_in_blocks(self.device_map) - # Tuning hyperparameters self.seed = seed set_seed(self.seed) @@ -416,13 +415,13 @@ def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kwargs) -> QuantizationScheme: """Parse and set the quantization scheme.""" - res= "" + res = "" if isinstance(scheme, QuantizationScheme): scheme = asdict(scheme) elif isinstance(scheme, dict): scheme = scheme elif isinstance(scheme, str): - res = scheme # gguf:q4_k_s and gguf_q4_k_m has the same dict scheme, but the result is different + res = scheme # gguf:q4_k_s and gguf_q4_k_m has the same dict scheme, but the result is different scheme = scheme.upper() scheme = asdict(preset_name_to_scheme(scheme)) scheme_keys = [f.name for f in fields(QuantizationScheme)] diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py index 240a94899..eaf3ad9ae 100644 --- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py +++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py @@ -174,7 +174,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs): for n, m in model.named_modules(): if type(m) in SUPPORTED_LAYER_TYPES: layer = m - if hasattr(layer,"act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None): + if hasattr(layer, "act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None): assert hasattr(layer, "act_max") from auto_round.data_type.nvfp import calculate_gparam diff --git a/auto_round/utils.py b/auto_round/utils.py index 187fc883d..a1c411373 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -35,7 +35,7 @@ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType from auto_round.logger import logger -from auto_round.schemes import QuantizationScheme, preset_name_to_scheme, get_gguf_scheme +from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings") @@ -1949,7 +1949,7 @@ def _set_config(config, target_config): for key in config.keys(): if key not in scheme_keys: config_tmp.pop(key, None) - matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched + matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched if not matched_scheme: if config.get("super_group_size", None) is not None: new_type = new_type[:bits_index] + str(config["bits"]) + "_k" @@ -1963,11 +1963,15 @@ def _set_config(config, target_config): if new_type not in GGUF_INNER_CONFIG: new_type = new_type[:bits_index] + str(config["bits"]) + "_0" if new_type not in GGUF_INNER_CONFIG: - raise ValueError(f"the setting in layer_config {layer_name} " - f"could not match any supported gguf format, please have a check.") + raise ValueError( + f"the setting in layer_config {layer_name} " + f"could not match any supported gguf format, please have a check." + ) else: - logger.warning_once(f"the setting in layer_config {layer_name} " - f"could not match any supported gguf format, reset to {new_type}") + logger.warning_once( + f"the setting in layer_config {layer_name} " + f"could not match any supported gguf format, reset to {new_type}" + ) new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :] new_type = _search_gguf_type(new_type) if new_type is None: @@ -2850,7 +2854,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str cfg["act_bits"] = b # 4. fill defaults - if isinstance(default_scheme,str): + if isinstance(default_scheme, str): default_dict = asdict(preset_name_to_scheme(default_scheme.upper())) else: default_dict = asdict(default_scheme) From 033d1f6ed3e4b4a128a07c964af01b156c820704 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Tue, 30 Sep 2025 16:48:32 +0800 Subject: [PATCH 34/88] update a little --- auto_round/auto_schemes/gen_scheme.py | 66 ++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/auto_round/auto_schemes/gen_scheme.py b/auto_round/auto_schemes/gen_scheme.py index ba6b0a679..03c253a6a 100644 --- a/auto_round/auto_schemes/gen_scheme.py +++ b/auto_round/auto_schemes/gen_scheme.py @@ -17,16 +17,68 @@ import torch from auto_round import AutoScheme +from auto_round.utils import get_layer_features class GenScheme: def __init__( - self, - auto_scheme: AutoScheme, - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - scale_dtype: str = "fp16", - dataset="pile-10k", + self, + auto_scheme: AutoScheme, + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scale_dtype: str = "fp16", + dataset="pile-10k", ): + self.auto_scheme = auto_scheme + self.model = model + self.quant_layer_names = quant_layer_names + self.fixed_layer_scheme = fixed_layer_scheme + self.scale_dtype = scale_dtype + self.dataset = dataset + + def _get_min_max_avg_bits(self) -> tuple[float, float]: pass + + # not validate yet + def get_layer_bits(self, layer): + weight = layer.weight + n_param = weight.numel() + weight_bits = getattr(layer, 'bits', 16) + group_size = getattr(layer, 'group_size', 128) + super_group_size = getattr(layer, 'super_group_size', None) + super_weight_bits = getattr(layer, 'super_bits', None) + + # Main quantization cost + weight_total_bits = weight_bits * n_param + if weight_bits>=16: # Unquantized layer + return weight_total_bits, 16 + + in_features, output_features = get_layer_features(layer) + # Determine number of groups + if group_size > 0: # group-wise + n_group = output_features * (in_features + group_size - 1) // group_size + elif group_size == 0: # per-tensor + n_group = 1 + elif group_size == -1: # per-channel + n_group = output_features # out_channels + else: + raise ValueError(f"Invalid group_size {group_size}") + aux_total_bits = 0 + if not super_group_size: + # Scale and zero point bitwidths + scale_bits = 16 + zp_bits = weight_bits if not super_group_size else 32 # default: same as weight_bits + # Overhead from scales and zero points + aux_total_bits = n_group * (scale_bits + zp_bits) + + # Double quantization case + if super_group_size: + # Number of super-groups + aux_total_bits+=n_group*super_weight_bits * 2 #sclae and min int count + n_super_group = (n_group + super_group_size - 1) // super_group_size + aux_total_bits += n_super_group * 32 * 2 # double quant scale and min_v + + total_bits = weight_total_bits + aux_total_bits + avg_bits = total_bits / n_param + return total_bits, avg_bits From 8ae1dfa56220727b39c2c29eacb22a98ab998e10 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Sep 2025 08:50:03 +0000 Subject: [PATCH 35/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/gen_scheme.py | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/auto_round/auto_schemes/gen_scheme.py b/auto_round/auto_schemes/gen_scheme.py index 03c253a6a..e009e12de 100644 --- a/auto_round/auto_schemes/gen_scheme.py +++ b/auto_round/auto_schemes/gen_scheme.py @@ -22,13 +22,13 @@ class GenScheme: def __init__( - self, - auto_scheme: AutoScheme, - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - scale_dtype: str = "fp16", - dataset="pile-10k", + self, + auto_scheme: AutoScheme, + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scale_dtype: str = "fp16", + dataset="pile-10k", ): self.auto_scheme = auto_scheme self.model = model @@ -44,14 +44,14 @@ def _get_min_max_avg_bits(self) -> tuple[float, float]: def get_layer_bits(self, layer): weight = layer.weight n_param = weight.numel() - weight_bits = getattr(layer, 'bits', 16) - group_size = getattr(layer, 'group_size', 128) - super_group_size = getattr(layer, 'super_group_size', None) - super_weight_bits = getattr(layer, 'super_bits', None) + weight_bits = getattr(layer, "bits", 16) + group_size = getattr(layer, "group_size", 128) + super_group_size = getattr(layer, "super_group_size", None) + super_weight_bits = getattr(layer, "super_bits", None) # Main quantization cost weight_total_bits = weight_bits * n_param - if weight_bits>=16: # Unquantized layer + if weight_bits >= 16: # Unquantized layer return weight_total_bits, 16 in_features, output_features = get_layer_features(layer) @@ -75,9 +75,9 @@ def get_layer_bits(self, layer): # Double quantization case if super_group_size: # Number of super-groups - aux_total_bits+=n_group*super_weight_bits * 2 #sclae and min int count + aux_total_bits += n_group * super_weight_bits * 2 # sclae and min int count n_super_group = (n_group + super_group_size - 1) // super_group_size - aux_total_bits += n_super_group * 32 * 2 # double quant scale and min_v + aux_total_bits += n_super_group * 32 * 2 # double quant scale and min_v total_bits = weight_total_bits + aux_total_bits avg_bits = total_bits / n_param From a3756ce3ac84771c5bbb1b3abc62d61f29526312 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 9 Oct 2025 11:57:44 +0800 Subject: [PATCH 36/88] fix some issues --- auto_round/auto_schemes/gen_scheme.py | 2 +- auto_round/export/export_to_autoround/export_to_fp8.py | 8 +++++--- auto_round/export/export_to_llmcompressor/export_to_fp.py | 2 +- auto_round/utils.py | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/auto_round/auto_schemes/gen_scheme.py b/auto_round/auto_schemes/gen_scheme.py index e009e12de..da4a6345b 100644 --- a/auto_round/auto_schemes/gen_scheme.py +++ b/auto_round/auto_schemes/gen_scheme.py @@ -75,7 +75,7 @@ def get_layer_bits(self, layer): # Double quantization case if super_group_size: # Number of super-groups - aux_total_bits += n_group * super_weight_bits * 2 # sclae and min int count + aux_total_bits += n_group * super_weight_bits * 2 # scale and min int count n_super_group = (n_group + super_group_size - 1) // super_group_size aux_total_bits += n_super_group * 32 * 2 # double quant scale and min_v diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py index 7f069cb60..a68d5b207 100644 --- a/auto_round/export/export_to_autoround/export_to_fp8.py +++ b/auto_round/export/export_to_autoround/export_to_fp8.py @@ -109,10 +109,12 @@ def pack_layer(layer_name, model, data_type, device=None): torch_dtype = torch.float8_e5m2 info = torch.finfo(torch_dtype) if zp is not None: + if not isinstance(zp, torch.Tensor): + zp = torch.tensor(zp, dtype=weight.dtype) + + zp = zp.to(packing_device) q_weight = ( - weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp.to(packing_device) - if isinstance(zp, torch.Tensor) - else zp + weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp ) else: q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py index 1a56ebf12..6fcce8472 100644 --- a/auto_round/export/export_to_llmcompressor/export_to_fp.py +++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py @@ -169,7 +169,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs): for n, m in model.named_modules(): if type(m) in SUPPORTED_LAYER_TYPES: layer = m - if layer.act_bits < 8 and not getattr(layer, "input_global_scale", None): + if hasattr(layer,"act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None): assert hasattr(layer, "act_max") from auto_round.data_type.nvfp import calculate_gparam diff --git a/auto_round/utils.py b/auto_round/utils.py index a1c411373..8e050c5df 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2223,9 +2223,9 @@ def get_reciprocal(tensor): def check_need_act_calibration( - is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: int = 16 + is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: Union[int,None] = 16 ) -> bool: - if act_bits > 8: + if act_bits is None or act_bits > 8: return False # None is dynamic if is_act_dynamic is not None and not is_act_dynamic: From 2f93471694d13decdbf29713155fc9d334d63da1 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 9 Oct 2025 14:09:28 +0800 Subject: [PATCH 37/88] fix some issues --- auto_round/compressors/base.py | 9 ++++++--- auto_round/schemes.py | 8 ++++++++ auto_round/utils.py | 11 ++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index f1036ecac..a687aecf5 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -463,14 +463,17 @@ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kw f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}." ) if tmp_act_bits is not None and tmp_act_bits < 16: - for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}} + for supported_dtype in SUPPORTED_DTYPES: # To easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}} if self.act_data_type.startswith(supported_dtype): - if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3 + if supported_dtype + str(tmp_act_bits) == self.act_data_type: # Could not replace FP8_e4m3 self.act_data_type = supported_dtype break for key in scheme_keys: scheme[key] = getattr(self, key) - return res if res else QuantizationScheme.from_dict(scheme) + if res and QuantizationScheme.from_dict(scheme) == preset_name_to_scheme(res): + return res + else: + return QuantizationScheme.from_dict(scheme) def _adjust_torch_compile(self, enable_torch_compile: bool) -> None: """Sets the torch compile configuration for the tuning.""" diff --git a/auto_round/schemes.py b/auto_round/schemes.py index cde37a0c9..bc84589fd 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -72,7 +72,15 @@ def get(self, key: str, default=None): def __eq__(self, other: "QuantizationScheme") -> bool: if not isinstance(other, QuantizationScheme): return False + skip_act_check= False + self_act_bits = 16 if self.act_bits is None else self.act_bits + other_act_bits = 16 if other.act_bits is None else other.act_bits + if self_act_bits==other_act_bits and other_act_bits>=16: + skip_act_check = True + for field in self.get_attributes(): + if skip_act_check and field.startswith("act_"): + continue if getattr(self, field) != getattr(other, field): return False return True diff --git a/auto_round/utils.py b/auto_round/utils.py index 8e050c5df..de61d6d2c 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2868,7 +2868,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str if gguf_name and torch.nn.Embedding not in supported_types: supported_types = (*supported_types, torch.nn.Embedding) - all_layer_names, embedding_layer_names = [], [] + all_supported_layer_names, embedding_layer_names = [], [] all_module_names = [] for n, m in model.named_modules(): all_module_names.append(n) @@ -2878,19 +2878,16 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str delattr(m, key) if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types: continue - all_layer_names.append(n) + all_supported_layer_names.append(n) if isinstance(m, torch.nn.Embedding): embedding_layer_names.append(n) # 6. expand regex configs for name in list(layer_config.keys()): - if name in all_layer_names: - continue - if name in all_module_names: - logger.warning_once(f"the type of `{name}` is not supported in your scheme, ignore it for now.") + if name in all_supported_layer_names: continue regex = re.compile(name) - matched = [ln for ln in all_layer_names if regex.search(ln)] + matched = [ln for ln in all_supported_layer_names if regex.search(ln)] if not matched: raise ValueError(f"Invalid '{name}' in layer_config, no match found.") val = layer_config.pop(name) From e0c3d4bb2fb49b2eb04333914604b054583b0419 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Oct 2025 06:10:15 +0000 Subject: [PATCH 38/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/compressors/base.py | 2 +- auto_round/export/export_to_autoround/export_to_fp8.py | 6 ++---- auto_round/export/export_to_llmcompressor/export_to_fp.py | 2 +- auto_round/schemes.py | 8 ++++---- auto_round/utils.py | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index a687aecf5..ffee2ae3e 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -473,7 +473,7 @@ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kw if res and QuantizationScheme.from_dict(scheme) == preset_name_to_scheme(res): return res else: - return QuantizationScheme.from_dict(scheme) + return QuantizationScheme.from_dict(scheme) def _adjust_torch_compile(self, enable_torch_compile: bool) -> None: """Sets the torch compile configuration for the tuning.""" diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py index a68d5b207..455768e37 100644 --- a/auto_round/export/export_to_autoround/export_to_fp8.py +++ b/auto_round/export/export_to_autoround/export_to_fp8.py @@ -112,10 +112,8 @@ def pack_layer(layer_name, model, data_type, device=None): if not isinstance(zp, torch.Tensor): zp = torch.tensor(zp, dtype=weight.dtype) - zp = zp.to(packing_device) - q_weight = ( - weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp - ) + zp = zp.to(packing_device) + q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp else: q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) q_weight = revert_tensor_by_pad(q_weight, orig_shape=orig_shape, pad_len=pad_len) diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py index 6fcce8472..633593075 100644 --- a/auto_round/export/export_to_llmcompressor/export_to_fp.py +++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py @@ -169,7 +169,7 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs): for n, m in model.named_modules(): if type(m) in SUPPORTED_LAYER_TYPES: layer = m - if hasattr(layer,"act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None): + if hasattr(layer, "act_bits") and layer.act_bits < 8 and not getattr(layer, "input_global_scale", None): assert hasattr(layer, "act_max") from auto_round.data_type.nvfp import calculate_gparam diff --git a/auto_round/schemes.py b/auto_round/schemes.py index bc84589fd..ac5393e2c 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -72,10 +72,10 @@ def get(self, key: str, default=None): def __eq__(self, other: "QuantizationScheme") -> bool: if not isinstance(other, QuantizationScheme): return False - skip_act_check= False - self_act_bits = 16 if self.act_bits is None else self.act_bits - other_act_bits = 16 if other.act_bits is None else other.act_bits - if self_act_bits==other_act_bits and other_act_bits>=16: + skip_act_check = False + self_act_bits = 16 if self.act_bits is None else self.act_bits + other_act_bits = 16 if other.act_bits is None else other.act_bits + if self_act_bits == other_act_bits and other_act_bits >= 16: skip_act_check = True for field in self.get_attributes(): diff --git a/auto_round/utils.py b/auto_round/utils.py index de61d6d2c..0e5637b66 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2223,7 +2223,7 @@ def get_reciprocal(tensor): def check_need_act_calibration( - is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: Union[int,None] = 16 + is_act_dynamic: Union[bool, None], act_data_type: Union[str, None] = None, act_bits: Union[int, None] = 16 ) -> bool: if act_bits is None or act_bits > 8: return False From 6e04d100336e80ba775c43b098bc7d54b4836161 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 9 Oct 2025 16:07:45 +0800 Subject: [PATCH 39/88] update --- auto_round/auto_schemes/__init__.py | 2 +- auto_round/auto_schemes/gen_auto_scheme.py | 71 ++++++++++ auto_round/auto_schemes/gen_scheme.py | 84 ----------- auto_round/auto_schemes/utils.py | 153 ++++++++++++++++++++- auto_round/compressors/base.py | 4 +- auto_round/schemes.py | 1 + auto_round/utils.py | 7 + 7 files changed, 231 insertions(+), 91 deletions(-) create mode 100644 auto_round/auto_schemes/gen_auto_scheme.py delete mode 100644 auto_round/auto_schemes/gen_scheme.py diff --git a/auto_round/auto_schemes/__init__.py b/auto_round/auto_schemes/__init__.py index d3b055be2..285c68666 100644 --- a/auto_round/auto_schemes/__init__.py +++ b/auto_round/auto_schemes/__init__.py @@ -15,7 +15,7 @@ AUTO_SCHEMES_ALGS = {} -def register_dtype(names): +def register_scheme_algs(names): """Class decorator to register a mixed precision algorithm to the registry. Decorator function used before a Pattern subclass. diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py new file mode 100644 index 000000000..a9510e10e --- /dev/null +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -0,0 +1,71 @@ +# Copyright (c) 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import asdict +from typing import Iterable +import torch +from auto_round import AutoScheme +from auto_round.logger import logger +from auto_round.auto_schemes.utils import compute_avg_bits_for_scheme + + +class GenScheme: + """Generate and validate quantization schemes for model layers.""" + + def __init__( + self, + auto_scheme: AutoScheme, + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scale_dtype: str = "fp16", + dataset: str = "pile-10k", + ): + self.auto_scheme = auto_scheme + self.model = model + self.quant_layer_names = quant_layer_names + self.fixed_layer_scheme = fixed_layer_scheme + self.scale_dtype = scale_dtype + self.dataset = dataset + + self._check_configs() + + def _check_configs(self) -> None: + """Validate auto_scheme configuration and ensure avg_bits target is valid.""" + if not isinstance(self.dataset, str): + raise TypeError("`dataset` must be a string, got {type(self.dataset).__name__}.") + + min_avg_bit, max_avg_bit = self.compute_avg_bit_range() + target = self.auto_scheme.avg_bits + + logger.info("Average bits range: [%.2f, %.2f], target = %.2f", min_avg_bit, max_avg_bit, target) + + if not (min_avg_bit <= target <= max_avg_bit): + raise ValueError( + f"Target avg_bits={target:.2f} is outside the valid range " + f"[{min_avg_bit:.2f}, {max_avg_bit:.2f}]." + ) + + def compute_avg_bit_range(self) -> tuple[float, float]: + """Compute the min and max average bitwidths among candidate quantization options.""" + avg_bits = [ + compute_avg_bits_for_scheme( + self.model, + self.quant_layer_names, + self.fixed_layer_scheme, + option, + self.auto_scheme.ignore_scale_zp_bits, + )[0] + for option in self.auto_scheme.options + ] + return min(avg_bits), max(avg_bits) \ No newline at end of file diff --git a/auto_round/auto_schemes/gen_scheme.py b/auto_round/auto_schemes/gen_scheme.py deleted file mode 100644 index da4a6345b..000000000 --- a/auto_round/auto_schemes/gen_scheme.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2025 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Iterable, Union - -import torch - -from auto_round import AutoScheme -from auto_round.utils import get_layer_features - - -class GenScheme: - def __init__( - self, - auto_scheme: AutoScheme, - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - scale_dtype: str = "fp16", - dataset="pile-10k", - ): - self.auto_scheme = auto_scheme - self.model = model - self.quant_layer_names = quant_layer_names - self.fixed_layer_scheme = fixed_layer_scheme - self.scale_dtype = scale_dtype - self.dataset = dataset - - def _get_min_max_avg_bits(self) -> tuple[float, float]: - pass - - # not validate yet - def get_layer_bits(self, layer): - weight = layer.weight - n_param = weight.numel() - weight_bits = getattr(layer, "bits", 16) - group_size = getattr(layer, "group_size", 128) - super_group_size = getattr(layer, "super_group_size", None) - super_weight_bits = getattr(layer, "super_bits", None) - - # Main quantization cost - weight_total_bits = weight_bits * n_param - if weight_bits >= 16: # Unquantized layer - return weight_total_bits, 16 - - in_features, output_features = get_layer_features(layer) - # Determine number of groups - if group_size > 0: # group-wise - n_group = output_features * (in_features + group_size - 1) // group_size - elif group_size == 0: # per-tensor - n_group = 1 - elif group_size == -1: # per-channel - n_group = output_features # out_channels - else: - raise ValueError(f"Invalid group_size {group_size}") - aux_total_bits = 0 - if not super_group_size: - # Scale and zero point bitwidths - scale_bits = 16 - zp_bits = weight_bits if not super_group_size else 32 # default: same as weight_bits - # Overhead from scales and zero points - aux_total_bits = n_group * (scale_bits + zp_bits) - - # Double quantization case - if super_group_size: - # Number of super-groups - aux_total_bits += n_group * super_weight_bits * 2 # scale and min int count - n_super_group = (n_group + super_group_size - 1) // super_group_size - aux_total_bits += n_super_group * 32 * 2 # double quant scale and min_v - - total_bits = weight_total_bits + aux_total_bits - avg_bits = total_bits / n_param - return total_bits, avg_bits diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index e01da9913..74fbc2029 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -11,11 +11,156 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from dataclasses import asdict +from typing import Union, Iterable +import torch +from auto_round.low_cpu_mem import get_module +from auto_round.schemes import preset_name_to_scheme +from auto_round.utils import get_layer_features -def get_total_bits(model, layer_config): - pass +def apply_quant_scheme( + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scheme: Union[str, dict], +) -> None: + """Apply a quantization scheme to each quantized layer. -def get_bits(layer): - pass + Args: + model: The model whose layers are to be updated. + scheme: The scheme preset name or dictionary to apply. + quant_layer_names: Iterable of layer names to quantize. + fixed_layer_scheme: Dictionary of fixed per-layer quantization schemes. + """ + for name in quant_layer_names: + layer_scheme = fixed_layer_scheme.get(name, scheme) + if isinstance(layer_scheme, str): + layer_scheme = asdict(preset_name_to_scheme(layer_scheme)) + + module = get_module(model, name) + for key, value in layer_scheme.items(): + setattr(module, key, value) + + +def remove_quant_scheme( + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scheme: Union[str, dict], +) -> None: + """Remove attributes corresponding to the applied quantization scheme. + + Args: + model: The model whose layers are to be cleared. + scheme: The scheme preset name or dictionary previously applied. + quant_layer_names: Iterable of layer names to clear. + fixed_layer_scheme: Dictionary of fixed per-layer quantization schemes. + """ + for name in quant_layer_names: + layer_scheme = fixed_layer_scheme.get(name, scheme) + if isinstance(layer_scheme, str): + layer_scheme = asdict(preset_name_to_scheme(layer_scheme)) + + module = get_module(model, name) + for key in layer_scheme.keys(): + if hasattr(module, key): + delattr(module, key) + + +def compute_avg_bits_for_scheme( + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scheme: Union[str, dict, None] = None, + ignore_scale_zp_bits: bool = False, +) -> tuple[float, float]: + """Compute the average and total bit usage for the given quantization scheme. + + Args: + model: The model to analyze. + quant_layer_names: Iterable of layer names to include. + fixed_layer_scheme: Dictionary of fixed per-layer quantization schemes. + scheme: Optional scheme to temporarily apply before measuring. + ignore_scale_zp_bits: If True, ignores overhead from scale and zero-points. + + Returns: + A tuple (avg_bits, total_quantized_bits): + avg_bits: Average bitwidth per parameter. + total_quantized_bits: Total quantized bit count. + """ + if scheme is not None: + apply_quant_scheme(model, quant_layer_names, fixed_layer_scheme,scheme) + + total_params = 0 + total_quantized_bits = 0 + + for name in quant_layer_names: + module = get_module(model, name) + if not hasattr(module, "weight"): + continue + total_params += module.weight.numel() + layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits) + total_quantized_bits += layer_bits + + avg_bits = float(total_quantized_bits) / total_params + + if scheme is not None: + remove_quant_scheme(model, quant_layer_names, fixed_layer_scheme, scheme) + + return avg_bits, total_quantized_bits + + +def compute_layer_bits( + layer: torch.nn.Module, + ignore_scale_zp_bits: bool = False, +) -> tuple[int, float]: + """Compute total and average bitwidth for a single quantized layer. + + Args: + layer: A PyTorch layer with quantization attributes. + ignore_scale_zp_bits: Whether to ignore scale/zero-point overhead. + + Returns: + A tuple (total_bits, avg_bits) representing bit usage. + """ + weight = layer.weight + n_param = weight.numel() + weight_bits = getattr(layer, "bits", 16) + group_size = getattr(layer, "group_size", 128) + super_group_size = getattr(layer, "super_group_size", None) + super_weight_bits = getattr(layer, "super_bits", None) + + # Unquantized layer or ignoring scale/zp overhead + if weight_bits >= 16 or ignore_scale_zp_bits: + if super_weight_bits is not None: # reset gguf 16 bits to 32 bits + return 32 * n_param, 32 + return weight_bits * n_param, 16.0 + + in_features, out_features = get_layer_features(layer) + + # Determine number of groups based on group size + if group_size > 0: + n_group = out_features * (in_features + group_size - 1) // group_size + elif group_size == 0: + n_group = 1 + elif group_size == -1: + n_group = out_features + else: + raise ValueError(f"Invalid group_size {group_size}") + + # Compute auxiliary bits (scales, zero-points, or double quantization) + aux_total_bits = 0 + if not super_group_size: + scale_bits = 16 + zp_bits = weight_bits + aux_total_bits = n_group * (scale_bits + zp_bits) + else: + aux_total_bits += n_group * super_weight_bits * 2 + n_super_group = (n_group + super_group_size - 1) // super_group_size + aux_total_bits += n_super_group * 32 * 2 # 32-bit scale and min_v + + total_bits = weight_bits * n_param + aux_total_bits + avg_bits = total_bits / n_param + return total_bits, avg_bits \ No newline at end of file diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index fdb7b6213..a71e15059 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -309,10 +309,10 @@ def __init__( quant_layer_names = layer_config.keys() fixed_layer_scheme = {k: v for k, v in layer_config.items() if v.get("fixed_by_user", False)} # mainly using quant_layers and fixed by users - from auto_round.auto_schemes.gen_scheme import GenScheme + from auto_round.auto_schemes.gen_auto_scheme import GenScheme gen_scheme = GenScheme( - scheme, self.model, quant_layer_names, fixed_layer_scheme, self.scale_dtype, self.dataset + scheme, self.model, quant_layer_names, fixed_layer_scheme, self.scale_dtype, dataset ) # Set device, must place after model loading diff --git a/auto_round/schemes.py b/auto_round/schemes.py index ac5393e2c..2c087285e 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -268,3 +268,4 @@ class AutoScheme: avg_bits: float shared_layers: Optional[Iterable[Iterable[str]]] = None method: str = "default" + ignore_scale_zp_bits = False diff --git a/auto_round/utils.py b/auto_round/utils.py index 0e5637b66..319f73b45 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2886,6 +2886,13 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str for name in list(layer_config.keys()): if name in all_supported_layer_names: continue + if name in all_module_names : + m = get_module(model, name) + if len(list(m.children())) == 0 and type(m) not in supported_types: + logger.warning(f"{name} is not supported in current scheme, ignoring its setting in `layer_config`") + continue + + regex = re.compile(name) matched = [ln for ln in all_supported_layer_names if regex.search(ln)] if not matched: From 04c604cd3eccfdaf62878b4daf7a2a5838eecc8c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Oct 2025 08:08:29 +0000 Subject: [PATCH 40/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/gen_auto_scheme.py | 9 +++++---- auto_round/auto_schemes/utils.py | 9 +++++---- auto_round/compressors/base.py | 4 +--- auto_round/utils.py | 3 +-- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index a9510e10e..f4b668f17 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -13,10 +13,12 @@ # limitations under the License. from dataclasses import asdict from typing import Iterable + import torch + from auto_round import AutoScheme -from auto_round.logger import logger from auto_round.auto_schemes.utils import compute_avg_bits_for_scheme +from auto_round.logger import logger class GenScheme: @@ -52,8 +54,7 @@ def _check_configs(self) -> None: if not (min_avg_bit <= target <= max_avg_bit): raise ValueError( - f"Target avg_bits={target:.2f} is outside the valid range " - f"[{min_avg_bit:.2f}, {max_avg_bit:.2f}]." + f"Target avg_bits={target:.2f} is outside the valid range " f"[{min_avg_bit:.2f}, {max_avg_bit:.2f}]." ) def compute_avg_bit_range(self) -> tuple[float, float]: @@ -68,4 +69,4 @@ def compute_avg_bit_range(self) -> tuple[float, float]: )[0] for option in self.auto_scheme.options ] - return min(avg_bits), max(avg_bits) \ No newline at end of file + return min(avg_bits), max(avg_bits) diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index 74fbc2029..3122f1600 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import asdict -from typing import Union, Iterable +from typing import Iterable, Union import torch + from auto_round.low_cpu_mem import get_module from auto_round.schemes import preset_name_to_scheme from auto_round.utils import get_layer_features @@ -91,7 +92,7 @@ def compute_avg_bits_for_scheme( total_quantized_bits: Total quantized bit count. """ if scheme is not None: - apply_quant_scheme(model, quant_layer_names, fixed_layer_scheme,scheme) + apply_quant_scheme(model, quant_layer_names, fixed_layer_scheme, scheme) total_params = 0 total_quantized_bits = 0 @@ -134,7 +135,7 @@ def compute_layer_bits( # Unquantized layer or ignoring scale/zp overhead if weight_bits >= 16 or ignore_scale_zp_bits: - if super_weight_bits is not None: # reset gguf 16 bits to 32 bits + if super_weight_bits is not None: # reset gguf 16 bits to 32 bits return 32 * n_param, 32 return weight_bits * n_param, 16.0 @@ -163,4 +164,4 @@ def compute_layer_bits( total_bits = weight_bits * n_param + aux_total_bits avg_bits = total_bits / n_param - return total_bits, avg_bits \ No newline at end of file + return total_bits, avg_bits diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index a71e15059..c72eeecb5 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -311,9 +311,7 @@ def __init__( # mainly using quant_layers and fixed by users from auto_round.auto_schemes.gen_auto_scheme import GenScheme - gen_scheme = GenScheme( - scheme, self.model, quant_layer_names, fixed_layer_scheme, self.scale_dtype, dataset - ) + gen_scheme = GenScheme(scheme, self.model, quant_layer_names, fixed_layer_scheme, self.scale_dtype, dataset) # Set device, must place after model loading self._set_device(device_map) diff --git a/auto_round/utils.py b/auto_round/utils.py index 319f73b45..1e4a5c131 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2886,13 +2886,12 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str for name in list(layer_config.keys()): if name in all_supported_layer_names: continue - if name in all_module_names : + if name in all_module_names: m = get_module(model, name) if len(list(m.children())) == 0 and type(m) not in supported_types: logger.warning(f"{name} is not supported in current scheme, ignoring its setting in `layer_config`") continue - regex = re.compile(name) matched = [ln for ln in all_supported_layer_names if regex.search(ln)] if not matched: From 87d36945c76185131ea0b88e16646e2b6c29c1cc Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 9 Oct 2025 21:07:26 +0800 Subject: [PATCH 41/88] fix one bug --- auto_round/schemes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 2c087285e..8932d7b3e 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -250,7 +250,8 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: if isinstance(scheme, str): return "" for key, val in PRESET_SCHEMES.items(): - if not key.upper().startswith("GGUF"): + # For q40 or q4_1 we only support it with str scheme, otherwise it will be matched incorrectly with W4G32 + if not key.upper().startswith("GGUF") and ("0" in key or "1" in key): continue equal = True for scheme_key in val.keys(): From 3855c8f10687c26305c4ff7478ce5089cb9b56b5 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 10 Oct 2025 09:52:49 +0800 Subject: [PATCH 42/88] fix --- auto_round/schemes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 8932d7b3e..5cd959363 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -251,7 +251,7 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: return "" for key, val in PRESET_SCHEMES.items(): # For q40 or q4_1 we only support it with str scheme, otherwise it will be matched incorrectly with W4G32 - if not key.upper().startswith("GGUF") and ("0" in key or "1" in key): + if not key.upper().startswith("GGUF") or ("0" in key or "1" in key): continue equal = True for scheme_key in val.keys(): From d3e28c27207c43e863485e96b4fc21295496bedd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Oct 2025 01:53:35 +0000 Subject: [PATCH 43/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/schemes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 5cd959363..c305646a8 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -251,7 +251,7 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: return "" for key, val in PRESET_SCHEMES.items(): # For q40 or q4_1 we only support it with str scheme, otherwise it will be matched incorrectly with W4G32 - if not key.upper().startswith("GGUF") or ("0" in key or "1" in key): + if not key.upper().startswith("GGUF") or ("0" in key or "1" in key): continue equal = True for scheme_key in val.keys(): From 2d557d095e2ac891948aac4e35870c6eb4513007 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 10 Oct 2025 16:06:45 +0800 Subject: [PATCH 44/88] set up the first version, there are many details to be handled --- auto_round/auto_schemes/__init__.py | 10 ++-- auto_round/auto_schemes/gen_auto_scheme.py | 28 ++++++--- auto_round/auto_schemes/utils.py | 60 ++++++++++++------- auto_round/compressors/base.py | 69 ++++++++++++++++------ auto_round/schemes.py | 7 ++- auto_round/utils.py | 8 +-- test/test_cuda/test_auto_scheme.py | 8 ++- 7 files changed, 132 insertions(+), 58 deletions(-) diff --git a/auto_round/auto_schemes/__init__.py b/auto_round/auto_schemes/__init__.py index 285c68666..de6ad8073 100644 --- a/auto_round/auto_schemes/__init__.py +++ b/auto_round/auto_schemes/__init__.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -AUTO_SCHEMES_ALGS = {} +AUTO_SCHEMES_METHODS = {} -def register_scheme_algs(names): +def register_scheme_methods(names): """Class decorator to register a mixed precision algorithm to the registry. Decorator function used before a Pattern subclass. @@ -30,10 +30,12 @@ def register_scheme_algs(names): def register(alg): if isinstance(names, (tuple, list)): for name in names: - AUTO_SCHEMES_ALGS[name] = alg + AUTO_SCHEMES_METHODS[name] = alg else: - AUTO_SCHEMES_ALGS[names] = alg + AUTO_SCHEMES_METHODS[names] = alg return alg return register + +import auto_round.auto_schemes.haha diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index f4b668f17..c6ca1f6b6 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -19,44 +19,58 @@ from auto_round import AutoScheme from auto_round.auto_schemes.utils import compute_avg_bits_for_scheme from auto_round.logger import logger - +from auto_round.auto_schemes import AUTO_SCHEMES_METHODS class GenScheme: """Generate and validate quantization schemes for model layers.""" def __init__( self, - auto_scheme: AutoScheme, + auto_scheme: AutoScheme, # TODO support shared layer model: torch.nn.Module, quant_layer_names: Iterable[str], fixed_layer_scheme: dict[str, dict], - scale_dtype: str = "fp16", - dataset: str = "pile-10k", + dataset: str = "pile-10k", # TODO use auto-round dataset + tokenizer=None, ): self.auto_scheme = auto_scheme self.model = model + self.tokenizer = tokenizer self.quant_layer_names = quant_layer_names self.fixed_layer_scheme = fixed_layer_scheme - self.scale_dtype = scale_dtype self.dataset = dataset self._check_configs() def _check_configs(self) -> None: """Validate auto_scheme configuration and ensure avg_bits target is valid.""" + if isinstance(self.model, torch.nn.Module) and self.tokenizer is None: + raise ValueError("tokenizer must not be None if model is nn.Module") + if not isinstance(self.dataset, str): raise TypeError("`dataset` must be a string, got {type(self.dataset).__name__}.") min_avg_bit, max_avg_bit = self.compute_avg_bit_range() target = self.auto_scheme.avg_bits - logger.info("Average bits range: [%.2f, %.2f], target = %.2f", min_avg_bit, max_avg_bit, target) + logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target) if not (min_avg_bit <= target <= max_avg_bit): raise ValueError( - f"Target avg_bits={target:.2f} is outside the valid range " f"[{min_avg_bit:.2f}, {max_avg_bit:.2f}]." + f"Target avg_bits={target:.3f} is outside the valid range " f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}]." ) + def get_layer_config(self): + method_name = self.auto_scheme.method + method_func = AUTO_SCHEMES_METHODS[method_name] + layer_config = method_func(self.auto_scheme, + self.model, + self.quant_layer_names, + self.fixed_layer_scheme, + self.dataset, + self.tokenizer) + return layer_config + def compute_avg_bit_range(self) -> tuple[float, float]: """Compute the min and max average bitwidths among candidate quantization options.""" avg_bits = [ diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index 3122f1600..0e7e1dc71 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -11,21 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import asdict +from dataclasses import asdict, fields from typing import Iterable, Union import torch from auto_round.low_cpu_mem import get_module -from auto_round.schemes import preset_name_to_scheme -from auto_round.utils import get_layer_features +from auto_round.schemes import preset_name_to_scheme, QuantizationScheme +from auto_round.utils import get_layer_features, check_to_quantized def apply_quant_scheme( model: torch.nn.Module, quant_layer_names: Iterable[str], fixed_layer_scheme: dict[str, dict], - scheme: Union[str, dict], + scheme: Union[str, dict], #TODO add scale_dtype ) -> None: """Apply a quantization scheme to each quantized layer. @@ -47,27 +47,17 @@ def apply_quant_scheme( def remove_quant_scheme( model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - scheme: Union[str, dict], ) -> None: """Remove attributes corresponding to the applied quantization scheme. Args: model: The model whose layers are to be cleared. - scheme: The scheme preset name or dictionary previously applied. - quant_layer_names: Iterable of layer names to clear. - fixed_layer_scheme: Dictionary of fixed per-layer quantization schemes. """ - for name in quant_layer_names: - layer_scheme = fixed_layer_scheme.get(name, scheme) - if isinstance(layer_scheme, str): - layer_scheme = asdict(preset_name_to_scheme(layer_scheme)) - - module = get_module(model, name) - for key in layer_scheme.keys(): - if hasattr(module, key): - delattr(module, key) + scheme_keys = [f.name for f in fields(QuantizationScheme)] + ["scale_dtype"] + for n,m in model.named_modules(): + for key in scheme_keys: + if hasattr(m, key): + delattr(m, key) def compute_avg_bits_for_scheme( @@ -108,11 +98,39 @@ def compute_avg_bits_for_scheme( avg_bits = float(total_quantized_bits) / total_params if scheme is not None: - remove_quant_scheme(model, quant_layer_names, fixed_layer_scheme, scheme) + remove_quant_scheme(model) + + return avg_bits, total_quantized_bits + +def compute_avg_bits_for_model(model:torch.nn.Module, ignore_scale_zp_bits: bool = False): + """Compute the average and total bit usage for the entire model. + + Args: + model: The model to analyze. + ignore_scale_zp_bits: If True, ignores overhead from scale and zero-points. + if scheme is not None: + apply_quant_scheme(model, quant_layer_names, fixed_layer_scheme, scheme) + """ + + total_params = 0 + total_quantized_bits = 0 + + for n,module in model.named_modules(): + if not hasattr(module, "bits"): + continue + if not hasattr(module, "weight"): + continue + total_params += module.weight.numel() + layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits) + total_quantized_bits += layer_bits + + avg_bits = float(total_quantized_bits) / total_params + return avg_bits, total_quantized_bits + def compute_layer_bits( layer: torch.nn.Module, ignore_scale_zp_bits: bool = False, @@ -135,7 +153,7 @@ def compute_layer_bits( # Unquantized layer or ignoring scale/zp overhead if weight_bits >= 16 or ignore_scale_zp_bits: - if super_weight_bits is not None: # reset gguf 16 bits to 32 bits + if super_weight_bits is not None: # reset gguf 16 bits to 32 bits, TODO gguf q4_0, q4_1 may have bug return 32 * n_param, 32 return weight_bits * n_param, 16.0 diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 8c74dff86..bb1528653 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -202,10 +202,32 @@ def __init__( ... # ... ... } """ - if isinstance(scheme, AutoScheme): # TODO AutoScheme could also be patched by group_size, etc - self.scheme = self._parse_and_set_scheme(scheme.options[0], kwargs) + + if isinstance(scheme, AutoScheme): + if len(scheme.options)<=0: + raise ValueError("options of AutoScheme must not be empty") + options=[] + for option in scheme.options: + new_option = self._parse_and_set_scheme(option, kwargs) + options.append(new_option) + scheme.options = options + for opt in options: + if isinstance(opt, str) and opt=="BF16": + continue + if isinstance(opt, QuantizationScheme): + if opt.bits>=16 and (opt.act_bits is None or opt.act_bits>=16): + continue + self.scheme= opt # Choose the first one that not 16 bits + self.is_auto_scheme = True + + else: self.scheme = self._parse_and_set_scheme(scheme, kwargs) + self.is_auto_scheme = False + + scheme_keys = [f.name for f in fields(QuantizationScheme)] + for key in scheme_keys: + kwargs.pop(key, None) gguf_scheme_name = get_gguf_scheme(self.scheme) # GGUF uses fp32 scale dtype as default @@ -293,7 +315,7 @@ def __init__( if self.mllm: logger.info("AutoScheme with MLLM is not supported yet.") sys.exit(1) - layer_config, _ = set_layer_config( + layer_config, self.has_qlayer_outside_block = set_layer_config( self.model, self.layer_config, self.scheme, @@ -311,7 +333,8 @@ def __init__( # mainly using quant_layers and fixed by users from auto_round.auto_schemes.gen_auto_scheme import GenScheme - gen_scheme = GenScheme(scheme, self.model, quant_layer_names, fixed_layer_scheme, self.scale_dtype, dataset) + gen_scheme = GenScheme(scheme, self.model, quant_layer_names, fixed_layer_scheme, dataset,tokenizer=self.tokenizer) + self.layer_config = gen_scheme.get_layer_config() # Set device, must place after model loading self._set_device(device_map) @@ -428,7 +451,7 @@ def _parse_and_set_scheme(self, scheme: Union[str, dict, QuantizationScheme], kw setattr(self, key, kwargs[key]) else: setattr(self, key, scheme.get(key, None)) - kwargs.pop(key, None) + # kwargs.pop(key, None) if self.act_dynamic is None: self.act_dynamic = True @@ -1588,19 +1611,29 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: self.model = _handle_moe_model(self.model, formats=formats) # TODO check scale_dtype - self.layer_config, self.has_qlayer_outside_block = set_layer_config( - self.model, - self.layer_config, - self.scheme, - self.scale_dtype, - self.supported_types, - self.inner_supported_types, - self.quant_block_list, - self.fp_layers, - self.quant_lm_head, - enable_gguf_official_mixed=True, - is_mllm=self.mllm, - ) + if not self.is_auto_scheme: + self.layer_config, self.has_qlayer_outside_block = set_layer_config( + self.model, + self.layer_config, + self.scheme, + self.scale_dtype, + self.supported_types, + self.inner_supported_types, + self.quant_block_list, + self.fp_layers, + self.quant_lm_head, + enable_gguf_official_mixed=True, + is_mllm=self.mllm, + ) + else: + for n,scheme in self.layer_config.items(): + module = get_module(self.model, n) + if not isinstance(scheme,dict): + raise ValueError("scheme return by scheme should be dict") + for key,item in scheme.items(): + setattr(module, key, item) + # set_extra scale_dtype + module.scale_dtype = self.scale_dtype if not hasattr(self, "formats"): logger.warning("this API is deprecated, please use `quantize_and_save` instead") diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 5cd959363..09a919445 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -214,7 +214,7 @@ def is_preset_scheme(name: str) -> bool: BF16 = QuantizationScheme.from_dict( { "bits": 16, - "group_size": 0, + "group_size": 128, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", @@ -265,8 +265,11 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: @dataclass class AutoScheme: - options: Optional[Iterable[QuantizationScheme | str]] + options: Union[list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str]]] avg_bits: float shared_layers: Optional[Iterable[Iterable[str]]] = None method: str = "default" ignore_scale_zp_bits = False + nsamples = None + seqlen = None + dataset: Optional[str] = None # Import Notice no comma for each item \ No newline at end of file diff --git a/auto_round/utils.py b/auto_round/utils.py index 1e4a5c131..deee6805d 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2861,7 +2861,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str default_dict["scale_dtype"] = default_scale_dtype for cfg in layer_config.values(): for key in scheme_keys: - cfg.setdefault(key, default_dict.copy().get(key)) + cfg.setdefault(key, copy.deepcopy(default_dict.get(key))) # 5. collect supported modules gguf_name = get_gguf_scheme(default_scheme) @@ -2913,21 +2913,21 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str ) if lm_head_name not in layer_config and quant_lm_head: - layer_config[lm_head_name] = default_dict.copy() + layer_config[lm_head_name] = copy.deepcopy(default_dict) # 8. enforce shape divisibility for int weight-only if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name: for n, m in model.named_modules(): if type(m) in supported_types or m.__class__.__name__ in inner_supported_types: if m.weight.shape[0] % 32 or m.weight.shape[1] % 32: - layer_config.setdefault(n, default_dict.copy()) + layer_config.setdefault(n, copy.deepcopy(default_dict)) layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True}) logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).") # 9. block layers: mark as in_blocks=True for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types): if name not in layer_config: - layer_config[name] = default_dict.copy() + layer_config[name] = copy.deepcopy(default_dict) layer_config[name]["fixed_by_user"] = False layer_config[name]["in_blocks"] = True diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index b9fffdee9..a813e0801 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -4,6 +4,8 @@ import sys import unittest +from auto_round.auto_schemes.utils import compute_avg_bits_for_model + sys.path.insert(0, "../..") from auto_round import AutoRound, AutoRoundConfig, AutoScheme @@ -23,5 +25,7 @@ def tearDownClass(self): def test_auto_scheme(self): model_name = "facebook/opt-125m" scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) - ar = AutoRound(model=model_name, scheme=scheme, iters=1, nsamples=1) - ar.quantize_and_save(self.save_dir) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, format="fake") + model, layer_config = ar.quantize() + avg_bits, _ = compute_avg_bits_for_model(model) + assert (2.9 < avg_bits <= 3.0) From cedad472cdf3653000dd94ebdb10a2d2db3c39b6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Oct 2025 08:08:42 +0000 Subject: [PATCH 45/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/__init__.py | 1 + auto_round/auto_schemes/gen_auto_scheme.py | 16 +++++++-------- auto_round/auto_schemes/utils.py | 17 ++++++++-------- auto_round/compressors/base.py | 23 +++++++++++----------- auto_round/schemes.py | 2 +- test/test_cuda/test_auto_scheme.py | 2 +- 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/auto_round/auto_schemes/__init__.py b/auto_round/auto_schemes/__init__.py index de6ad8073..550458961 100644 --- a/auto_round/auto_schemes/__init__.py +++ b/auto_round/auto_schemes/__init__.py @@ -38,4 +38,5 @@ def register(alg): return register + import auto_round.auto_schemes.haha diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index c6ca1f6b6..cf96c06a9 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -17,20 +17,21 @@ import torch from auto_round import AutoScheme +from auto_round.auto_schemes import AUTO_SCHEMES_METHODS from auto_round.auto_schemes.utils import compute_avg_bits_for_scheme from auto_round.logger import logger -from auto_round.auto_schemes import AUTO_SCHEMES_METHODS + class GenScheme: """Generate and validate quantization schemes for model layers.""" def __init__( self, - auto_scheme: AutoScheme, # TODO support shared layer + auto_scheme: AutoScheme, # TODO support shared layer model: torch.nn.Module, quant_layer_names: Iterable[str], fixed_layer_scheme: dict[str, dict], - dataset: str = "pile-10k", # TODO use auto-round dataset + dataset: str = "pile-10k", # TODO use auto-round dataset tokenizer=None, ): self.auto_scheme = auto_scheme @@ -63,12 +64,9 @@ def _check_configs(self) -> None: def get_layer_config(self): method_name = self.auto_scheme.method method_func = AUTO_SCHEMES_METHODS[method_name] - layer_config = method_func(self.auto_scheme, - self.model, - self.quant_layer_names, - self.fixed_layer_scheme, - self.dataset, - self.tokenizer) + layer_config = method_func( + self.auto_scheme, self.model, self.quant_layer_names, self.fixed_layer_scheme, self.dataset, self.tokenizer + ) return layer_config def compute_avg_bit_range(self) -> tuple[float, float]: diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index 0e7e1dc71..06eedcc0d 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -17,15 +17,15 @@ import torch from auto_round.low_cpu_mem import get_module -from auto_round.schemes import preset_name_to_scheme, QuantizationScheme -from auto_round.utils import get_layer_features, check_to_quantized +from auto_round.schemes import QuantizationScheme, preset_name_to_scheme +from auto_round.utils import check_to_quantized, get_layer_features def apply_quant_scheme( model: torch.nn.Module, quant_layer_names: Iterable[str], fixed_layer_scheme: dict[str, dict], - scheme: Union[str, dict], #TODO add scale_dtype + scheme: Union[str, dict], # TODO add scale_dtype ) -> None: """Apply a quantization scheme to each quantized layer. @@ -54,7 +54,7 @@ def remove_quant_scheme( model: The model whose layers are to be cleared. """ scheme_keys = [f.name for f in fields(QuantizationScheme)] + ["scale_dtype"] - for n,m in model.named_modules(): + for n, m in model.named_modules(): for key in scheme_keys: if hasattr(m, key): delattr(m, key) @@ -102,7 +102,8 @@ def compute_avg_bits_for_scheme( return avg_bits, total_quantized_bits -def compute_avg_bits_for_model(model:torch.nn.Module, ignore_scale_zp_bits: bool = False): + +def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: bool = False): """Compute the average and total bit usage for the entire model. Args: @@ -115,8 +116,8 @@ def compute_avg_bits_for_model(model:torch.nn.Module, ignore_scale_zp_bits: bool total_params = 0 total_quantized_bits = 0 - for n,module in model.named_modules(): - if not hasattr(module, "bits"): + for n, module in model.named_modules(): + if not hasattr(module, "bits"): continue if not hasattr(module, "weight"): continue @@ -126,11 +127,9 @@ def compute_avg_bits_for_model(model:torch.nn.Module, ignore_scale_zp_bits: bool avg_bits = float(total_quantized_bits) / total_params - return avg_bits, total_quantized_bits - def compute_layer_bits( layer: torch.nn.Module, ignore_scale_zp_bits: bool = False, diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index bb1528653..aea6f420a 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -204,23 +204,22 @@ def __init__( """ if isinstance(scheme, AutoScheme): - if len(scheme.options)<=0: + if len(scheme.options) <= 0: raise ValueError("options of AutoScheme must not be empty") - options=[] + options = [] for option in scheme.options: new_option = self._parse_and_set_scheme(option, kwargs) options.append(new_option) scheme.options = options for opt in options: - if isinstance(opt, str) and opt=="BF16": + if isinstance(opt, str) and opt == "BF16": continue if isinstance(opt, QuantizationScheme): - if opt.bits>=16 and (opt.act_bits is None or opt.act_bits>=16): + if opt.bits >= 16 and (opt.act_bits is None or opt.act_bits >= 16): continue - self.scheme= opt # Choose the first one that not 16 bits + self.scheme = opt # Choose the first one that not 16 bits self.is_auto_scheme = True - else: self.scheme = self._parse_and_set_scheme(scheme, kwargs) self.is_auto_scheme = False @@ -315,7 +314,7 @@ def __init__( if self.mllm: logger.info("AutoScheme with MLLM is not supported yet.") sys.exit(1) - layer_config, self.has_qlayer_outside_block = set_layer_config( + layer_config, self.has_qlayer_outside_block = set_layer_config( self.model, self.layer_config, self.scheme, @@ -333,7 +332,9 @@ def __init__( # mainly using quant_layers and fixed by users from auto_round.auto_schemes.gen_auto_scheme import GenScheme - gen_scheme = GenScheme(scheme, self.model, quant_layer_names, fixed_layer_scheme, dataset,tokenizer=self.tokenizer) + gen_scheme = GenScheme( + scheme, self.model, quant_layer_names, fixed_layer_scheme, dataset, tokenizer=self.tokenizer + ) self.layer_config = gen_scheme.get_layer_config() # Set device, must place after model loading @@ -1626,11 +1627,11 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: is_mllm=self.mllm, ) else: - for n,scheme in self.layer_config.items(): + for n, scheme in self.layer_config.items(): module = get_module(self.model, n) - if not isinstance(scheme,dict): + if not isinstance(scheme, dict): raise ValueError("scheme return by scheme should be dict") - for key,item in scheme.items(): + for key, item in scheme.items(): setattr(module, key, item) # set_extra scale_dtype module.scale_dtype = self.scale_dtype diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 765b7e447..24688eeed 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -272,4 +272,4 @@ class AutoScheme: ignore_scale_zp_bits = False nsamples = None seqlen = None - dataset: Optional[str] = None # Import Notice no comma for each item \ No newline at end of file + dataset: Optional[str] = None # Import Notice no comma for each item diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index a813e0801..5cb543ca6 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -28,4 +28,4 @@ def test_auto_scheme(self): ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, format="fake") model, layer_config = ar.quantize() avg_bits, _ = compute_avg_bits_for_model(model) - assert (2.9 < avg_bits <= 3.0) + assert 2.9 < avg_bits <= 3.0 From 0c3a0e20b0e6ecfae3044d24ea4490ad37ab78cd Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 10 Oct 2025 17:26:13 +0800 Subject: [PATCH 46/88] fix one bug --- auto_round/auto_schemes/__init__.py | 2 +- auto_round/compressors/base.py | 34 ++++++++++++++++++++++------- test/test_cuda/test_auto_scheme.py | 21 ++++++++++++++---- 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/auto_round/auto_schemes/__init__.py b/auto_round/auto_schemes/__init__.py index 550458961..35664c1db 100644 --- a/auto_round/auto_schemes/__init__.py +++ b/auto_round/auto_schemes/__init__.py @@ -39,4 +39,4 @@ def register(alg): return register -import auto_round.auto_schemes.haha +import auto_round.auto_schemes.haha # pylint: disable=E0611 diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index aea6f420a..21d7d32c0 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -218,6 +218,11 @@ def __init__( if opt.bits >= 16 and (opt.act_bits is None or opt.act_bits >= 16): continue self.scheme = opt # Choose the first one that not 16 bits + break + + # apply scheme to set default bits + self._parse_and_set_scheme(self.scheme, kwargs) + self.is_auto_scheme = True else: @@ -1627,14 +1632,27 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: is_mllm=self.mllm, ) else: - for n, scheme in self.layer_config.items(): - module = get_module(self.model, n) - if not isinstance(scheme, dict): - raise ValueError("scheme return by scheme should be dict") - for key, item in scheme.items(): - setattr(module, key, item) - # set_extra scale_dtype - module.scale_dtype = self.scale_dtype + # for n, scheme in self.layer_config.items(): + # module = get_module(self.model, n) + # if not isinstance(scheme, dict): + # raise ValueError("scheme return by scheme should be dict") + # for key, item in scheme.items(): + # setattr(module, key, item) + # # set_extra scale_dtype + # module.scale_dtype = self.scale_dtype + self.layer_config, self.has_qlayer_outside_block = set_layer_config( + self.model, + self.layer_config, + self.scheme, + self.scale_dtype, + self.supported_types, + self.inner_supported_types, + self.quant_block_list, + self.fp_layers, + self.quant_lm_head, + enable_gguf_official_mixed=False, + is_mllm=self.mllm, + ) if not hasattr(self, "formats"): logger.warning("this API is deprecated, please use `quantize_and_save` instead") diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 5cb543ca6..f5833f006 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -5,6 +5,7 @@ import unittest from auto_round.auto_schemes.utils import compute_avg_bits_for_model +from auto_round.eval.evaluation import simple_evaluate sys.path.insert(0, "../..") @@ -22,10 +23,22 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) + # def test_auto_scheme(self): + # model_name = "facebook/opt-125m" + # scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) + # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, format="fake") + # model, layer_config = ar.quantize() + # avg_bits, _ = compute_avg_bits_for_model(model) + # print(avg_bits) + # assert 2.9 < avg_bits <= 3.0 + def test_auto_scheme(self): model_name = "facebook/opt-125m" scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, format="fake") - model, layer_config = ar.quantize() - avg_bits, _ = compute_avg_bits_for_model(model) - assert 2.9 < avg_bits <= 3.0 + ar = AutoRound(model=model_name, scheme=scheme) + ar.quantize_and_save(self.save_dir) + model_args = f"pretrained={self.save_dir}" + result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto") + print(result["results"]["lambada_openai"]["acc,none"]) + self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25) + From cced6d84d84ad081d317590d04560e6a2d8b032b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Oct 2025 09:27:00 +0000 Subject: [PATCH 47/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/__init__.py | 2 +- test/test_cuda/test_auto_scheme.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/auto_round/auto_schemes/__init__.py b/auto_round/auto_schemes/__init__.py index 35664c1db..6ac6723a1 100644 --- a/auto_round/auto_schemes/__init__.py +++ b/auto_round/auto_schemes/__init__.py @@ -39,4 +39,4 @@ def register(alg): return register -import auto_round.auto_schemes.haha # pylint: disable=E0611 +import auto_round.auto_schemes.haha # pylint: disable=E0611 diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index f5833f006..490b79c64 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -41,4 +41,3 @@ def test_auto_scheme(self): result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto") print(result["results"]["lambada_openai"]["acc,none"]) self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25) - From 58d5ae21f5d964500f6431f09d40c91f1e61bc8c Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 10 Oct 2025 17:27:34 +0800 Subject: [PATCH 48/88] uncomment ut --- test/test_cuda/test_auto_scheme.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index f5833f006..35d672342 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -23,14 +23,14 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) - # def test_auto_scheme(self): - # model_name = "facebook/opt-125m" - # scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) - # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, format="fake") - # model, layer_config = ar.quantize() - # avg_bits, _ = compute_avg_bits_for_model(model) - # print(avg_bits) - # assert 2.9 < avg_bits <= 3.0 + def test_auto_scheme(self): + model_name = "facebook/opt-125m" + scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, format="fake") + model, layer_config = ar.quantize() + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert 2.9 < avg_bits <= 3.0 def test_auto_scheme(self): model_name = "facebook/opt-125m" From ea489c35d163f2b01872142363bd4c70fd8207f1 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Fri, 10 Oct 2025 17:29:28 +0800 Subject: [PATCH 49/88] rename functions --- test/test_cuda/test_auto_scheme.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 4b0148df3..a2a0c55fc 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -23,7 +23,7 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) - def test_auto_scheme(self): + def test_avg_bits(self): model_name = "facebook/opt-125m" scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, format="fake") @@ -32,7 +32,7 @@ def test_auto_scheme(self): print(avg_bits) assert 2.9 < avg_bits <= 3.0 - def test_auto_scheme(self): + def test_auto_scheme_export(self): model_name = "facebook/opt-125m" scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) ar = AutoRound(model=model_name, scheme=scheme) From f74fcb49131787a7d3539782ebb0c1bb03a1ad71 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 11 Oct 2025 01:43:22 +0000 Subject: [PATCH 50/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/__init__.py | 1 + auto_round/autoround.py | 1 - auto_round/utils.py | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/auto_round/__init__.py b/auto_round/__init__.py index e3572ef1a..268065ba4 100644 --- a/auto_round/__init__.py +++ b/auto_round/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from auto_round.autoround import AutoRound + # support for old api from auto_round.autoround import AutoRoundLLM, AutoRoundMLLM, AutoRoundAdam, AutoRoundDiffusion from auto_round.schemes import QuantizationScheme, AutoScheme diff --git a/auto_round/autoround.py b/auto_round/autoround.py index 090e3bff2..a335b343c 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -26,7 +26,6 @@ MLLMCompressor, ) from auto_round.logger import deprecated, logger - from auto_round.schemes import AutoScheme, QuantizationScheme from auto_round.utils import is_diffusion_model, is_mllm_model diff --git a/auto_round/utils.py b/auto_round/utils.py index ecce4f505..0b61cb9dc 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -3006,6 +3006,8 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str dispatch_layer_config(layer_config) return layer_config, has_qlayer_outside_block + + def check_diffusers_installed(): # pragma: no cover try: import diffusers # noqa: F401 From 9cfa4e5869f62ee99b74e1712c44d59f7c618e5c Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Sat, 11 Oct 2025 17:51:39 +0800 Subject: [PATCH 51/88] update --- auto_round/auto_schemes/__init__.py | 2 +- auto_round/auto_schemes/gen_auto_scheme.py | 6 +++++ auto_round/compressors/base.py | 10 +++++++-- auto_round/data_type/int.py | 1 + auto_round/data_type/mxfp.py | 4 ++-- auto_round/schemes.py | 5 ++++- test/test_cuda/test_auto_scheme.py | 26 +++++++++++++++++++--- 7 files changed, 45 insertions(+), 9 deletions(-) diff --git a/auto_round/auto_schemes/__init__.py b/auto_round/auto_schemes/__init__.py index 6ac6723a1..e0e5ccb66 100644 --- a/auto_round/auto_schemes/__init__.py +++ b/auto_round/auto_schemes/__init__.py @@ -39,4 +39,4 @@ def register(alg): return register -import auto_round.auto_schemes.haha # pylint: disable=E0611 +import auto_round.auto_schemes.haha # pylint: disable=E0611,E0401 diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index cf96c06a9..1903f6519 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -55,6 +55,12 @@ def _check_configs(self) -> None: target = self.auto_scheme.avg_bits logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target) + if abs(target-min_avg_bit)<1e-3 or abs(target-max_avg_bit)<1e-3: + if target <=min_avg_bit: + target = min_avg_bit + else: + target = max_avg_bit + self.auto_scheme.avg_bits = target if not (min_avg_bit <= target <= max_avg_bit): raise ValueError( diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index af008861e..f624ba54c 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -335,12 +335,18 @@ def __init__( is_mllm=self.mllm, ) quant_layer_names = layer_config.keys() - fixed_layer_scheme = {k: v for k, v in layer_config.items() if v.get("fixed_by_user", False)} + scheme_keys = {f.name for f in fields(QuantizationScheme)} + fixed_layer_scheme_new = { + k: {key: v[key] for key in scheme_keys & v.keys()} + for k, v in layer_config.items() + if v.get("fixed_by_user", False) + } + # mainly using quant_layers and fixed by users from auto_round.auto_schemes.gen_auto_scheme import GenScheme gen_scheme = GenScheme( - scheme, self.model, quant_layer_names, fixed_layer_scheme, dataset, tokenizer=self.tokenizer + scheme, self.model, quant_layer_names, fixed_layer_scheme_new, dataset, tokenizer=self.tokenizer ) self.layer_config = gen_scheme.get_layer_config() diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py index 5b151fdfe..637485470 100644 --- a/auto_round/data_type/int.py +++ b/auto_round/data_type/int.py @@ -19,6 +19,7 @@ @register_dtype("int_sym") +@torch.compile def quant_tensor_sym( tensor, bits=4, diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py index 862ff0a9a..b93628fc2 100644 --- a/auto_round/data_type/mxfp.py +++ b/auto_round/data_type/mxfp.py @@ -76,7 +76,7 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"): tensor = torch.clamp(tensor, min=-max_norm, max=max_norm) return tensor - +@torch.compile def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs): """Quantize the given tensor using the specified parameters. @@ -126,7 +126,7 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_roundin tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len) return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None - +@torch.compile def quant_mx_rceil( tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs ): diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 24688eeed..ccee9461c 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -265,7 +265,7 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: @dataclass class AutoScheme: - options: Union[list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str]]] + options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str]]] avg_bits: float shared_layers: Optional[Iterable[Iterable[str]]] = None method: str = "default" @@ -273,3 +273,6 @@ class AutoScheme: nsamples = None seqlen = None dataset: Optional[str] = None # Import Notice no comma for each item + def __post_init__(self): + if isinstance(self.options,str): + self.options = self.options.split(",") diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index a2a0c55fc..0956f24fe 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -3,11 +3,12 @@ import shutil import sys import unittest +sys.path.insert(0, "../..") from auto_round.auto_schemes.utils import compute_avg_bits_for_model from auto_round.eval.evaluation import simple_evaluate +from auto_round.utils import get_module -sys.path.insert(0, "../..") from auto_round import AutoRound, AutoRoundConfig, AutoScheme @@ -24,14 +25,33 @@ def tearDownClass(self): shutil.rmtree("runs", ignore_errors=True) def test_avg_bits(self): - model_name = "facebook/opt-125m" + model_name = "/models/opt-125m" scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, format="fake") + user_layer_config = {"model.decoder.layers.10.fc1":{"bits":8,"group_size":32, "sym":False}} + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1,layer_config=user_layer_config) model, layer_config = ar.quantize() + self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["bits"], 8) + self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["sym"], False) + self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["group_size"], 32) + layer = get_module(model, "model.decoder.layers.10.fc1") + self.assertEqual(layer.bits, 8) + self.assertEqual(layer.sym, False) + self.assertEqual(layer.group_size,32) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert 2.9 < avg_bits <= 3.0 + def test_lm_head_and_mix_dtype(self): + model_name = "/models/Qwen3-8B" + target_bits = 8.192 + scheme = AutoScheme(avg_bits=target_bits, options=( "MXFP4", "W8A16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + model, layer_config = ar.quantize() + # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits-0.1 < avg_bits <= target_bits + def test_auto_scheme_export(self): model_name = "facebook/opt-125m" scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) From ac4036ef5ae373e790ce5a7d5352999edd0f3113 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 11 Oct 2025 09:52:44 +0000 Subject: [PATCH 52/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/gen_auto_scheme.py | 4 ++-- auto_round/data_type/mxfp.py | 2 ++ auto_round/schemes.py | 3 ++- test/test_cuda/test_auto_scheme.py | 15 +++++++-------- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index 1903f6519..e2fc6eb0d 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -55,8 +55,8 @@ def _check_configs(self) -> None: target = self.auto_scheme.avg_bits logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target) - if abs(target-min_avg_bit)<1e-3 or abs(target-max_avg_bit)<1e-3: - if target <=min_avg_bit: + if abs(target - min_avg_bit) < 1e-3 or abs(target - max_avg_bit) < 1e-3: + if target <= min_avg_bit: target = min_avg_bit else: target = max_avg_bit diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py index b93628fc2..8eb78a8e5 100644 --- a/auto_round/data_type/mxfp.py +++ b/auto_round/data_type/mxfp.py @@ -76,6 +76,7 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"): tensor = torch.clamp(tensor, min=-max_norm, max=max_norm) return tensor + @torch.compile def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs): """Quantize the given tensor using the specified parameters. @@ -126,6 +127,7 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_roundin tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len) return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None + @torch.compile def quant_mx_rceil( tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs diff --git a/auto_round/schemes.py b/auto_round/schemes.py index ccee9461c..e8b14f772 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -273,6 +273,7 @@ class AutoScheme: nsamples = None seqlen = None dataset: Optional[str] = None # Import Notice no comma for each item + def __post_init__(self): - if isinstance(self.options,str): + if isinstance(self.options, str): self.options = self.options.split(",") diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 0956f24fe..e19c1893d 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -3,16 +3,15 @@ import shutil import sys import unittest + sys.path.insert(0, "../..") +from auto_round import AutoRound, AutoRoundConfig, AutoScheme from auto_round.auto_schemes.utils import compute_avg_bits_for_model from auto_round.eval.evaluation import simple_evaluate from auto_round.utils import get_module -from auto_round import AutoRound, AutoRoundConfig, AutoScheme - - class TestAutoScheme(unittest.TestCase): @classmethod def setUpClass(self): @@ -27,8 +26,8 @@ def tearDownClass(self): def test_avg_bits(self): model_name = "/models/opt-125m" scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) - user_layer_config = {"model.decoder.layers.10.fc1":{"bits":8,"group_size":32, "sym":False}} - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1,layer_config=user_layer_config) + user_layer_config = {"model.decoder.layers.10.fc1": {"bits": 8, "group_size": 32, "sym": False}} + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, layer_config=user_layer_config) model, layer_config = ar.quantize() self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["bits"], 8) self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["sym"], False) @@ -36,7 +35,7 @@ def test_avg_bits(self): layer = get_module(model, "model.decoder.layers.10.fc1") self.assertEqual(layer.bits, 8) self.assertEqual(layer.sym, False) - self.assertEqual(layer.group_size,32) + self.assertEqual(layer.group_size, 32) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert 2.9 < avg_bits <= 3.0 @@ -44,13 +43,13 @@ def test_avg_bits(self): def test_lm_head_and_mix_dtype(self): model_name = "/models/Qwen3-8B" target_bits = 8.192 - scheme = AutoScheme(avg_bits=target_bits, options=( "MXFP4", "W8A16")) + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) model, layer_config = ar.quantize() # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) - assert target_bits-0.1 < avg_bits <= target_bits + assert target_bits - 0.1 < avg_bits <= target_bits def test_auto_scheme_export(self): model_name = "facebook/opt-125m" From 8a8cb6112f837602d3b7e52cca3626cbe80e8e73 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Sat, 11 Oct 2025 21:00:44 +0800 Subject: [PATCH 53/88] fix --- auto_round/data_type/int.py | 1 - auto_round/data_type/mxfp.py | 2 -- test/test_cuda/test_auto_scheme.py | 1 + 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py index 637485470..5b151fdfe 100644 --- a/auto_round/data_type/int.py +++ b/auto_round/data_type/int.py @@ -19,7 +19,6 @@ @register_dtype("int_sym") -@torch.compile def quant_tensor_sym( tensor, bits=4, diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py index 8eb78a8e5..862ff0a9a 100644 --- a/auto_round/data_type/mxfp.py +++ b/auto_round/data_type/mxfp.py @@ -77,7 +77,6 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"): return tensor -@torch.compile def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs): """Quantize the given tensor using the specified parameters. @@ -128,7 +127,6 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_roundin return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None -@torch.compile def quant_mx_rceil( tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs ): diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index e19c1893d..a0e55c225 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -60,3 +60,4 @@ def test_auto_scheme_export(self): result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto") print(result["results"]["lambada_openai"]["acc,none"]) self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25) + shutil.rmtree(self.save_dir, ignore_errors=True) From 0e2be6c7024fda33c8b148e86a3644ca86590e60 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 13 Oct 2025 11:50:12 +0800 Subject: [PATCH 54/88] fix a bug --- auto_round/auto_schemes/gen_auto_scheme.py | 2 +- auto_round/data_type/gguf.py | 2 +- test/test_cuda/test_auto_scheme.py | 64 +++++++++++++++++++--- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index e2fc6eb0d..8557ccabc 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -56,7 +56,7 @@ def _check_configs(self) -> None: logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target) if abs(target - min_avg_bit) < 1e-3 or abs(target - max_avg_bit) < 1e-3: - if target <= min_avg_bit: + if abs(target - min_avg_bit) < 1e-3: target = min_avg_bit else: target = max_avg_bit diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py index 6456b817f..6aa19a3d5 100644 --- a/auto_round/data_type/gguf.py +++ b/auto_round/data_type/gguf.py @@ -337,7 +337,7 @@ def quant_tensor_gguf_asym_dq( if bits == 2: quant_weights = torch.abs(tensor) elif bits == 4 or bits == 5: - sigma2 = torch.sum(torch.pow(tensor, 2), dim=-1, keepdim=True) / 32 ##Note 32 is different from QK_K + sigma2 = torch.sum(torch.pow(tensor, 2), dim=-1, keepdim=True) / 32 # Note 32 is different from QK_K av_x = torch.sqrt(sigma2) quant_weights = torch.abs(tensor) + av_x params = search_kwargs[bits] diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index a0e55c225..d171a667f 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -4,6 +4,8 @@ import sys import unittest +from auto_round.testing_utils import multi_card + sys.path.insert(0, "../..") from auto_round import AutoRound, AutoRoundConfig, AutoScheme @@ -23,7 +25,55 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) - def test_avg_bits(self): + # @multi_card + # def test_multi_card(self): + # model_name = "/models/Qwen3-8B" + # target_bits = 4.644 + # scheme = AutoScheme(avg_bits=target_bits, options=("NVFP4", "W8A16")) + # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + # model, layer_config = ar.quantize() + # # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + # avg_bits, _ = compute_avg_bits_for_model(model) + # print(avg_bits) + # assert target_bits - 0.1 < avg_bits <= target_bits+1e-3 + + def test_min_target_bits(self): + model_name = "/models/opt-125m" + target_bits = 4.644 + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + model, layer_config = ar.quantize() + # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + + def test_max_target_bits(self): + model_name = "/models/opt-125m" + target_bits = 8.211 + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + model, layer_config = ar.quantize() + # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + + def test_patch_scheme(self): + model_name = "/models/opt-125m" + target_bits = 5 + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, group_size=32) + model, layer_config = ar.quantize() + for n, m in model.named_modules(): + if hasattr(m, "group_size"): + self.assertEqual(m.group_size, 32) + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + + def test_layer_config(self): + target_bits = 3.0 model_name = "/models/opt-125m" scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) user_layer_config = {"model.decoder.layers.10.fc1": {"bits": 8, "group_size": 32, "sym": False}} @@ -38,21 +88,21 @@ def test_avg_bits(self): self.assertEqual(layer.group_size, 32) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) - assert 2.9 < avg_bits <= 3.0 + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 def test_lm_head_and_mix_dtype(self): model_name = "/models/Qwen3-8B" - target_bits = 8.192 + target_bits = 6 scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, quant_lm_head=True) model, layer_config = ar.quantize() - # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + self.assertLessEqual(layer_config["lm_head"]["bits"], 8) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) - assert target_bits - 0.1 < avg_bits <= target_bits + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 def test_auto_scheme_export(self): - model_name = "facebook/opt-125m" + model_name = "/models/opt-125m" scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) ar = AutoRound(model=model_name, scheme=scheme) ar.quantize_and_save(self.save_dir) From 8d854dbf97cb2ae621d980b57bcfb4bdc788005b Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 13 Oct 2025 13:07:41 +0800 Subject: [PATCH 55/88] update --- auto_round/__main__.py | 11 +++++++++-- auto_round/schemes.py | 11 ++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 04d9f4421..c75dc3ce9 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -19,7 +19,7 @@ from auto_round.compressors import BaseCompressor from auto_round.eval.eval_cli import EvalArgumentParser, _eval_init, eval, eval_task_by_task -from auto_round.schemes import PRESET_SCHEMES +from auto_round.schemes import PRESET_SCHEMES, AutoScheme from auto_round.utils import ( clear_memory, get_device_and_parallelism, @@ -47,7 +47,9 @@ def __init__(self, *args, **kwargs): # choices=["W4A16", "W2A16", "W3A16", "W8A16", "MXFP4", "MXFP8", "NVFP4", "FPW8A16", "FP8_STATIC"], help="quantization scheme", ) - + self.add_argument("--avg_bits", default=None, type=float, help="for auto scheme, number of avg weight bits") + self.add_argument("--options", default=None, type=str,help="for auto scheme, options for auto scheme, e.g. 'W4A16,W8A16'") + self.add_argument("--ignore_scale_zp_bits", action="store_true", help="for auto scheme whether ignore scale zp bits calculation ") self.add_argument("--bits", default=None, type=int, help="number of weight bits") self.add_argument("--group_size", default=None, type=int, help="group size") self.add_argument("--asym", action="store_true", help="whether to use asym quantization") @@ -520,6 +522,11 @@ def tune(args): # layer_config[item[0]] = {} # layer_config[item[0]]["bits"] = item[1] + if args.avg_bits is not None: + if args.options is None: + raise ValueError("please set --options for auto scheme") + scheme = AutoScheme(options=args.options, avg_bits=args.avg_bits, ignore_scale_zp_bits=args.ignore_scale_zp_bits) + autoround: BaseCompressor = AutoRound( model=model_name, scheme=scheme, diff --git a/auto_round/schemes.py b/auto_round/schemes.py index e8b14f772..fa4cbe025 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -265,15 +265,16 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: @dataclass class AutoScheme: - options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str]]] avg_bits: float + options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str]]] shared_layers: Optional[Iterable[Iterable[str]]] = None method: str = "default" - ignore_scale_zp_bits = False - nsamples = None - seqlen = None + ignore_scale_zp_bits: bool = False + nsamples: Optional[int] = None + seqlen: Optional[int] = None dataset: Optional[str] = None # Import Notice no comma for each item def __post_init__(self): if isinstance(self.options, str): - self.options = self.options.split(",") + options = self.options.upper().replace(" ", "") + self.options = options.split(",") From d7908f461a707538d7682d1fb611a2f9351e2494 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 05:08:32 +0000 Subject: [PATCH 56/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/__main__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index c75dc3ce9..d2ff36912 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -48,8 +48,14 @@ def __init__(self, *args, **kwargs): help="quantization scheme", ) self.add_argument("--avg_bits", default=None, type=float, help="for auto scheme, number of avg weight bits") - self.add_argument("--options", default=None, type=str,help="for auto scheme, options for auto scheme, e.g. 'W4A16,W8A16'") - self.add_argument("--ignore_scale_zp_bits", action="store_true", help="for auto scheme whether ignore scale zp bits calculation ") + self.add_argument( + "--options", default=None, type=str, help="for auto scheme, options for auto scheme, e.g. 'W4A16,W8A16'" + ) + self.add_argument( + "--ignore_scale_zp_bits", + action="store_true", + help="for auto scheme whether ignore scale zp bits calculation ", + ) self.add_argument("--bits", default=None, type=int, help="number of weight bits") self.add_argument("--group_size", default=None, type=int, help="group size") self.add_argument("--asym", action="store_true", help="whether to use asym quantization") @@ -525,7 +531,9 @@ def tune(args): if args.avg_bits is not None: if args.options is None: raise ValueError("please set --options for auto scheme") - scheme = AutoScheme(options=args.options, avg_bits=args.avg_bits, ignore_scale_zp_bits=args.ignore_scale_zp_bits) + scheme = AutoScheme( + options=args.options, avg_bits=args.avg_bits, ignore_scale_zp_bits=args.ignore_scale_zp_bits + ) autoround: BaseCompressor = AutoRound( model=model_name, From 21dd1c2f48e519e10e4cadded451eb1ad1297f5f Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 13 Oct 2025 17:24:40 +0800 Subject: [PATCH 57/88] support multiple gpu via device_map --- auto_round/auto_schemes/gen_auto_scheme.py | 8 +- auto_round/auto_schemes/utils.py | 168 ++++++++++++++++++-- auto_round/compressors/base.py | 6 +- auto_round/utils.py | 6 +- auto_round/wrapper.py | 2 +- test/test_cuda/test_auto_scheme.py | 172 ++++++++++++--------- 6 files changed, 267 insertions(+), 95 deletions(-) diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index 8557ccabc..c5e92ef4c 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import asdict -from typing import Iterable +from typing import Iterable, Union import torch @@ -32,6 +32,7 @@ def __init__( quant_layer_names: Iterable[str], fixed_layer_scheme: dict[str, dict], dataset: str = "pile-10k", # TODO use auto-round dataset + device_map:Union[str, torch.device, int, dict, None] = None, tokenizer=None, ): self.auto_scheme = auto_scheme @@ -40,7 +41,7 @@ def __init__( self.quant_layer_names = quant_layer_names self.fixed_layer_scheme = fixed_layer_scheme self.dataset = dataset - + self.device_map = device_map self._check_configs() def _check_configs(self) -> None: @@ -71,7 +72,8 @@ def get_layer_config(self): method_name = self.auto_scheme.method method_func = AUTO_SCHEMES_METHODS[method_name] layer_config = method_func( - self.auto_scheme, self.model, self.quant_layer_names, self.fixed_layer_scheme, self.dataset, self.tokenizer + self.auto_scheme, self.model, self.quant_layer_names, self.fixed_layer_scheme, + self.dataset, self.tokenizer,device_map=self.device_map, ) return layer_config diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index 06eedcc0d..7e44eceb6 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -15,17 +15,19 @@ from typing import Iterable, Union import torch +from accelerate import infer_auto_device_map, dispatch_model +from accelerate.utils import get_balanced_memory from auto_round.low_cpu_mem import get_module from auto_round.schemes import QuantizationScheme, preset_name_to_scheme -from auto_round.utils import check_to_quantized, get_layer_features +from auto_round.utils import check_to_quantized, get_layer_features, is_hpex_available def apply_quant_scheme( - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - scheme: Union[str, dict], # TODO add scale_dtype + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scheme: Union[str, dict], # TODO add scale_dtype ) -> None: """Apply a quantization scheme to each quantized layer. @@ -46,7 +48,7 @@ def apply_quant_scheme( def remove_quant_scheme( - model: torch.nn.Module, + model: torch.nn.Module, ) -> None: """Remove attributes corresponding to the applied quantization scheme. @@ -61,11 +63,11 @@ def remove_quant_scheme( def compute_avg_bits_for_scheme( - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - scheme: Union[str, dict, None] = None, - ignore_scale_zp_bits: bool = False, + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scheme: Union[str, dict, None] = None, + ignore_scale_zp_bits: bool = False, ) -> tuple[float, float]: """Compute the average and total bit usage for the given quantization scheme. @@ -131,8 +133,8 @@ def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: boo def compute_layer_bits( - layer: torch.nn.Module, - ignore_scale_zp_bits: bool = False, + layer: torch.nn.Module, + ignore_scale_zp_bits: bool = False, ) -> tuple[int, float]: """Compute total and average bitwidth for a single quantized layer. @@ -182,3 +184,143 @@ def compute_layer_bits( total_bits = weight_bits * n_param + aux_total_bits avg_bits = total_bits / n_param return total_bits, avg_bits + + +def parse_all_available_device(device_map: Union[str, torch.device, int, dict, None] = None) -> list: + """ + Parse the device map and return a list of all available devices. + + Supported input formats: + - None: Automatically detect all available devices + - int: A single device index (e.g., 0) + - str: Examples: + "cpu" + "cuda:0,cuda:1" + "0,1" (numeric device indices) + - dict: Extract all device values from the dictionary + - torch.device: e.g. torch.device("cuda:0") + + Returns: + list[str]: Normalized device names, e.g., ["cuda:0", "cuda:1"] or ["cpu"] + """ + + # === Step 1. Detect available device types === + device_types = [] + if torch.cuda.is_available(): + device_types.append("cuda") + if hasattr(torch, "xpu") and torch.xpu.is_available(): + device_types.append("xpu") + if hasattr(torch, "hpu") and is_hpex_available(): + device_types.append("hpu") + + # Always include CPU as a fallback + if not device_types: + device_types = ["cpu"] + + # === Step 2. Parse different input formats === + if device_map is None: + # Automatically detect one available device + if "cuda" in device_types: + return ["cuda:0"] + elif "xpu" in device_types: + return ["xpu:0"] + elif "hpu" in device_types: + return ["hpu:0"] + else: + return ["cpu"] + + if isinstance(device_map, torch.device): + # Handle torch.device objects + dev_type = device_map.type + index = device_map.index + if dev_type == "cpu": + return ["cpu"] + if index is None: + index = 0 + return [f"{dev_type}:{index}"] + + if isinstance(device_map, int): + # Integer input → use primary available device type + device_type = device_types[0] + return [f"{device_type}:{device_map}"] if device_type != "cpu" else ["cpu"] + + if isinstance(device_map, str): + # Remove whitespace + device_map = device_map.strip() + if device_map.lower() == "cpu": + return ["cpu"] + + # Split by commas + parts = [x.strip() for x in device_map.split(",") if x.strip()] + parsed = [] + for p in parts: + if p.isdigit(): + # Numeric → assign to first available device type + device_type = device_types[0] + parsed.append(f"{device_type}:{p}" if device_type != "cpu" else "cpu") + else: + parsed.append(p) + return parsed + + if isinstance(device_map, dict): + # Extract all devices recursively from dict values + devices = set() + for v in device_map.values(): + devices.update(parse_all_available_device(v)) + return sorted(devices) + + raise TypeError(f"Unsupported device_map type: {type(device_map)}") + + +# Important Notice This dispatch does not follow dict device_map, just extract all available devices and use them +def dispatch_model_by_all_available_devices(model: torch.nn.Module, + device_map: Union[str, int, dict, None]) -> torch.nn.Module: + if device_map is None: + device_map = 0 + + no_split_modules = getattr(model, "_no_split_modules", []) + if device_map == "auto": + max_memory = get_balanced_memory( + model, + max_memory=None, + no_split_module_classes=no_split_modules, + ) + device_map = infer_auto_device_map( + model, + max_memory=max_memory, + no_split_module_classes=no_split_modules + ) + model = dispatch_model(model, device_map=device_map) + return model + + devices = parse_all_available_device(device_map) + + if len(devices) == 1: + model.to(devices[0]) + return model + + max_memory = get_balanced_memory( + model, + max_memory=None, + no_split_module_classes=no_split_modules, + ) + + # Filter max_memory with devices + # assume only one GPU model + new_max_memory = {} + for device in devices: + if ":" in device: + device = int(device.split(":")[-1]) + elif device == "cpu": + device = "cpu" + else: + raise ValueError(f"Unsupported device {device} in device_map: {device_map}") + new_max_memory[device] = max_memory[device] + + device_map = infer_auto_device_map( + model, + max_memory=max_memory, + no_split_module_classes=no_split_modules + ) + model = dispatch_model(model, device_map=device_map) + return model diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index f624ba54c..d3207c2bf 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -346,7 +346,7 @@ def __init__( from auto_round.auto_schemes.gen_auto_scheme import GenScheme gen_scheme = GenScheme( - scheme, self.model, quant_layer_names, fixed_layer_scheme_new, dataset, tokenizer=self.tokenizer + scheme, self.model, quant_layer_names, fixed_layer_scheme_new, dataset, device_map=device_map,tokenizer=self.tokenizer ) self.layer_config = gen_scheme.get_layer_config() @@ -1581,9 +1581,7 @@ def _quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) if self.device_map is not None: accelerate.hooks.remove_hook_from_submodules(block) - if ( - is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats) - ) or is_static_wfp8afp8(self): + if (is_nv_fp(self.act_data_type) ) or is_static_wfp8afp8(self): # enable moe experts act_max automatic generation for Linear set_amax_for_all_moe_layers(block, attr_name="act_max") # Normalize imatrix and quantize layers diff --git a/auto_round/utils.py b/auto_round/utils.py index 0b61cb9dc..7130fc054 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -30,6 +30,7 @@ import cpuinfo import torch import transformers +from accelerate.utils import get_balanced_memory from packaging import version from torch.amp import autocast @@ -3018,7 +3019,7 @@ def check_diffusers_installed(): # pragma: no cover exit(-1) -def is_diffusion_model(model_or_path: Union[str, object]): +def is_diffusion_model(model_or_path: Union[str, object]) -> bool: if isinstance(model_or_path, str): index_file = None if not os.path.isdir(model_or_path): @@ -3041,3 +3042,6 @@ def is_diffusion_model(model_or_path: Union[str, object]): return isinstance(model_or_path, pipeline_utils.DiffusionPipeline) else: return False + + + diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py index f6bffa94d..0a90c3965 100644 --- a/auto_round/wrapper.py +++ b/auto_round/wrapper.py @@ -150,7 +150,7 @@ def _init_tuning_params_and_quant_func(self): ) self._init_params("act_max_scale", p_dtype, (1), 1.0, not orig_layer.act_dynamic) - ## bias tuning + # Bias tuning if self.enable_norm_bias_tuning: self._init_params("bias_v", p_dtype, self.orig_layer.bias.shape, 0, True) from auto_round.data_type.int import quant_tensor_asym_wo_round diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index d171a667f..f8c36476f 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -25,89 +25,115 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) - # @multi_card - # def test_multi_card(self): - # model_name = "/models/Qwen3-8B" - # target_bits = 4.644 - # scheme = AutoScheme(avg_bits=target_bits, options=("NVFP4", "W8A16")) - # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) - # model, layer_config = ar.quantize() - # # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) - # avg_bits, _ = compute_avg_bits_for_model(model) - # print(avg_bits) - # assert target_bits - 0.1 < avg_bits <= target_bits+1e-3 - - def test_min_target_bits(self): - model_name = "/models/opt-125m" - target_bits = 4.644 - scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) - model, layer_config = ar.quantize() - # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) - avg_bits, _ = compute_avg_bits_for_model(model) - print(avg_bits) - assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - - def test_max_target_bits(self): - model_name = "/models/opt-125m" - target_bits = 8.211 - scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) - model, layer_config = ar.quantize() - # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) - avg_bits, _ = compute_avg_bits_for_model(model) - print(avg_bits) - assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + @multi_card + def test_multi_card(self): + model_name = "/models/Qwen3-8B" + target_bits = 5.254 + for device_map in ["auto","0,1","0",None]: + scheme = AutoScheme(avg_bits=target_bits, options=("NVFP4")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, device_map=device_map) + model, layer_config = ar.quantize() + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits+1e-3 - def test_patch_scheme(self): - model_name = "/models/opt-125m" - target_bits = 5 - scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, group_size=32) - model, layer_config = ar.quantize() - for n, m in model.named_modules(): - if hasattr(m, "group_size"): - self.assertEqual(m.group_size, 32) - avg_bits, _ = compute_avg_bits_for_model(model) - print(avg_bits) - assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + @multi_card + def test_dict_device_map(self): + model_name = "/models/Qwen3-8B" + target_bits = 8.755 + device_map = {"up_proj":0,"down_proj":1} - def test_layer_config(self): - target_bits = 3.0 - model_name = "/models/opt-125m" - scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) - user_layer_config = {"model.decoder.layers.10.fc1": {"bits": 8, "group_size": 32, "sym": False}} - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, layer_config=user_layer_config) + # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP8")) + ar = AutoRound(model=model_name, scheme="W4A16", iters=0, nsamples=1, device_map=device_map) model, layer_config = ar.quantize() - self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["bits"], 8) - self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["sym"], False) - self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["group_size"], 32) - layer = get_module(model, "model.decoder.layers.10.fc1") - self.assertEqual(layer.bits, 8) - self.assertEqual(layer.sym, False) - self.assertEqual(layer.group_size, 32) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - def test_lm_head_and_mix_dtype(self): + @multi_card + def test_dict_device_map(self): # TODO rtn mode has bug model_name = "/models/Qwen3-8B" - target_bits = 6 - scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, quant_lm_head=True) + target_bits = 8.755 + device_map = {"up_proj":0,"down_proj":1} + + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP8")) + ar = AutoRound(model=model_name, scheme=scheme, iters=1, nsamples=1, device_map=device_map) model, layer_config = ar.quantize() - self.assertLessEqual(layer_config["lm_head"]["bits"], 8) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - def test_auto_scheme_export(self): - model_name = "/models/opt-125m" - scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) - ar = AutoRound(model=model_name, scheme=scheme) - ar.quantize_and_save(self.save_dir) - model_args = f"pretrained={self.save_dir}" - result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto") - print(result["results"]["lambada_openai"]["acc,none"]) - self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25) - shutil.rmtree(self.save_dir, ignore_errors=True) + # def test_min_target_bits(self): + # model_name = "/models/opt-125m" + # target_bits = 4.644 + # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + # model, layer_config = ar.quantize() + # # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + # avg_bits, _ = compute_avg_bits_for_model(model) + # print(avg_bits) + # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + # + # def test_max_target_bits(self): + # model_name = "/models/opt-125m" + # target_bits = 8.211 + # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + # model, layer_config = ar.quantize() + # # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + # avg_bits, _ = compute_avg_bits_for_model(model) + # print(avg_bits) + # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + # + # def test_patch_scheme(self): + # model_name = "/models/opt-125m" + # target_bits = 5 + # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, group_size=32) + # model, layer_config = ar.quantize() + # for n, m in model.named_modules(): + # if hasattr(m, "group_size"): + # self.assertEqual(m.group_size, 32) + # avg_bits, _ = compute_avg_bits_for_model(model) + # print(avg_bits) + # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + # + # def test_layer_config(self): + # target_bits = 3.0 + # model_name = "/models/opt-125m" + # scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) + # user_layer_config = {"model.decoder.layers.10.fc1": {"bits": 8, "group_size": 32, "sym": False}} + # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, layer_config=user_layer_config) + # model, layer_config = ar.quantize() + # self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["bits"], 8) + # self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["sym"], False) + # self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["group_size"], 32) + # layer = get_module(model, "model.decoder.layers.10.fc1") + # self.assertEqual(layer.bits, 8) + # self.assertEqual(layer.sym, False) + # self.assertEqual(layer.group_size, 32) + # avg_bits, _ = compute_avg_bits_for_model(model) + # print(avg_bits) + # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + # + # def test_lm_head_and_mix_dtype(self): + # model_name = "/models/Qwen3-8B" + # target_bits = 6 + # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, quant_lm_head=True) + # model, layer_config = ar.quantize() + # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + # avg_bits, _ = compute_avg_bits_for_model(model) + # print(avg_bits) + # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + # + # def test_auto_scheme_export(self): + # model_name = "/models/opt-125m" + # scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) + # ar = AutoRound(model=model_name, scheme=scheme) + # ar.quantize_and_save(self.save_dir) + # model_args = f"pretrained={self.save_dir}" + # result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto") + # print(result["results"]["lambada_openai"]["acc,none"]) + # self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25) + # shutil.rmtree(self.save_dir, ignore_errors=True) From ab81181c97a4840801b6ebaef5fc7cc6cc2c8342 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 09:25:17 +0000 Subject: [PATCH 58/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/gen_auto_scheme.py | 11 ++++-- auto_round/auto_schemes/utils.py | 43 +++++++++------------- auto_round/compressors/base.py | 10 ++++- auto_round/utils.py | 3 -- test/test_cuda/test_auto_scheme.py | 10 ++--- 5 files changed, 39 insertions(+), 38 deletions(-) diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index c5e92ef4c..12e956eba 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -32,7 +32,7 @@ def __init__( quant_layer_names: Iterable[str], fixed_layer_scheme: dict[str, dict], dataset: str = "pile-10k", # TODO use auto-round dataset - device_map:Union[str, torch.device, int, dict, None] = None, + device_map: Union[str, torch.device, int, dict, None] = None, tokenizer=None, ): self.auto_scheme = auto_scheme @@ -72,8 +72,13 @@ def get_layer_config(self): method_name = self.auto_scheme.method method_func = AUTO_SCHEMES_METHODS[method_name] layer_config = method_func( - self.auto_scheme, self.model, self.quant_layer_names, self.fixed_layer_scheme, - self.dataset, self.tokenizer,device_map=self.device_map, + self.auto_scheme, + self.model, + self.quant_layer_names, + self.fixed_layer_scheme, + self.dataset, + self.tokenizer, + device_map=self.device_map, ) return layer_config diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index 7e44eceb6..82bdcb99d 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -15,7 +15,7 @@ from typing import Iterable, Union import torch -from accelerate import infer_auto_device_map, dispatch_model +from accelerate import dispatch_model, infer_auto_device_map from accelerate.utils import get_balanced_memory from auto_round.low_cpu_mem import get_module @@ -24,10 +24,10 @@ def apply_quant_scheme( - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - scheme: Union[str, dict], # TODO add scale_dtype + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scheme: Union[str, dict], # TODO add scale_dtype ) -> None: """Apply a quantization scheme to each quantized layer. @@ -48,7 +48,7 @@ def apply_quant_scheme( def remove_quant_scheme( - model: torch.nn.Module, + model: torch.nn.Module, ) -> None: """Remove attributes corresponding to the applied quantization scheme. @@ -63,11 +63,11 @@ def remove_quant_scheme( def compute_avg_bits_for_scheme( - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - scheme: Union[str, dict, None] = None, - ignore_scale_zp_bits: bool = False, + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + scheme: Union[str, dict, None] = None, + ignore_scale_zp_bits: bool = False, ) -> tuple[float, float]: """Compute the average and total bit usage for the given quantization scheme. @@ -133,8 +133,8 @@ def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: boo def compute_layer_bits( - layer: torch.nn.Module, - ignore_scale_zp_bits: bool = False, + layer: torch.nn.Module, + ignore_scale_zp_bits: bool = False, ) -> tuple[int, float]: """Compute total and average bitwidth for a single quantized layer. @@ -273,8 +273,9 @@ def parse_all_available_device(device_map: Union[str, torch.device, int, dict, N # Important Notice This dispatch does not follow dict device_map, just extract all available devices and use them -def dispatch_model_by_all_available_devices(model: torch.nn.Module, - device_map: Union[str, int, dict, None]) -> torch.nn.Module: +def dispatch_model_by_all_available_devices( + model: torch.nn.Module, device_map: Union[str, int, dict, None] +) -> torch.nn.Module: if device_map is None: device_map = 0 @@ -285,11 +286,7 @@ def dispatch_model_by_all_available_devices(model: torch.nn.Module, max_memory=None, no_split_module_classes=no_split_modules, ) - device_map = infer_auto_device_map( - model, - max_memory=max_memory, - no_split_module_classes=no_split_modules - ) + device_map = infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=no_split_modules) model = dispatch_model(model, device_map=device_map) return model @@ -317,10 +314,6 @@ def dispatch_model_by_all_available_devices(model: torch.nn.Module, raise ValueError(f"Unsupported device {device} in device_map: {device_map}") new_max_memory[device] = max_memory[device] - device_map = infer_auto_device_map( - model, - max_memory=max_memory, - no_split_module_classes=no_split_modules - ) + device_map = infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=no_split_modules) model = dispatch_model(model, device_map=device_map) return model diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index d3207c2bf..db7e7a190 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -346,7 +346,13 @@ def __init__( from auto_round.auto_schemes.gen_auto_scheme import GenScheme gen_scheme = GenScheme( - scheme, self.model, quant_layer_names, fixed_layer_scheme_new, dataset, device_map=device_map,tokenizer=self.tokenizer + scheme, + self.model, + quant_layer_names, + fixed_layer_scheme_new, + dataset, + device_map=device_map, + tokenizer=self.tokenizer, ) self.layer_config = gen_scheme.get_layer_config() @@ -1581,7 +1587,7 @@ def _quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) if self.device_map is not None: accelerate.hooks.remove_hook_from_submodules(block) - if (is_nv_fp(self.act_data_type) ) or is_static_wfp8afp8(self): + if (is_nv_fp(self.act_data_type)) or is_static_wfp8afp8(self): # enable moe experts act_max automatic generation for Linear set_amax_for_all_moe_layers(block, attr_name="act_max") # Normalize imatrix and quantize layers diff --git a/auto_round/utils.py b/auto_round/utils.py index 7130fc054..51defb832 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -3042,6 +3042,3 @@ def is_diffusion_model(model_or_path: Union[str, object]) -> bool: return isinstance(model_or_path, pipeline_utils.DiffusionPipeline) else: return False - - - diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index f8c36476f..525a8d508 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -29,19 +29,19 @@ def tearDownClass(self): def test_multi_card(self): model_name = "/models/Qwen3-8B" target_bits = 5.254 - for device_map in ["auto","0,1","0",None]: + for device_map in ["auto", "0,1", "0", None]: scheme = AutoScheme(avg_bits=target_bits, options=("NVFP4")) ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, device_map=device_map) model, layer_config = ar.quantize() avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) - assert target_bits - 0.1 < avg_bits <= target_bits+1e-3 + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 @multi_card def test_dict_device_map(self): model_name = "/models/Qwen3-8B" target_bits = 8.755 - device_map = {"up_proj":0,"down_proj":1} + device_map = {"up_proj": 0, "down_proj": 1} # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP8")) ar = AutoRound(model=model_name, scheme="W4A16", iters=0, nsamples=1, device_map=device_map) @@ -51,10 +51,10 @@ def test_dict_device_map(self): assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 @multi_card - def test_dict_device_map(self): # TODO rtn mode has bug + def test_dict_device_map(self): # TODO rtn mode has bug model_name = "/models/Qwen3-8B" target_bits = 8.755 - device_map = {"up_proj":0,"down_proj":1} + device_map = {"up_proj": 0, "down_proj": 1} scheme = AutoScheme(avg_bits=target_bits, options=("MXFP8")) ar = AutoRound(model=model_name, scheme=scheme, iters=1, nsamples=1, device_map=device_map) From 48e4feb995ba410874df6d9da1a296e0b2f1f2f6 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Mon, 13 Oct 2025 17:32:20 +0800 Subject: [PATCH 59/88] update ut --- test/test_cuda/test_auto_scheme.py | 157 ++++++++++++++--------------- 1 file changed, 73 insertions(+), 84 deletions(-) diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 525a8d508..7397f7312 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -37,103 +37,92 @@ def test_multi_card(self): print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + + @multi_card - def test_dict_device_map(self): + def test_dict_device_map(self): # TODO rtn mode has bug model_name = "/models/Qwen3-8B" target_bits = 8.755 device_map = {"up_proj": 0, "down_proj": 1} - # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP8")) - ar = AutoRound(model=model_name, scheme="W4A16", iters=0, nsamples=1, device_map=device_map) + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP8")) + ar = AutoRound(model=model_name, scheme=scheme, iters=1, nsamples=1, device_map=device_map) model, layer_config = ar.quantize() avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - @multi_card - def test_dict_device_map(self): # TODO rtn mode has bug - model_name = "/models/Qwen3-8B" - target_bits = 8.755 - device_map = {"up_proj": 0, "down_proj": 1} + def test_min_target_bits(self): + model_name = "/models/opt-125m" + target_bits = 4.644 + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + model, layer_config = ar.quantize() + # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - scheme = AutoScheme(avg_bits=target_bits, options=("MXFP8")) - ar = AutoRound(model=model_name, scheme=scheme, iters=1, nsamples=1, device_map=device_map) + def test_max_target_bits(self): + model_name = "/models/opt-125m" + target_bits = 8.211 + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + model, layer_config = ar.quantize() + # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + + def test_patch_scheme(self): + model_name = "/models/opt-125m" + target_bits = 5 + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, group_size=32) + model, layer_config = ar.quantize() + for n, m in model.named_modules(): + if hasattr(m, "group_size"): + self.assertEqual(m.group_size, 32) + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + + def test_layer_config(self): + target_bits = 3.0 + model_name = "/models/opt-125m" + scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) + user_layer_config = {"model.decoder.layers.10.fc1": {"bits": 8, "group_size": 32, "sym": False}} + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, layer_config=user_layer_config) + model, layer_config = ar.quantize() + self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["bits"], 8) + self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["sym"], False) + self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["group_size"], 32) + layer = get_module(model, "model.decoder.layers.10.fc1") + self.assertEqual(layer.bits, 8) + self.assertEqual(layer.sym, False) + self.assertEqual(layer.group_size, 32) + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + + def test_lm_head_and_mix_dtype(self): + model_name = "/models/Qwen3-8B" + target_bits = 6 + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, quant_lm_head=True) model, layer_config = ar.quantize() + self.assertLessEqual(layer_config["lm_head"]["bits"], 8) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - # def test_min_target_bits(self): - # model_name = "/models/opt-125m" - # target_bits = 4.644 - # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) - # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) - # model, layer_config = ar.quantize() - # # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) - # avg_bits, _ = compute_avg_bits_for_model(model) - # print(avg_bits) - # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - # - # def test_max_target_bits(self): - # model_name = "/models/opt-125m" - # target_bits = 8.211 - # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) - # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) - # model, layer_config = ar.quantize() - # # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) - # avg_bits, _ = compute_avg_bits_for_model(model) - # print(avg_bits) - # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - # - # def test_patch_scheme(self): - # model_name = "/models/opt-125m" - # target_bits = 5 - # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) - # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, group_size=32) - # model, layer_config = ar.quantize() - # for n, m in model.named_modules(): - # if hasattr(m, "group_size"): - # self.assertEqual(m.group_size, 32) - # avg_bits, _ = compute_avg_bits_for_model(model) - # print(avg_bits) - # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - # - # def test_layer_config(self): - # target_bits = 3.0 - # model_name = "/models/opt-125m" - # scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) - # user_layer_config = {"model.decoder.layers.10.fc1": {"bits": 8, "group_size": 32, "sym": False}} - # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, layer_config=user_layer_config) - # model, layer_config = ar.quantize() - # self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["bits"], 8) - # self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["sym"], False) - # self.assertEqual(layer_config["model.decoder.layers.10.fc1"]["group_size"], 32) - # layer = get_module(model, "model.decoder.layers.10.fc1") - # self.assertEqual(layer.bits, 8) - # self.assertEqual(layer.sym, False) - # self.assertEqual(layer.group_size, 32) - # avg_bits, _ = compute_avg_bits_for_model(model) - # print(avg_bits) - # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - # - # def test_lm_head_and_mix_dtype(self): - # model_name = "/models/Qwen3-8B" - # target_bits = 6 - # scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) - # ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, quant_lm_head=True) - # model, layer_config = ar.quantize() - # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) - # avg_bits, _ = compute_avg_bits_for_model(model) - # print(avg_bits) - # assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - # - # def test_auto_scheme_export(self): - # model_name = "/models/opt-125m" - # scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) - # ar = AutoRound(model=model_name, scheme=scheme) - # ar.quantize_and_save(self.save_dir) - # model_args = f"pretrained={self.save_dir}" - # result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto") - # print(result["results"]["lambada_openai"]["acc,none"]) - # self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25) - # shutil.rmtree(self.save_dir, ignore_errors=True) + def test_auto_scheme_export(self): + model_name = "/models/opt-125m" + scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) + ar = AutoRound(model=model_name, scheme=scheme) + ar.quantize_and_save(self.save_dir) + model_args = f"pretrained={self.save_dir}" + result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto") + print(result["results"]["lambada_openai"]["acc,none"]) + self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25) + shutil.rmtree(self.save_dir, ignore_errors=True) From da7eac140dca0bc2875c2199e328046c87a664cc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 09:32:57 +0000 Subject: [PATCH 60/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/test_cuda/test_auto_scheme.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 7397f7312..b1b5b923c 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -37,8 +37,6 @@ def test_multi_card(self): print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - - @multi_card def test_dict_device_map(self): # TODO rtn mode has bug model_name = "/models/Qwen3-8B" From 2b9c5bbacf4b1ab0f84ae41a28b0fb9b664fd25e Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Tue, 14 Oct 2025 11:28:47 +0800 Subject: [PATCH 61/88] support large models --- auto_round/compressors/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index db7e7a190..6c243aa61 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -26,6 +26,7 @@ import accelerate import torch from accelerate.big_modeling import dispatch_model, infer_auto_device_map +from accelerate.utils import get_balanced_memory from torch import autocast from tqdm import tqdm from transformers import set_seed @@ -2075,8 +2076,12 @@ def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, l if str(self.model.device) == "cpu" and ( self.device.startswith("xpu") or self.device.startswith("cuda") ): - max_memory = get_max_vram() # TODO model is not evenly split no_split_modules = getattr(self.model, "_no_split_modules", []) + max_memory = get_balanced_memory( + self.model, + max_memory=None, + no_split_module_classes=no_split_modules, + ) device_map = infer_auto_device_map( self.model, max_memory=max_memory, no_split_module_classes=no_split_modules ) From fcfb9c6a2268d2dd5cb61e2b91bad31d9d5ee506 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Tue, 14 Oct 2025 14:32:35 +0800 Subject: [PATCH 62/88] support shared layers --- auto_round/auto_schemes/utils.py | 118 ++++++++++++++++++++++++++++- auto_round/schemes.py | 2 +- test/test_cuda/test_auto_scheme.py | 34 ++++++++- 3 files changed, 150 insertions(+), 4 deletions(-) diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index 82bdcb99d..2b5ceb188 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -20,8 +20,12 @@ from auto_round.low_cpu_mem import get_module from auto_round.schemes import QuantizationScheme, preset_name_to_scheme -from auto_round.utils import check_to_quantized, get_layer_features, is_hpex_available +from auto_round.utils import check_to_quantized, get_layer_features, is_hpex_available, get_block_names, \ + SUPPORTED_LAYER_TYPES +import re +import torch +from typing import Union, Iterable def apply_quant_scheme( model: torch.nn.Module, @@ -317,3 +321,115 @@ def dispatch_model_by_all_available_devices( device_map = infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=no_split_modules) model = dispatch_model(model, device_map=device_map) return model + + +def merge_lists_unionfind(list_of_lists): + parent = {} + + def find(x): + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + def union(x, y): + root_x, root_y = find(x), find(y) + if root_x != root_y: + parent[root_y] = root_x + + # 初始化并查集 + for lst in list_of_lists: + for item in lst: + if item not in parent: + parent[item] = item + for i in range(1, len(lst)): + union(lst[0], lst[i]) + + # 收集结果 + groups = {} + for item in parent: + root = find(item) + groups.setdefault(root, []).append(item) + return list(groups.values()) + +def parse_shared_layers( + model: torch.nn.Module, + shared_patterns: Iterable[Iterable[str]] +) -> list[list[str]]: + """ + Parse shared layer groups based on regex or substring matches. + + Args: + model (torch.nn.Module): The model whose modules will be analyzed. + shared_patterns (Iterable[Iterable[str]]): + Each inner iterable defines one shared group. Each element can be: + - a string: checked by full-name or substring match + - a regex pattern: checked by re.fullmatch or re.search + + Returns: + list[list[str]]: A list of matched shared layer groups. + """ + if not shared_patterns: + return [] + # Retrieve all high-level block names (for example, transformer blocks) + for n,m in model.named_modules(): + m.tmp_name = n # attach global name + + block_names = get_block_names(model, quant_vision=True) + block_names = [item for sublist in block_names for item in sublist] + + # Collect all supported layer names from the model + supported_layer_names = [ + name for name, module in model.named_modules() + if type(module) in SUPPORTED_LAYER_TYPES + ] + + # Separate groups into those already fully matched and those requiring pattern matching + direct_match_groups = [] + fuzzy_match_groups = [] + for group in shared_patterns: + match_status = {name: (name in supported_layer_names) for name in group} + if all(match_status.values()): + direct_match_groups.append(list(match_status.keys())) + else: + fuzzy_match_groups.append(match_status) + + matched_groups = list(direct_match_groups) + + # Search each block for modules matching remaining patterns + for block_name in block_names: + block_module = get_module(model,block_name) + block_layer_local_names = [ + name for name, module in block_module.named_modules() + if type(module) in SUPPORTED_LAYER_TYPES + ] + block_layer_names = [] + for name in block_layer_local_names: + module = get_module(block_module,name) + block_layer_names.append(module.tmp_name) + + + for group in fuzzy_match_groups: + matched_layers = set() + for pattern, is_direct in group.items(): + if is_direct: + matched_layers.add(pattern) + continue + + for layer_name in block_layer_names: + # Try regex match first + try: + if re.fullmatch(pattern, layer_name) or re.search(pattern, layer_name): + matched_layers.add(layer_name) + continue + except re.error: + pass # Not a valid regex, fallback to substring matching + + # Substring or partial match + if pattern in layer_name: + matched_layers.add(layer_name) + + if matched_layers: + matched_groups.append(sorted(matched_layers)) + matched_groups = merge_lists_unionfind(matched_groups) + return matched_groups \ No newline at end of file diff --git a/auto_round/schemes.py b/auto_round/schemes.py index fa4cbe025..471517931 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -266,7 +266,7 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: @dataclass class AutoScheme: avg_bits: float - options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str]]] + options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str],...]] shared_layers: Optional[Iterable[Iterable[str]]] = None method: str = "default" ignore_scale_zp_bits: bool = False diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index b1b5b923c..2cb6cfa13 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -3,7 +3,6 @@ import shutil import sys import unittest - from auto_round.testing_utils import multi_card sys.path.insert(0, "../..") @@ -25,6 +24,37 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) + def test_shared_layers(self): + model_name = "/models/opt-125m" + from transformers import AutoModelForCausalLM, AutoTokenizer + model = AutoModelForCausalLM.from_pretrained(model_name) + shared_layers = [["*.self_attn.k_proj", "v_proj", "q_proj", "out_proj"], + ("model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"), ("fc1", "fc2")] + from auto_round.auto_schemes.utils import parse_shared_layers + res = parse_shared_layers(model, shared_layers) + self.assertEqual(len(res), 24) + assert ['model.decoder.layers.2.self_attn.out_proj', 'model.decoder.layers.2.self_attn.q_proj', + 'model.decoder.layers.2.self_attn.v_proj'] in res + assert ['model.decoder.layers.6.fc1', 'model.decoder.layers.6.fc2'] in res + assert ['model.decoder.layers.7.fc1', 'model.decoder.layers.7.fc2'] in res + target_bits=5.0 + scheme = AutoScheme(avg_bits=target_bits, options=("W4A16","MXFP8"),shared_layers=shared_layers) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + model, layer_config = ar.quantize() + avg_bits, _ = compute_avg_bits_for_model(model) + for names in res: + bits=[] + for name in names: + module = get_module(model, name) + if hasattr(module,"orig_layer"): + bits.append(module.orig_layer.bits) + else: + bits.append(module.bits) + bits=set(bits) + self.assertEqual(len(bits), 1) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + @multi_card def test_multi_card(self): model_name = "/models/Qwen3-8B" @@ -44,7 +74,7 @@ def test_dict_device_map(self): # TODO rtn mode has bug device_map = {"up_proj": 0, "down_proj": 1} scheme = AutoScheme(avg_bits=target_bits, options=("MXFP8")) - ar = AutoRound(model=model_name, scheme=scheme, iters=1, nsamples=1, device_map=device_map) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, device_map=device_map) model, layer_config = ar.quantize() avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) From 91ec73da984fb4f12a08d5973556f8223e1a5e81 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Oct 2025 06:36:26 +0000 Subject: [PATCH 63/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/utils.py | 38 +++++++++++++----------------- auto_round/schemes.py | 2 +- test/test_cuda/test_auto_scheme.py | 31 +++++++++++++++--------- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index 2b5ceb188..2c435d31e 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import re from dataclasses import asdict, fields from typing import Iterable, Union @@ -20,12 +21,14 @@ from auto_round.low_cpu_mem import get_module from auto_round.schemes import QuantizationScheme, preset_name_to_scheme -from auto_round.utils import check_to_quantized, get_layer_features, is_hpex_available, get_block_names, \ - SUPPORTED_LAYER_TYPES +from auto_round.utils import ( + SUPPORTED_LAYER_TYPES, + check_to_quantized, + get_block_names, + get_layer_features, + is_hpex_available, +) -import re -import torch -from typing import Union, Iterable def apply_quant_scheme( model: torch.nn.Module, @@ -352,10 +355,8 @@ def union(x, y): groups.setdefault(root, []).append(item) return list(groups.values()) -def parse_shared_layers( - model: torch.nn.Module, - shared_patterns: Iterable[Iterable[str]] -) -> list[list[str]]: + +def parse_shared_layers(model: torch.nn.Module, shared_patterns: Iterable[Iterable[str]]) -> list[list[str]]: """ Parse shared layer groups based on regex or substring matches. @@ -372,17 +373,14 @@ def parse_shared_layers( if not shared_patterns: return [] # Retrieve all high-level block names (for example, transformer blocks) - for n,m in model.named_modules(): - m.tmp_name = n # attach global name + for n, m in model.named_modules(): + m.tmp_name = n # attach global name block_names = get_block_names(model, quant_vision=True) block_names = [item for sublist in block_names for item in sublist] # Collect all supported layer names from the model - supported_layer_names = [ - name for name, module in model.named_modules() - if type(module) in SUPPORTED_LAYER_TYPES - ] + supported_layer_names = [name for name, module in model.named_modules() if type(module) in SUPPORTED_LAYER_TYPES] # Separate groups into those already fully matched and those requiring pattern matching direct_match_groups = [] @@ -398,17 +396,15 @@ def parse_shared_layers( # Search each block for modules matching remaining patterns for block_name in block_names: - block_module = get_module(model,block_name) + block_module = get_module(model, block_name) block_layer_local_names = [ - name for name, module in block_module.named_modules() - if type(module) in SUPPORTED_LAYER_TYPES + name for name, module in block_module.named_modules() if type(module) in SUPPORTED_LAYER_TYPES ] block_layer_names = [] for name in block_layer_local_names: - module = get_module(block_module,name) + module = get_module(block_module, name) block_layer_names.append(module.tmp_name) - for group in fuzzy_match_groups: matched_layers = set() for pattern, is_direct in group.items(): @@ -432,4 +428,4 @@ def parse_shared_layers( if matched_layers: matched_groups.append(sorted(matched_layers)) matched_groups = merge_lists_unionfind(matched_groups) - return matched_groups \ No newline at end of file + return matched_groups diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 471517931..789cbe1cb 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -266,7 +266,7 @@ def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str: @dataclass class AutoScheme: avg_bits: float - options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str],...]] + options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str], ...]] shared_layers: Optional[Iterable[Iterable[str]]] = None method: str = "default" ignore_scale_zp_bits: bool = False diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 2cb6cfa13..451867a1e 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -3,6 +3,7 @@ import shutil import sys import unittest + from auto_round.testing_utils import multi_card sys.path.insert(0, "../..") @@ -27,30 +28,38 @@ def tearDownClass(self): def test_shared_layers(self): model_name = "/models/opt-125m" from transformers import AutoModelForCausalLM, AutoTokenizer + model = AutoModelForCausalLM.from_pretrained(model_name) - shared_layers = [["*.self_attn.k_proj", "v_proj", "q_proj", "out_proj"], - ("model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"), ("fc1", "fc2")] + shared_layers = [ + ["*.self_attn.k_proj", "v_proj", "q_proj", "out_proj"], + ("model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"), + ("fc1", "fc2"), + ] from auto_round.auto_schemes.utils import parse_shared_layers + res = parse_shared_layers(model, shared_layers) self.assertEqual(len(res), 24) - assert ['model.decoder.layers.2.self_attn.out_proj', 'model.decoder.layers.2.self_attn.q_proj', - 'model.decoder.layers.2.self_attn.v_proj'] in res - assert ['model.decoder.layers.6.fc1', 'model.decoder.layers.6.fc2'] in res - assert ['model.decoder.layers.7.fc1', 'model.decoder.layers.7.fc2'] in res - target_bits=5.0 - scheme = AutoScheme(avg_bits=target_bits, options=("W4A16","MXFP8"),shared_layers=shared_layers) + assert [ + "model.decoder.layers.2.self_attn.out_proj", + "model.decoder.layers.2.self_attn.q_proj", + "model.decoder.layers.2.self_attn.v_proj", + ] in res + assert ["model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"] in res + assert ["model.decoder.layers.7.fc1", "model.decoder.layers.7.fc2"] in res + target_bits = 5.0 + scheme = AutoScheme(avg_bits=target_bits, options=("W4A16", "MXFP8"), shared_layers=shared_layers) ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) model, layer_config = ar.quantize() avg_bits, _ = compute_avg_bits_for_model(model) for names in res: - bits=[] + bits = [] for name in names: module = get_module(model, name) - if hasattr(module,"orig_layer"): + if hasattr(module, "orig_layer"): bits.append(module.orig_layer.bits) else: bits.append(module.bits) - bits=set(bits) + bits = set(bits) self.assertEqual(len(bits), 1) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 From 9a2738ace6dacf2a4f1e1b05a3422c0f08b1d782 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Tue, 14 Oct 2025 15:16:40 +0800 Subject: [PATCH 64/88] update a little --- auto_round/compressors/base.py | 36 ++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index b003ec6cc..d5110348c 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -320,8 +320,40 @@ def __init__( if isinstance(scheme, AutoScheme): if self.mllm: - logger.info("AutoScheme with MLLM is not supported yet.") - sys.exit(1) + logger.info("AutoScheme is not yet supported for multimodal LLMs.") + sys.exit(-1) + + if getattr(model, "is_fp8", False): + logger.info("AutoScheme does not currently support FP8 models.") + sys.exit(-1) + + all_dtypes = [] + for option in scheme.options: + # Skip pure BF16 option + if option == "BF16": + continue + + # Resolve the quantization scheme or data type + dtype = "int" + if isinstance(option, str): + option = preset_name_to_scheme(option) + + if isinstance(option, QuantizationScheme): + dtype = option.data_type + elif isinstance(option, dict): + dtype = option.get("data_type", "int") + + all_dtypes.append(dtype) + + # Check for mixed data types + unique_dtypes = set(all_dtypes) + if len(unique_dtypes) > 1: + logger.warning( + "Models with mixed data_types " + "cannot yet be exported to real formats except GGUF. " + "Please save the model using the `fake` format for now." + ) + layer_config, self.has_qlayer_outside_block = set_layer_config( self.model, self.layer_config, From 6132faa6938457998ba7fe864b48bab81c59954e Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Tue, 14 Oct 2025 15:54:45 +0800 Subject: [PATCH 65/88] fix gguf issue --- auto_round/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/auto_round/utils.py b/auto_round/utils.py index 51defb832..37460f119 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2997,7 +2997,10 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str for emd_name in embedding_layer_names: if emd_name in layer_config: continue - cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]] + if not tie_word_embeddings: + cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]] + else: + cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]] cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype} layer_config[emd_name] = cfg From c111608403f370220aa75fbf9969dd30cdef33d6 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Tue, 14 Oct 2025 17:41:02 +0800 Subject: [PATCH 66/88] support gguf --- auto_round/auto_schemes/utils.py | 13 +- test/test_cpu/test_gguf_format.py | 485 ++++++++++++++--------------- test/test_cuda/test_auto_scheme.py | 20 ++ 3 files changed, 271 insertions(+), 247 deletions(-) diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index 2c435d31e..d75959ce6 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -98,6 +98,8 @@ def compute_avg_bits_for_scheme( for name in quant_layer_names: module = get_module(model, name) + # if isinstance(module,torch.nn.Embedding): + # continue if not hasattr(module, "weight"): continue total_params += module.weight.numel() @@ -130,6 +132,8 @@ def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: boo continue if not hasattr(module, "weight"): continue + # if isinstance(module,torch.nn.Embedding): # Tricky setting for Embedding + # continue total_params += module.weight.numel() layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits) total_quantized_bits += layer_bits @@ -161,9 +165,11 @@ def compute_layer_bits( # Unquantized layer or ignoring scale/zp overhead if weight_bits >= 16 or ignore_scale_zp_bits: - if super_weight_bits is not None: # reset gguf 16 bits to 32 bits, TODO gguf q4_0, q4_1 may have bug - return 32 * n_param, 32 - return weight_bits * n_param, 16.0 + if super_weight_bits is not None: # reset gguf 16 bits to 32 bits, TODO gguf q4_0, q4_1 have bug (wenhua) + if weight_bits >= 16: + return 32 * n_param, 32 + + return weight_bits * n_param, weight_bits if weight_bits<16 else 16 in_features, out_features = get_layer_features(layer) @@ -192,7 +198,6 @@ def compute_layer_bits( avg_bits = total_bits / n_param return total_bits, avg_bits - def parse_all_available_device(device_map: Union[str, torch.device, int, dict, None] = None) -> list: """ Parse the device map and return a list of all available devices. diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py index 96b489ffe..44453af4c 100644 --- a/test/test_cpu/test_gguf_format.py +++ b/test/test_cpu/test_gguf_format.py @@ -23,10 +23,10 @@ def __iter__(self): class TestGGUF(unittest.TestCase): @classmethod def setUpClass(self): - self.model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-0.5B-Instruct" - self.model_name = "Qwen/Qwen2.5-0.5B-Instruct" - self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True) - self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True) + # self.model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-0.5B-Instruct" + # self.model_name = "Qwen/Qwen2.5-0.5B-Instruct" + # self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True) + # self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True) self.llm_dataloader = LLMDataLoader() @classmethod @@ -34,60 +34,40 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) - def test_basic_usage(self): - python_path = sys.executable - res = os.system( - f"cd ../.. && {python_path} -m auto_round --model /tf_dataset/auto_round/models/benzart/gemma-2b-it-fine-tuning-for-code-test " - f" --bs 16 --iters 0 --nsamples 1 --format gguf:q4_k_m" - ) - if res > 0 or res == -1: - assert False, "cmd line test fail, please have a check" - shutil.rmtree("./saved", ignore_errors=True) - - res = os.system( - f"cd ../.. && {python_path} -m auto_round --model {self.model_name}" - f" --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0" - ) - if res > 0 or res == -1: - assert False, "cmd line test fail, please have a check" - shutil.rmtree("./saved", ignore_errors=True) - - def test_q4_0(self): - bits, group_size, sym = 4, 32, True - autoround = AutoRound( - self.model, - self.tokenizer, - bits=bits, - group_size=group_size, - sym=sym, - iters=1, - data_type="int", - nsamples=1, - seqlen=8, - ) - quantized_model_path = "./saved" - - autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0") - gguf_file = os.listdir(quantized_model_path)[0] - model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") - text = "There is a girl who likes adventure," - inputs = self.tokenizer(text, return_tensors="pt").to(model.device) - print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) - - # from auto_round.eval.evaluation import simple_evaluate_user_model - # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16") - # # 0.246 - # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.23) - shutil.rmtree("./saved", ignore_errors=True) - - # def test_q4_1(self): - # bits, group_size, sym = 4, 32, False + # def test_basic_usage(self): + # python_path = sys.executable + # res = os.system( + # f"cd ../.. && {python_path} -m auto_round --model /tf_dataset/auto_round/models/benzart/gemma-2b-it-fine-tuning-for-code-test " + # f" --bs 16 --iters 0 --nsamples 1 --format gguf:q4_k_m" + # ) + # if res > 0 or res == -1: + # assert False, "cmd line test fail, please have a check" + # shutil.rmtree("./saved", ignore_errors=True) + # + # res = os.system( + # f"cd ../.. && {python_path} -m auto_round --model {self.model_name}" + # f" --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0" + # ) + # if res > 0 or res == -1: + # assert False, "cmd line test fail, please have a check" + # shutil.rmtree("./saved", ignore_errors=True) + # + # def test_q4_0(self): + # bits, group_size, sym = 4, 32, True # autoround = AutoRound( - # self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int", nsamples=1 + # self.model, + # self.tokenizer, + # bits=bits, + # group_size=group_size, + # sym=sym, + # iters=1, + # data_type="int", + # nsamples=1, + # seqlen=8, # ) # quantized_model_path = "./saved" # - # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1") + # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0") # gguf_file = os.listdir(quantized_model_path)[0] # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") # text = "There is a girl who likes adventure," @@ -96,223 +76,243 @@ def test_q4_0(self): # # # from auto_round.eval.evaluation import simple_evaluate_user_model # # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16") - # # # 0.23 - # # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22) + # # # 0.246 + # # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.23) # shutil.rmtree("./saved", ignore_errors=True) - - def test_func(self): - bits, group_size, sym = 4, 128, True - autoround = AutoRound( - self.model, - self.tokenizer, - # bits=bits, - # group_size=group_size, - # sym=sym, - iters=1, - nsamples=1, - seqlen=10, - # data_type="int" - ) - quantized_model_path = "./saved" - autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_1") - self.assertTrue(autoround.group_size == 32) - self.assertFalse(autoround.sym) - gguf_file = os.listdir("saved")[0] - model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") - text = "There is a girl who likes adventure," - inputs = self.tokenizer(text, return_tensors="pt").to(model.device) - print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) - shutil.rmtree("./saved", ignore_errors=True) - - # model_name = "Qwen/Qwen2.5-1.5B-Instruct" - # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - # autoround = AutoRound( - # model, - # self.tokenizer, - # bits=3, - # group_size=16, - # sym=True, - # iters=1, - # nsamples=1, - # data_type="int_sym_dq", - # super_group_size=16, - # super_bits=6, - # ) - quantized_model_path = "./saved" - # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s") - # from auto_round.eval.evaluation import simple_evaluate_user_model - # gguf_file = os.listdir("saved")[0] - # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") - # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="lambada_openai", eval_model_dtype="bf16") - # self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.5) - shutil.rmtree("./saved", ignore_errors=True) - # - # def test_q5_k(self): - # model_name = "Qwen/Qwen2.5-1.5B-Instruct" - # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + # # def test_q4_1(self): + # # bits, group_size, sym = 4, 32, False + # # autoround = AutoRound( + # # self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int", nsamples=1 + # # ) + # # quantized_model_path = "./saved" + # # + # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1") + # # gguf_file = os.listdir(quantized_model_path)[0] + # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") + # # text = "There is a girl who likes adventure," + # # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) + # # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) + # # + # # # from auto_round.eval.evaluation import simple_evaluate_user_model + # # # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16") + # # # # 0.23 + # # # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22) + # # shutil.rmtree("./saved", ignore_errors=True) + # + # def test_func(self): + # bits, group_size, sym = 4, 128, True # autoround = AutoRound( - # model, + # self.model, # self.tokenizer, - # bits=5, - # group_size=32, - # sym=False, + # # bits=bits, + # # group_size=group_size, + # # sym=sym, # iters=1, # nsamples=1, - # data_type="int_asym_dq", - # super_group_size=8, - # super_bits=6, + # seqlen=10, + # # data_type="int" # ) # quantized_model_path = "./saved" - # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s") + # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_1") + # self.assertTrue(autoround.group_size == 32) + # self.assertFalse(autoround.sym) # gguf_file = os.listdir("saved")[0] # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") # text = "There is a girl who likes adventure," # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) # shutil.rmtree("./saved", ignore_errors=True) - - # def test_q6_k(self): - # model_name = "Qwen/Qwen2.5-1.5B-Instruct" + # + # # model_name = "Qwen/Qwen2.5-1.5B-Instruct" + # # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + # # autoround = AutoRound( + # # model, + # # self.tokenizer, + # # bits=3, + # # group_size=16, + # # sym=True, + # # iters=1, + # # nsamples=1, + # # data_type="int_sym_dq", + # # super_group_size=16, + # # super_bits=6, + # # ) + # quantized_model_path = "./saved" + # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s") + # # from auto_round.eval.evaluation import simple_evaluate_user_model + # # gguf_file = os.listdir("saved")[0] + # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") + # # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="lambada_openai", eval_model_dtype="bf16") + # # self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.5) + # shutil.rmtree("./saved", ignore_errors=True) + # + # # + # # def test_q5_k(self): + # # model_name = "Qwen/Qwen2.5-1.5B-Instruct" + # # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + # # autoround = AutoRound( + # # model, + # # self.tokenizer, + # # bits=5, + # # group_size=32, + # # sym=False, + # # iters=1, + # # nsamples=1, + # # data_type="int_asym_dq", + # # super_group_size=8, + # # super_bits=6, + # # ) + # # quantized_model_path = "./saved" + # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s") + # # gguf_file = os.listdir("saved")[0] + # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") + # # text = "There is a girl who likes adventure," + # # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) + # # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) + # # shutil.rmtree("./saved", ignore_errors=True) + # + # # def test_q6_k(self): + # # model_name = "Qwen/Qwen2.5-1.5B-Instruct" + # # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + # # autoround = AutoRound( + # # model, + # # self.tokenizer, + # # bits=6, + # # group_size=16, + # # sym=True, + # # iters=1, + # # nsamples=1, + # # data_type="int_sym_dq", + # # super_group_size=16, + # # super_bits=8, + # # ) + # # quantized_model_path = "./saved" + # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k") + # # gguf_file = os.listdir("saved")[0] + # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") + # # text = "There is a girl who likes adventure," + # # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) + # # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) + # # shutil.rmtree("./saved", ignore_errors=True) + # + # def test_gguf_baseline(self): + # model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) # autoround = AutoRound( # model, # self.tokenizer, - # bits=6, + # bits=3, # group_size=16, # sym=True, - # iters=1, - # nsamples=1, - # data_type="int_sym_dq", + # iters=0, + # nsamples=8, + # seqlen=2, + # data_type="rtn_int_sym_dq", # super_group_size=16, - # super_bits=8, + # super_bits=6, + # disable_opt_rtn=True, # ) # quantized_model_path = "./saved" - # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k") - # gguf_file = os.listdir("saved")[0] - # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") + # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="fake") + # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto") # text = "There is a girl who likes adventure," # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) # shutil.rmtree("./saved", ignore_errors=True) - - def test_gguf_baseline(self): - model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" - model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - autoround = AutoRound( - model, - self.tokenizer, - bits=3, - group_size=16, - sym=True, - iters=0, - nsamples=8, - seqlen=2, - data_type="rtn_int_sym_dq", - super_group_size=16, - super_bits=6, - disable_opt_rtn=True, - ) - quantized_model_path = "./saved" - autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="fake") - model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto") - text = "There is a girl who likes adventure," - inputs = self.tokenizer(text, return_tensors="pt").to(model.device) - print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) - shutil.rmtree("./saved", ignore_errors=True) - # - # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - # autoround = AutoRound( - # model, - # self.tokenizer, - # bits=5, - # group_size=32, - # sym=True, - # iters=0, - # nsamples=8, - # data_type="int_asym_dq", - # super_group_size=8, - # super_bits=6, - # disable_opt_rtn=True, - # ) - # quantized_model_path = "./saved" - # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q5_k_s,fake") - # model = AutoModelForCausalLM.from_pretrained(quantized_model_path + "/fake", device_map="auto") - # text = "There is a girl who likes adventure," - # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) - # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) - # shutil.rmtree("./saved", ignore_errors=True) - - def test_q4_k_m(self): - model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" - model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - layer_config = { - "lm_head": { - "bits": 4, - "group_size": 32, - "sym": False, - "data_type": "int_asym_dq", - "super_bits": 6, - "super_group_size": 8, - }, - "model.embed_tokens": {"bits": 6, "group_size": 32, "super_bits": 6, "super_group_size": 8}, - "model.layers.12.mlp.gate_proj": {"bits": 3}, - "model.layers.10.mlp.gate_proj": {"bits": 8}, - } - autoround = AutoRound( - model, - tokenizer, - layer_config=layer_config, - iters=0, - seqlen=1, - nsamples=8, - dataset=self.llm_dataloader, - disable_opt_rtn=True, - ) - quantized_model_path = "./saved" - autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake") - self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16) - self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "int_sym_dq") - self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "int_asym_dq") - self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6) - self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4) - self.assertEqual(autoround.model.model.embed_tokens.bits, 6) - self.assertEqual(autoround.model.model.embed_tokens.group_size, 16) - self.assertEqual(autoround.model.model.layers[12].mlp.gate_proj.bits, 3) - self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8) - self.assertEqual(autoround.layer_config["model.layers.10.mlp.gate_proj"]["mostly"], "gguf:q8_0") - shutil.rmtree("./saved", ignore_errors=True) - - model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - autoround = AutoRound(model, tokenizer, iters=0, nsamples=1, seqlen=128, disable_opt_rtn=False) - quantized_model_path = "./saved" - autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake") - shutil.rmtree("./saved", ignore_errors=True) - - def test_all_format(self): - model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" - python_path = sys.executable - # for gguf_format in ["gguf:q4_0", "gguf:q4_1", "gguf:q4_k_m", "gguf:q6_k"]: - for gguf_format in ["gguf:q4_k_m"]: - res = os.system( - f"cd ../.. && {python_path} -m auto_round --model {model_name} " - f" --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format {gguf_format}" - ) - if res > 0 or res == -1: - assert False, "cmd line test fail, please have a check" - shutil.rmtree("../../tmp_autoround", ignore_errors=True) - - res = os.system( - f"cd ../.. && {python_path} -m auto_round --model {model_name}" - f" --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format fake,{gguf_format}" - ) - if res > 0 or res == -1: - assert False, "cmd line test fail, please have a check" - shutil.rmtree("../../tmp_autoround", ignore_errors=True) + # # + # # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + # # autoround = AutoRound( + # # model, + # # self.tokenizer, + # # bits=5, + # # group_size=32, + # # sym=True, + # # iters=0, + # # nsamples=8, + # # data_type="int_asym_dq", + # # super_group_size=8, + # # super_bits=6, + # # disable_opt_rtn=True, + # # ) + # # quantized_model_path = "./saved" + # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q5_k_s,fake") + # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path + "/fake", device_map="auto") + # # text = "There is a girl who likes adventure," + # # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) + # # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) + # # shutil.rmtree("./saved", ignore_errors=True) + # + # def test_q4_k_m(self): + # model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" + # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + # tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + # layer_config = { + # "lm_head": { + # "bits": 4, + # "group_size": 32, + # "sym": False, + # "data_type": "int_asym_dq", + # "super_bits": 6, + # "super_group_size": 8, + # }, + # "model.embed_tokens": {"bits": 6, "group_size": 32, "super_bits": 6, "super_group_size": 8}, + # "model.layers.12.mlp.gate_proj": {"bits": 3}, + # "model.layers.10.mlp.gate_proj": {"bits": 8}, + # } + # autoround = AutoRound( + # model, + # tokenizer, + # layer_config=layer_config, + # iters=0, + # seqlen=1, + # nsamples=8, + # dataset=self.llm_dataloader, + # disable_opt_rtn=True, + # ) + # quantized_model_path = "./saved" + # autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake") + # self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16) + # self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "int_sym_dq") + # self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "int_asym_dq") + # self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6) + # self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4) + # self.assertEqual(autoround.model.model.embed_tokens.bits, 6) + # self.assertEqual(autoround.model.model.embed_tokens.group_size, 16) + # self.assertEqual(autoround.model.model.layers[12].mlp.gate_proj.bits, 3) + # self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8) + # self.assertEqual(autoround.layer_config["model.layers.10.mlp.gate_proj"]["mostly"], "gguf:q8_0") + # shutil.rmtree("./saved", ignore_errors=True) + # + # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + # autoround = AutoRound(model, tokenizer, iters=0, nsamples=1, seqlen=128, disable_opt_rtn=False) + # quantized_model_path = "./saved" + # autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake") + # shutil.rmtree("./saved", ignore_errors=True) + # + # def test_all_format(self): + # model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" + # python_path = sys.executable + # # for gguf_format in ["gguf:q4_0", "gguf:q4_1", "gguf:q4_k_m", "gguf:q6_k"]: + # for gguf_format in ["gguf:q4_k_m"]: + # res = os.system( + # f"cd ../.. && {python_path} -m auto_round --model {model_name} " + # f" --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format {gguf_format}" + # ) + # if res > 0 or res == -1: + # assert False, "cmd line test fail, please have a check" + # shutil.rmtree("../../tmp_autoround", ignore_errors=True) + # + # res = os.system( + # f"cd ../.. && {python_path} -m auto_round --model {model_name}" + # f" --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format fake,{gguf_format}" + # ) + # if res > 0 or res == -1: + # assert False, "cmd line test fail, please have a check" + # shutil.rmtree("../../tmp_autoround", ignore_errors=True) def test_vlm_gguf(self): - model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2-VL-2B-Instruct" + model_name = "/models/Qwen2-VL-2B-Instruct" from auto_round import AutoRoundMLLM from auto_round.utils import mllm_load_model @@ -329,7 +329,6 @@ def test_vlm_gguf(self): autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0") self.assertTrue("mmproj-model.gguf" in os.listdir("./saved")) for file_name in os.listdir(quantized_model_path): - file_name = os.listdir(quantized_model_path)[0] file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2 if file_name == "mmproj-model.gguf": self.assertAlmostEqual(file_size, 2535, delta=1.0) diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 451867a1e..e19a43e95 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -25,6 +25,26 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) + def test_gguf_export(self): + model_name = "/models/Qwen3-0.6B" + target_bits = 3 + scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M"), ignore_scale_zp_bits=True) + ar = AutoRound(model=model_name, scheme=scheme, iters=0) + ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s") + shutil.rmtree("./saved", ignore_errors=True) + + def test_gguf(self): + model_name = "/models/Qwen3-8B" + target_bits = 3 + scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M"), ignore_scale_zp_bits=True) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, disable_opt_rtn=True) + model, layer_config = ar.quantize() + # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) + avg_bits, _ = compute_avg_bits_for_model(model, ignore_scale_zp_bits=True) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + + def test_shared_layers(self): model_name = "/models/opt-125m" from transformers import AutoModelForCausalLM, AutoTokenizer From c74df50851f518de322bfc8e1f3192990e55eb2a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Oct 2025 09:41:59 +0000 Subject: [PATCH 67/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/utils.py | 3 ++- test/test_cuda/test_auto_scheme.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_schemes/utils.py index d75959ce6..73191c40f 100644 --- a/auto_round/auto_schemes/utils.py +++ b/auto_round/auto_schemes/utils.py @@ -169,7 +169,7 @@ def compute_layer_bits( if weight_bits >= 16: return 32 * n_param, 32 - return weight_bits * n_param, weight_bits if weight_bits<16 else 16 + return weight_bits * n_param, min(16, weight_bits) in_features, out_features = get_layer_features(layer) @@ -198,6 +198,7 @@ def compute_layer_bits( avg_bits = total_bits / n_param return total_bits, avg_bits + def parse_all_available_device(device_map: Union[str, torch.device, int, dict, None] = None) -> list: """ Parse the device map and return a list of all available devices. diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index e19a43e95..e11ce1a01 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -44,7 +44,6 @@ def test_gguf(self): print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - def test_shared_layers(self): model_name = "/models/opt-125m" from transformers import AutoModelForCausalLM, AutoTokenizer From ae87b77872faba7859f375d86d32a430964b0445 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Tue, 14 Oct 2025 17:54:18 +0800 Subject: [PATCH 68/88] revert test --- test/test_cpu/test_gguf_format.py | 484 +++++++++++++++--------------- 1 file changed, 242 insertions(+), 242 deletions(-) diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py index 44453af4c..d71920b39 100644 --- a/test/test_cpu/test_gguf_format.py +++ b/test/test_cpu/test_gguf_format.py @@ -23,10 +23,10 @@ def __iter__(self): class TestGGUF(unittest.TestCase): @classmethod def setUpClass(self): - # self.model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-0.5B-Instruct" - # self.model_name = "Qwen/Qwen2.5-0.5B-Instruct" - # self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True) - # self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True) + self.model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-0.5B-Instruct" + self.model_name = "Qwen/Qwen2.5-0.5B-Instruct" + self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True) + self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True) self.llm_dataloader = LLMDataLoader() @classmethod @@ -34,40 +34,60 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) - # def test_basic_usage(self): - # python_path = sys.executable - # res = os.system( - # f"cd ../.. && {python_path} -m auto_round --model /tf_dataset/auto_round/models/benzart/gemma-2b-it-fine-tuning-for-code-test " - # f" --bs 16 --iters 0 --nsamples 1 --format gguf:q4_k_m" - # ) - # if res > 0 or res == -1: - # assert False, "cmd line test fail, please have a check" - # shutil.rmtree("./saved", ignore_errors=True) - # - # res = os.system( - # f"cd ../.. && {python_path} -m auto_round --model {self.model_name}" - # f" --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0" - # ) - # if res > 0 or res == -1: - # assert False, "cmd line test fail, please have a check" - # shutil.rmtree("./saved", ignore_errors=True) - # - # def test_q4_0(self): - # bits, group_size, sym = 4, 32, True + def test_basic_usage(self): + python_path = sys.executable + res = os.system( + f"cd ../.. && {python_path} -m auto_round --model /tf_dataset/auto_round/models/benzart/gemma-2b-it-fine-tuning-for-code-test " + f" --bs 16 --iters 0 --nsamples 1 --format gguf:q4_k_m" + ) + if res > 0 or res == -1: + assert False, "cmd line test fail, please have a check" + shutil.rmtree("./saved", ignore_errors=True) + + res = os.system( + f"cd ../.. && {python_path} -m auto_round --model {self.model_name}" + f" --bs 16 --iters 1 --nsamples 1 --format fake,gguf:q4_0" + ) + if res > 0 or res == -1: + assert False, "cmd line test fail, please have a check" + shutil.rmtree("./saved", ignore_errors=True) + + def test_q4_0(self): + bits, group_size, sym = 4, 32, True + autoround = AutoRound( + self.model, + self.tokenizer, + bits=bits, + group_size=group_size, + sym=sym, + iters=1, + data_type="int", + nsamples=1, + seqlen=8, + ) + quantized_model_path = "./saved" + + autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0") + gguf_file = os.listdir(quantized_model_path)[0] + model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") + text = "There is a girl who likes adventure," + inputs = self.tokenizer(text, return_tensors="pt").to(model.device) + print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) + + # from auto_round.eval.evaluation import simple_evaluate_user_model + # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16") + # # 0.246 + # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.23) + shutil.rmtree("./saved", ignore_errors=True) + + # def test_q4_1(self): + # bits, group_size, sym = 4, 32, False # autoround = AutoRound( - # self.model, - # self.tokenizer, - # bits=bits, - # group_size=group_size, - # sym=sym, - # iters=1, - # data_type="int", - # nsamples=1, - # seqlen=8, + # self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int", nsamples=1 # ) # quantized_model_path = "./saved" # - # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0") + # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1") # gguf_file = os.listdir(quantized_model_path)[0] # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") # text = "There is a girl who likes adventure," @@ -76,243 +96,223 @@ def tearDownClass(self): # # # from auto_round.eval.evaluation import simple_evaluate_user_model # # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16") - # # # 0.246 - # # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.23) + # # # 0.23 + # # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22) # shutil.rmtree("./saved", ignore_errors=True) + + def test_func(self): + bits, group_size, sym = 4, 128, True + autoround = AutoRound( + self.model, + self.tokenizer, + # bits=bits, + # group_size=group_size, + # sym=sym, + iters=1, + nsamples=1, + seqlen=10, + # data_type="int" + ) + quantized_model_path = "./saved" + autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_1") + self.assertTrue(autoround.group_size == 32) + self.assertFalse(autoround.sym) + gguf_file = os.listdir("saved")[0] + model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") + text = "There is a girl who likes adventure," + inputs = self.tokenizer(text, return_tensors="pt").to(model.device) + print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) + shutil.rmtree("./saved", ignore_errors=True) + + # model_name = "Qwen/Qwen2.5-1.5B-Instruct" + # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + # autoround = AutoRound( + # model, + # self.tokenizer, + # bits=3, + # group_size=16, + # sym=True, + # iters=1, + # nsamples=1, + # data_type="int_sym_dq", + # super_group_size=16, + # super_bits=6, + # ) + quantized_model_path = "./saved" + # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s") + # from auto_round.eval.evaluation import simple_evaluate_user_model + # gguf_file = os.listdir("saved")[0] + # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") + # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="lambada_openai", eval_model_dtype="bf16") + # self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.5) + shutil.rmtree("./saved", ignore_errors=True) + # - # # def test_q4_1(self): - # # bits, group_size, sym = 4, 32, False - # # autoround = AutoRound( - # # self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=1, data_type="int", nsamples=1 - # # ) - # # quantized_model_path = "./saved" - # # - # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1") - # # gguf_file = os.listdir(quantized_model_path)[0] - # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") - # # text = "There is a girl who likes adventure," - # # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) - # # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) - # # - # # # from auto_round.eval.evaluation import simple_evaluate_user_model - # # # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16") - # # # # 0.23 - # # # self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22) - # # shutil.rmtree("./saved", ignore_errors=True) - # - # def test_func(self): - # bits, group_size, sym = 4, 128, True + # def test_q5_k(self): + # model_name = "Qwen/Qwen2.5-1.5B-Instruct" + # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) # autoround = AutoRound( - # self.model, + # model, # self.tokenizer, - # # bits=bits, - # # group_size=group_size, - # # sym=sym, + # bits=5, + # group_size=32, + # sym=False, # iters=1, # nsamples=1, - # seqlen=10, - # # data_type="int" + # data_type="int_asym_dq", + # super_group_size=8, + # super_bits=6, # ) # quantized_model_path = "./saved" - # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_1") - # self.assertTrue(autoround.group_size == 32) - # self.assertFalse(autoround.sym) + # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s") # gguf_file = os.listdir("saved")[0] # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") # text = "There is a girl who likes adventure," # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) # shutil.rmtree("./saved", ignore_errors=True) - # - # # model_name = "Qwen/Qwen2.5-1.5B-Instruct" - # # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - # # autoround = AutoRound( - # # model, - # # self.tokenizer, - # # bits=3, - # # group_size=16, - # # sym=True, - # # iters=1, - # # nsamples=1, - # # data_type="int_sym_dq", - # # super_group_size=16, - # # super_bits=6, - # # ) - # quantized_model_path = "./saved" - # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s") - # # from auto_round.eval.evaluation import simple_evaluate_user_model - # # gguf_file = os.listdir("saved")[0] - # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") - # # result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="lambada_openai", eval_model_dtype="bf16") - # # self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.5) - # shutil.rmtree("./saved", ignore_errors=True) - # - # # - # # def test_q5_k(self): - # # model_name = "Qwen/Qwen2.5-1.5B-Instruct" - # # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - # # autoround = AutoRound( - # # model, - # # self.tokenizer, - # # bits=5, - # # group_size=32, - # # sym=False, - # # iters=1, - # # nsamples=1, - # # data_type="int_asym_dq", - # # super_group_size=8, - # # super_bits=6, - # # ) - # # quantized_model_path = "./saved" - # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s") - # # gguf_file = os.listdir("saved")[0] - # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") - # # text = "There is a girl who likes adventure," - # # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) - # # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) - # # shutil.rmtree("./saved", ignore_errors=True) - # - # # def test_q6_k(self): - # # model_name = "Qwen/Qwen2.5-1.5B-Instruct" - # # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - # # autoround = AutoRound( - # # model, - # # self.tokenizer, - # # bits=6, - # # group_size=16, - # # sym=True, - # # iters=1, - # # nsamples=1, - # # data_type="int_sym_dq", - # # super_group_size=16, - # # super_bits=8, - # # ) - # # quantized_model_path = "./saved" - # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k") - # # gguf_file = os.listdir("saved")[0] - # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") - # # text = "There is a girl who likes adventure," - # # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) - # # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) - # # shutil.rmtree("./saved", ignore_errors=True) - # - # def test_gguf_baseline(self): - # model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" + + # def test_q6_k(self): + # model_name = "Qwen/Qwen2.5-1.5B-Instruct" # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) # autoround = AutoRound( # model, # self.tokenizer, - # bits=3, + # bits=6, # group_size=16, # sym=True, - # iters=0, - # nsamples=8, - # seqlen=2, - # data_type="rtn_int_sym_dq", + # iters=1, + # nsamples=1, + # data_type="int_sym_dq", # super_group_size=16, - # super_bits=6, - # disable_opt_rtn=True, + # super_bits=8, # ) # quantized_model_path = "./saved" - # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="fake") - # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto") + # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k") + # gguf_file = os.listdir("saved")[0] + # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto") # text = "There is a girl who likes adventure," # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) # shutil.rmtree("./saved", ignore_errors=True) - # # - # # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - # # autoround = AutoRound( - # # model, - # # self.tokenizer, - # # bits=5, - # # group_size=32, - # # sym=True, - # # iters=0, - # # nsamples=8, - # # data_type="int_asym_dq", - # # super_group_size=8, - # # super_bits=6, - # # disable_opt_rtn=True, - # # ) - # # quantized_model_path = "./saved" - # # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q5_k_s,fake") - # # model = AutoModelForCausalLM.from_pretrained(quantized_model_path + "/fake", device_map="auto") - # # text = "There is a girl who likes adventure," - # # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) - # # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) - # # shutil.rmtree("./saved", ignore_errors=True) - # - # def test_q4_k_m(self): - # model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" - # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - # tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - # layer_config = { - # "lm_head": { - # "bits": 4, - # "group_size": 32, - # "sym": False, - # "data_type": "int_asym_dq", - # "super_bits": 6, - # "super_group_size": 8, - # }, - # "model.embed_tokens": {"bits": 6, "group_size": 32, "super_bits": 6, "super_group_size": 8}, - # "model.layers.12.mlp.gate_proj": {"bits": 3}, - # "model.layers.10.mlp.gate_proj": {"bits": 8}, - # } - # autoround = AutoRound( - # model, - # tokenizer, - # layer_config=layer_config, - # iters=0, - # seqlen=1, - # nsamples=8, - # dataset=self.llm_dataloader, - # disable_opt_rtn=True, - # ) - # quantized_model_path = "./saved" - # autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake") - # self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16) - # self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "int_sym_dq") - # self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "int_asym_dq") - # self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6) - # self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4) - # self.assertEqual(autoround.model.model.embed_tokens.bits, 6) - # self.assertEqual(autoround.model.model.embed_tokens.group_size, 16) - # self.assertEqual(autoround.model.model.layers[12].mlp.gate_proj.bits, 3) - # self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8) - # self.assertEqual(autoround.layer_config["model.layers.10.mlp.gate_proj"]["mostly"], "gguf:q8_0") - # shutil.rmtree("./saved", ignore_errors=True) - # - # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) - # autoround = AutoRound(model, tokenizer, iters=0, nsamples=1, seqlen=128, disable_opt_rtn=False) - # quantized_model_path = "./saved" - # autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake") - # shutil.rmtree("./saved", ignore_errors=True) - # - # def test_all_format(self): - # model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" - # python_path = sys.executable - # # for gguf_format in ["gguf:q4_0", "gguf:q4_1", "gguf:q4_k_m", "gguf:q6_k"]: - # for gguf_format in ["gguf:q4_k_m"]: - # res = os.system( - # f"cd ../.. && {python_path} -m auto_round --model {model_name} " - # f" --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format {gguf_format}" - # ) - # if res > 0 or res == -1: - # assert False, "cmd line test fail, please have a check" - # shutil.rmtree("../../tmp_autoround", ignore_errors=True) - # - # res = os.system( - # f"cd ../.. && {python_path} -m auto_round --model {model_name}" - # f" --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format fake,{gguf_format}" - # ) - # if res > 0 or res == -1: - # assert False, "cmd line test fail, please have a check" - # shutil.rmtree("../../tmp_autoround", ignore_errors=True) + + def test_gguf_baseline(self): + model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" + model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + autoround = AutoRound( + model, + self.tokenizer, + bits=3, + group_size=16, + sym=True, + iters=0, + nsamples=8, + seqlen=2, + data_type="rtn_int_sym_dq", + super_group_size=16, + super_bits=6, + disable_opt_rtn=True, + ) + quantized_model_path = "./saved" + autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="fake") + model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto") + text = "There is a girl who likes adventure," + inputs = self.tokenizer(text, return_tensors="pt").to(model.device) + print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) + shutil.rmtree("./saved", ignore_errors=True) + # + # model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + # autoround = AutoRound( + # model, + # self.tokenizer, + # bits=5, + # group_size=32, + # sym=True, + # iters=0, + # nsamples=8, + # data_type="int_asym_dq", + # super_group_size=8, + # super_bits=6, + # disable_opt_rtn=True, + # ) + # quantized_model_path = "./saved" + # autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q5_k_s,fake") + # model = AutoModelForCausalLM.from_pretrained(quantized_model_path + "/fake", device_map="auto") + # text = "There is a girl who likes adventure," + # inputs = self.tokenizer(text, return_tensors="pt").to(model.device) + # print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0])) + # shutil.rmtree("./saved", ignore_errors=True) + + def test_q4_k_m(self): + model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" + model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + layer_config = { + "lm_head": { + "bits": 4, + "group_size": 32, + "sym": False, + "data_type": "int_asym_dq", + "super_bits": 6, + "super_group_size": 8, + }, + "model.embed_tokens": {"bits": 6, "group_size": 32, "super_bits": 6, "super_group_size": 8}, + "model.layers.12.mlp.gate_proj": {"bits": 3}, + "model.layers.10.mlp.gate_proj": {"bits": 8}, + } + autoround = AutoRound( + model, + tokenizer, + layer_config=layer_config, + iters=0, + seqlen=1, + nsamples=8, + dataset=self.llm_dataloader, + disable_opt_rtn=True, + ) + quantized_model_path = "./saved" + autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake") + self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16) + self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "int_sym_dq") + self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "int_asym_dq") + self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6) + self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4) + self.assertEqual(autoround.model.model.embed_tokens.bits, 6) + self.assertEqual(autoround.model.model.embed_tokens.group_size, 16) + self.assertEqual(autoround.model.model.layers[12].mlp.gate_proj.bits, 3) + self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8) + self.assertEqual(autoround.layer_config["model.layers.10.mlp.gate_proj"]["mostly"], "gguf:q8_0") + shutil.rmtree("./saved", ignore_errors=True) + + model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True) + autoround = AutoRound(model, tokenizer, iters=0, nsamples=1, seqlen=128, disable_opt_rtn=False) + quantized_model_path = "./saved" + autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m,fake") + shutil.rmtree("./saved", ignore_errors=True) + + def test_all_format(self): + model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2.5-1.5B-Instruct" + python_path = sys.executable + # for gguf_format in ["gguf:q4_0", "gguf:q4_1", "gguf:q4_k_m", "gguf:q6_k"]: + for gguf_format in ["gguf:q4_k_m"]: + res = os.system( + f"cd ../.. && {python_path} -m auto_round --model {model_name} " + f" --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format {gguf_format}" + ) + if res > 0 or res == -1: + assert False, "cmd line test fail, please have a check" + shutil.rmtree("../../tmp_autoround", ignore_errors=True) + + res = os.system( + f"cd ../.. && {python_path} -m auto_round --model {model_name}" + f" --bs 16 --iters 0 --nsamples 1 --seqlen 16 --format fake,{gguf_format}" + ) + if res > 0 or res == -1: + assert False, "cmd line test fail, please have a check" + shutil.rmtree("../../tmp_autoround", ignore_errors=True) def test_vlm_gguf(self): - model_name = "/models/Qwen2-VL-2B-Instruct" + model_name = "/tf_dataset/auto_round/models/Qwen/Qwen2-VL-2B-Instruct" from auto_round import AutoRoundMLLM from auto_round.utils import mllm_load_model From 26822471304592b17c96f8b67b0a206f9d3050ea Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Wed, 15 Oct 2025 15:26:25 +0800 Subject: [PATCH 69/88] update --- auto_round/auto_schemes/gen_auto_scheme.py | 47 +++++++++++++++++++++- auto_round/utils.py | 3 +- test/test_cuda/test_auto_scheme.py | 22 +++++----- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index 12e956eba..cd735e60c 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -19,8 +19,10 @@ from auto_round import AutoScheme from auto_round.auto_schemes import AUTO_SCHEMES_METHODS from auto_round.auto_schemes.utils import compute_avg_bits_for_scheme +from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG from auto_round.logger import logger - +from auto_round.utils import get_layer_features, get_module, _gguf_type_fallback +import math class GenScheme: """Generate and validate quantization schemes for model layers.""" @@ -68,7 +70,7 @@ def _check_configs(self) -> None: f"Target avg_bits={target:.3f} is outside the valid range " f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}]." ) - def get_layer_config(self): + def get_layer_config(self) -> dict[str, dict]: method_name = self.auto_scheme.method method_func = AUTO_SCHEMES_METHODS[method_name] layer_config = method_func( @@ -80,6 +82,47 @@ def get_layer_config(self): self.tokenizer, device_map=self.device_map, ) + layer_config = self.fallback_gguf_layer_config(layer_config) + return layer_config + + def fallback_gguf_layer_config(self, layer_config: dict[str, dict]) -> dict[str, dict]: + """ + Apply fallback configurations for GGUF quantized layers when the current + layer configuration is incompatible with input feature alignment. + + Args: + layer_config (dict[str, dict]): Mapping from layer name to its quantization scheme. + + Returns: + dict[str, dict]: Updated layer configuration with applied fallbacks if necessary. + """ + for name, scheme in layer_config.items(): # TODO: add unit test (wenhua), the code is a little tricky + if scheme.get("super_bits") is None: + continue # Skip non-GGUF k-quant layers + + layer = get_module(self.model, name) + input_features = get_layer_features(layer) + + # Determine fallback quantization type + if input_features % 256 != 0 and input_features % 32 != 0: + new_type = "gguf:bf16" + else: + bits = scheme["bits"] + new_type = f"gguf:q{bits}_0" + + if new_type not in GGUF_INNER_CONFIG: + new_type = f"gguf:q{bits}_1" + + if new_type not in GGUF_INNER_CONFIG: + current_type = f"gguf:q{bits}_k" + new_type = _gguf_type_fallback(current_type) + + # Apply fallback configuration + target_config = GGUF_INNER_CONFIG[new_type] + scheme.update(target_config) + + logger.warning(f"Fallback applied: {name} → {new_type}") + return layer_config def compute_avg_bit_range(self) -> tuple[float, float]: diff --git a/auto_round/utils.py b/auto_round/utils.py index 37460f119..0309fd9ef 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -1879,7 +1879,8 @@ def _search_gguf_type(gguf_type): return None -def _gguf_type_fallback(gguf_type): +def _gguf_type_fallback(gguf_type:str)->str: + gguf_type = gguf_type.lower() if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"): gguf_type = "gguf:q5_0" elif gguf_type == "gguf:q5_k": diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index e11ce1a01..7dc9f6f65 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -31,7 +31,7 @@ def test_gguf_export(self): scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M"), ignore_scale_zp_bits=True) ar = AutoRound(model=model_name, scheme=scheme, iters=0) ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s") - shutil.rmtree("./saved", ignore_errors=True) + shutil.rmtree(self.save_dir, ignore_errors=True) def test_gguf(self): model_name = "/models/Qwen3-8B" @@ -82,18 +82,18 @@ def test_shared_layers(self): self.assertEqual(len(bits), 1) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - + # @multi_card def test_multi_card(self): model_name = "/models/Qwen3-8B" target_bits = 5.254 - for device_map in ["auto", "0,1", "0", None]: - scheme = AutoScheme(avg_bits=target_bits, options=("NVFP4")) - ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, device_map=device_map) - model, layer_config = ar.quantize() - avg_bits, _ = compute_avg_bits_for_model(model) - print(avg_bits) - assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + # for device_map in ["auto", "0,1", "0", None]: + scheme = AutoScheme(avg_bits=target_bits, options=("NVFP4")) + ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) + model, layer_config = ar.quantize() + avg_bits, _ = compute_avg_bits_for_model(model) + print(avg_bits) + assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 @multi_card def test_dict_device_map(self): # TODO rtn mode has bug @@ -118,7 +118,7 @@ def test_min_target_bits(self): avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 - + # def test_max_target_bits(self): model_name = "/models/opt-125m" target_bits = 8.211 @@ -174,7 +174,7 @@ def test_lm_head_and_mix_dtype(self): def test_auto_scheme_export(self): model_name = "/models/opt-125m" - scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16")) + scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "W8A16", "BF16")) ar = AutoRound(model=model_name, scheme=scheme) ar.quantize_and_save(self.save_dir) model_args = f"pretrained={self.save_dir}" From 30fee225246a56b43b00834e9c834f8b61c366b6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 Oct 2025 07:27:10 +0000 Subject: [PATCH 70/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_schemes/gen_auto_scheme.py | 5 +++-- auto_round/utils.py | 2 +- test/test_cuda/test_auto_scheme.py | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_schemes/gen_auto_scheme.py index cd735e60c..106e3d46d 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_schemes/gen_auto_scheme.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import math from dataclasses import asdict from typing import Iterable, Union @@ -21,8 +22,8 @@ from auto_round.auto_schemes.utils import compute_avg_bits_for_scheme from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG from auto_round.logger import logger -from auto_round.utils import get_layer_features, get_module, _gguf_type_fallback -import math +from auto_round.utils import _gguf_type_fallback, get_layer_features, get_module + class GenScheme: """Generate and validate quantization schemes for model layers.""" diff --git a/auto_round/utils.py b/auto_round/utils.py index 0309fd9ef..49354bd64 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -1879,7 +1879,7 @@ def _search_gguf_type(gguf_type): return None -def _gguf_type_fallback(gguf_type:str)->str: +def _gguf_type_fallback(gguf_type: str) -> str: gguf_type = gguf_type.lower() if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"): gguf_type = "gguf:q5_0" diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 7dc9f6f65..4b5155e6f 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -82,6 +82,7 @@ def test_shared_layers(self): self.assertEqual(len(bits), 1) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + # @multi_card def test_multi_card(self): @@ -118,6 +119,7 @@ def test_min_target_bits(self): avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 + # def test_max_target_bits(self): model_name = "/models/opt-125m" From 8ce5b1eec75adfaf30d9bb8781eb9b88e3140376 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Wed, 15 Oct 2025 16:11:57 +0800 Subject: [PATCH 71/88] fix merge issue --- auto_round/__main__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 110107acd..0d4e12985 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import argparse -import logging import os -import re import sys from auto_round.compressors import BaseCompressor @@ -24,7 +22,6 @@ clear_memory, get_device_and_parallelism, get_model_dtype, - set_cuda_visible_devices, ) RECIPES = { @@ -82,7 +79,7 @@ def __init__(self, *args, **kwargs): self.add_argument("--act_bits", default=None, type=int, help="activation bits") self.add_argument("--act_group_size", default=None, type=int, help="activation group size") self.add_argument( - "--super_group_size", default=None, type=int, help="the number of super group size when use double quant." + "--super_group_size", default=None, type=int, help="the number of super group size when use double quant.") basic.add_argument( "--iters", "--iter", @@ -469,8 +466,6 @@ def tune(args): if "marlin" in args.format and args.asym is True: raise RuntimeError("marlin backend only supports sym quantization, please remove --asym") - # Must set this before import torch - # set_cuda_visible_devices(args.device_map) device_str, use_auto_mapping = get_device_and_parallelism(args.device_map) import torch From 8e163257bee9cfad30ce86e7c2172bbb9da9da67 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 Oct 2025 08:12:48 +0000 Subject: [PATCH 72/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 0d4e12985..3468cedb1 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -79,7 +79,8 @@ def __init__(self, *args, **kwargs): self.add_argument("--act_bits", default=None, type=int, help="activation bits") self.add_argument("--act_group_size", default=None, type=int, help="activation group size") self.add_argument( - "--super_group_size", default=None, type=int, help="the number of super group size when use double quant.") + "--super_group_size", default=None, type=int, help="the number of super group size when use double quant." + ) basic.add_argument( "--iters", "--iter", From decdcce5ebe3ee621822fb60ce3fe8c333541f43 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Wed, 15 Oct 2025 16:47:13 +0800 Subject: [PATCH 73/88] fix merge issue --- README.md | 9 +++++-- auto_round/__main__.py | 8 ------ auto_round/compressors/base.py | 47 ++++++++++------------------------ auto_round/utils.py | 2 +- 4 files changed, 22 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index bbebb901c..f00e8d0b1 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri). For usage instructions, ## 🆕 What's New +[2025/10] AutoRound now includes experimental support for a fast algorithm developed by the AutoRound team for generating mixed-bit and data-type schemes. See the documentation + for accuracy results and this Doc for usage [2025/09] AutoRound now includes experimental support for the mxfp4 and nvfp4 dtypes. For accuracy results, see the [documentation](./docs/mxnv_acc.md) . We currently recommend exporting to the LLM-Compressor format. @@ -38,7 +40,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri). For usage instructions, all bits other than 3 bits. Example models: [Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-mixed-AutoRound) and [Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-AutoRound). **A more advanced algorithm** tailored for specific configurations may be available in - v0.7.1. + v0.8.1. [2025/05] AutoRound has been integrated into **vLLM**. You can now run models in the AutoRound format directly with vLLM versions later than v0.85.post1. @@ -65,6 +67,9 @@ Support **AutoRound, AutoAWQ, AutoGPTQ, and GGUF** for maximum compatibility. De ✅ **Affordable Quantization Cost** Quantize 7B models in about 10 minutes on a single GPU. Details are shown in [quantization costs](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#quantization-costs) +✅ **Fast mixed-bit/data-type scheme generation** +Achieve automatic configuration in minutes, with only ~2× VRAM overhead. + ✅ **10+ VLMs Support** Out-of-the-box quantization for 10+ vision-language models [example models](https://huggingface.co/collections/OPEA/vlms-autoround-675bc712fdd6a55ebaf11bfa), [support matrix](https://github.com/intel/auto-round/tree/main/auto_round/mllm#support-matrix) @@ -111,7 +116,7 @@ pip install auto-round-lib ## Model Quantization (CPU/Intel GPU/Gaudi/CUDA) ### CLI Usage -Please change to `auto-round-mllm` for visual-language models (VLMs) quantization. The full list of supported arguments is provided by calling `auto-round -h` on the terminal. +The full list of supported arguments is provided by calling `auto-round -h` on the terminal. ```bash auto-round \ diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 0d4e12985..2f4d7d71b 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -72,14 +72,6 @@ def __init__(self, *args, **kwargs): action="store_true", help="for auto scheme whether ignore scale zp bits calculation ", ) - self.add_argument("--bits", default=None, type=int, help="number of weight bits") - self.add_argument("--group_size", default=None, type=int, help="group size") - self.add_argument("--asym", action="store_true", help="whether to use asym quantization") - self.add_argument("--data_type", "--dtype", default=None, help="data type for tuning, 'int', 'mx_fp' and etc") - self.add_argument("--act_bits", default=None, type=int, help="activation bits") - self.add_argument("--act_group_size", default=None, type=int, help="activation group size") - self.add_argument( - "--super_group_size", default=None, type=int, help="the number of super group size when use double quant.") basic.add_argument( "--iters", "--iter", diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index d5110348c..bc5f09e4d 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -1676,40 +1676,21 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: # TODO check scale_dtype if not self.is_auto_scheme: - self.layer_config, self.has_qlayer_outside_block = set_layer_config( - self.model, - self.layer_config, - self.scheme, - self.scale_dtype, - self.supported_types, - self.inner_supported_types, - self.quant_block_list, - self.fp_layers, - self.quant_lm_head, - enable_gguf_official_mixed=True, - is_mllm=self.mllm, - ) + enable_gguf_official_mixed = True else: - # for n, scheme in self.layer_config.items(): - # module = get_module(self.model, n) - # if not isinstance(scheme, dict): - # raise ValueError("scheme return by scheme should be dict") - # for key, item in scheme.items(): - # setattr(module, key, item) - # # set_extra scale_dtype - # module.scale_dtype = self.scale_dtype - self.layer_config, self.has_qlayer_outside_block = set_layer_config( - self.model, - self.layer_config, - self.scheme, - self.scale_dtype, - self.supported_types, - self.inner_supported_types, - self.quant_block_list, - self.fp_layers, - self.quant_lm_head, - enable_gguf_official_mixed=False, - is_mllm=self.mllm, + enable_gguf_official_mixed = False + self.layer_config, self.has_qlayer_outside_block = set_layer_config( + self.model, + self.layer_config, + self.scheme, + self.scale_dtype, + self.supported_types, + self.inner_supported_types, + self.quant_block_list, + self.fp_layers, + self.quant_lm_head, + enable_gguf_official_mixed=enable_gguf_official_mixed, + is_mllm=self.mllm, ) if not hasattr(self, "formats"): diff --git a/auto_round/utils.py b/auto_round/utils.py index f212acd28..380383f0e 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2855,7 +2855,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str elif isinstance(item, QuantizationScheme): config = asdict(item) elif isinstance(item, dict): - invalid = set(item) - set(scheme_keys) + invalid = set(item) - set(scheme_keys+("fixed_by_user", "scale_dtype")) if invalid: raise ValueError( f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}" From f9e80abf428e278160c3f4d6c033d753182c44e0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 Oct 2025 08:51:47 +0000 Subject: [PATCH 74/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/compressors/base.py | 2 +- auto_round/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index bc5f09e4d..d8238a0de 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -1691,7 +1691,7 @@ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]: self.quant_lm_head, enable_gguf_official_mixed=enable_gguf_official_mixed, is_mllm=self.mllm, - ) + ) if not hasattr(self, "formats"): logger.warning("this API is deprecated, please use `quantize_and_save` instead") diff --git a/auto_round/utils.py b/auto_round/utils.py index 380383f0e..bc8b02dd5 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -2855,7 +2855,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str elif isinstance(item, QuantizationScheme): config = asdict(item) elif isinstance(item, dict): - invalid = set(item) - set(scheme_keys+("fixed_by_user", "scale_dtype")) + invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype")) if invalid: raise ValueError( f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}" From efc69de1fd7c1e7754bcbf1c7f6d495b05d08920 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Wed, 15 Oct 2025 17:21:28 +0800 Subject: [PATCH 75/88] update --- auto_round/__main__.py | 6 --- .../{auto_schemes => auto_scheme}/__init__.py | 8 +-- .../gen_auto_scheme.py | 6 +-- .../{auto_schemes => auto_scheme}/utils.py | 0 auto_round/compressors/base.py | 2 +- docs/alg_202508.md | 12 ++--- docs/auto_scheme_acc.md | 51 +++++++++++++++++++ test/test_cuda/test_auto_scheme.py | 4 +- 8 files changed, 67 insertions(+), 22 deletions(-) rename auto_round/{auto_schemes => auto_scheme}/__init__.py (84%) rename auto_round/{auto_schemes => auto_scheme}/gen_auto_scheme.py (96%) rename auto_round/{auto_schemes => auto_scheme}/utils.py (100%) create mode 100644 docs/auto_scheme_acc.md diff --git a/auto_round/__main__.py b/auto_round/__main__.py index 2f4d7d71b..a9cc01985 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -554,12 +554,6 @@ def tune(args): extra_config.diffusion_config = diffusion_config layer_config = {} - # from auto_round.auto_schemes.haha import get_mixed_config_layer_config - # layer_config = {} - # best_path = get_mixed_config_layer_config(model_name, target_bits=3) - # for item in best_path: - # layer_config[item[0]] = {} - # layer_config[item[0]]["bits"] = item[1] if args.avg_bits is not None: if args.options is None: diff --git a/auto_round/auto_schemes/__init__.py b/auto_round/auto_scheme/__init__.py similarity index 84% rename from auto_round/auto_schemes/__init__.py rename to auto_round/auto_scheme/__init__.py index e0e5ccb66..d6db762ef 100644 --- a/auto_round/auto_schemes/__init__.py +++ b/auto_round/auto_scheme/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -AUTO_SCHEMES_METHODS = {} +AUTO_SCHEME_METHODS = {} def register_scheme_methods(names): @@ -30,13 +30,13 @@ def register_scheme_methods(names): def register(alg): if isinstance(names, (tuple, list)): for name in names: - AUTO_SCHEMES_METHODS[name] = alg + AUTO_SCHEME_METHODS[name] = alg else: - AUTO_SCHEMES_METHODS[names] = alg + AUTO_SCHEME_METHODS[names] = alg return alg return register -import auto_round.auto_schemes.haha # pylint: disable=E0611,E0401 +import auto_round.auto_scheme.haha # pylint: disable=E0611,E0401 diff --git a/auto_round/auto_schemes/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py similarity index 96% rename from auto_round/auto_schemes/gen_auto_scheme.py rename to auto_round/auto_scheme/gen_auto_scheme.py index 106e3d46d..fb430e3fb 100644 --- a/auto_round/auto_schemes/gen_auto_scheme.py +++ b/auto_round/auto_scheme/gen_auto_scheme.py @@ -18,8 +18,8 @@ import torch from auto_round import AutoScheme -from auto_round.auto_schemes import AUTO_SCHEMES_METHODS -from auto_round.auto_schemes.utils import compute_avg_bits_for_scheme +from auto_round.auto_scheme import AUTO_SCHEME_METHODS +from auto_round.auto_scheme.utils import compute_avg_bits_for_scheme from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG from auto_round.logger import logger from auto_round.utils import _gguf_type_fallback, get_layer_features, get_module @@ -73,7 +73,7 @@ def _check_configs(self) -> None: def get_layer_config(self) -> dict[str, dict]: method_name = self.auto_scheme.method - method_func = AUTO_SCHEMES_METHODS[method_name] + method_func = AUTO_SCHEME_METHODS[method_name] layer_config = method_func( self.auto_scheme, self.model, diff --git a/auto_round/auto_schemes/utils.py b/auto_round/auto_scheme/utils.py similarity index 100% rename from auto_round/auto_schemes/utils.py rename to auto_round/auto_scheme/utils.py diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index d8238a0de..0f3c4c57b 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -376,7 +376,7 @@ def __init__( } # mainly using quant_layers and fixed by users - from auto_round.auto_schemes.gen_auto_scheme import GenScheme + from auto_round.auto_scheme.gen_auto_scheme import GenScheme gen_scheme = GenScheme( scheme, diff --git a/docs/alg_202508.md b/docs/alg_202508.md index 3fb74ace8..086cd5cf6 100644 --- a/docs/alg_202508.md +++ b/docs/alg_202508.md @@ -3,12 +3,12 @@ We use **lm-eval** for evaluation. For LLaMA, we enabled `add_bos_token` and in [modeling_llama.py](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L52C1-L52C40) to stabilize accuracy during evaluation. All other settings follow the default configurations of AutoRound and lm-eval. -| Qwen3-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande | -|----------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------| -| origin | 0.4373 | 0.4019 | 0.4437 | 0.4215 | 0.4826 | 0.5474 | 0.263 | 0.3072 | 0.6314 | -| new | 0.4787 | 0.4275 | 0.4516 | 0.5944 | 0.5181 | 0.5773 | 0.2807 | 0.3305 | 0.6496 | +| Qwen3-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande | +|-------------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------| +| AutoRound | 0.4373 | 0.4019 | 0.4437 | 0.4215 | 0.4826 | 0.5474 | 0.263 | 0.3072 | 0.6314 | +| AutoRound+alg_ext | 0.4787 | 0.4275 | 0.4516 | 0.5944 | 0.5181 | 0.5773 | 0.2807 | 0.3305 | 0.6496 | | Llama3.1-8B W2G64 | Avg. | arc_challenge | hellaswag | gsm8k | lambada_openai | mmlu | mmlupro | truthfulqa_mc1 | winogrande | |-------------------|--------|---------------|-----------|--------|----------------|--------|---------|----------------|------------| -| origin | 0.382 | 0.3635 | 0.4562 | 0.1622 | 0.5069 | 0.4411 | 0.1661 | 0.3207 | 0.6393 | -| new | 0.4166 | 0.3712 | 0.4729 | 0.2039 | 0.5946 | 0.4981 | 0.2163 | 0.3011 | 0.6748 | \ No newline at end of file +| AutoRound | 0.382 | 0.3635 | 0.4562 | 0.1622 | 0.5069 | 0.4411 | 0.1661 | 0.3207 | 0.6393 | +| AutoRound+alg_ext | 0.4166 | 0.3712 | 0.4729 | 0.2039 | 0.5946 | 0.4981 | 0.2163 | 0.3011 | 0.6748 | \ No newline at end of file diff --git a/docs/auto_scheme_acc.md b/docs/auto_scheme_acc.md new file mode 100644 index 000000000..3a284d73a --- /dev/null +++ b/docs/auto_scheme_acc.md @@ -0,0 +1,51 @@ +We use **lm-eval** for evaluation. For LLaMA, we enabled `add_bos_token` and +`removed @use_kernel_forward_from_hub("RMSNorm")` +in [modeling_llama.py](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L52C1-L52C40) +to stabilize accuracy during evaluation. All other settings follow the default configurations of AutoRound and lm-eval. + +We ignore the scale and zp bits in the tables below. The accuracy may change a little as we modified a little of the +implementation. We will rerun all the experiments. + +For mxfp experiment, we use fake model while for weight only model we use real model. **No tuning is applied unless explict stated. +** + +*Average accuracy across `lambada_openai`, `hellaswag`, `piqa`, `winogrande`, and `mmlu`.* + +### Table 1 MXFP4/8 mixed accuracy. + +| Average bits | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B | Qwen3-32B | +|------------------|----------------|----------------|----------------|----------------| +| **BF16** | 0.7076 (100%) | 0.7075 (100%) | 0.6764 (100%) | 0.7321 (100%) | +| **Pure 4-bit** | 0.6626 (93.6%) | 0.6550 (92.6%) | 0.6316 (93.4%) | 0.6901 (94.3%) | +| **Ours 4.5-bit** | 0.6808 (96.2%) | 0.6776 (95.8%) | 0.6550 (96.8%) | 0.7176 (98.0%) | +| **Ours 5-bit** | 0.6857 (96.9%) | 0.6823 (96.4%) | 0.6594 (97.5%) | 0.7201 (98.3%) | +| **Ours 6-bit** | 0.6975 (98.6%) | 0.6970 (98.5%) | 0.6716 (99.3%) | 0.7303 (99.8%) | + +We compare the proposed method against naive layer-wise bit allocation strategies, such as assigning higher +precision to the network’s head((near lm-head) or tailad(close to embedding)) layers, to demonstrate its relative +performance advantages. + +### Table 2 Comparison with other recipes at an average of 5 bits of mxfp datatype + +| Avg. bits = 5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B | +|-----------------------|-------------------:|-------------------:|-------------------:| +| **Tail layers 8-bit** | 0.6671 (94.3%) | 0.6616 (93.5%) | 0.6410 (94.8%) | +| **Head layers 8-bit** | 0.6657 (94.1%) | 0.6686 (94.5%) | 0.6356 (94.0%) | +| **Ours** | **0.6857 (96.9%)** | **0.6823 (96.4%)** | **0.6594 (97.5%)** | + +### Table 3 Comparison with other recipes at an average of 4.5 bits of mxfp datatype + +| Avg. bits = 4.5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B | +|-----------------------|-------------------:|-------------------:|-------------------:| +| **Tail layers 8-bit** | 0.6614 (93.5%) | 0.6535 (92.4%) | 0.6373 (94.2%) | +| **Head layers 8-bit** | 0.6568 (92.8%) | 0.6642 (93.9%) | 0.6305 (93.2%) | +| **Ours** | **0.6808 (96.2%)** | **0.6776 (95.5%)** | **0.6550 (95.8%)** | + + +### Table4 Comparison with other recipes at an average of 3 bits of W2G128 and W4G128 + +| Avg. bits = 4.5 | Llama3.1-8B-I | Qwen2.5-7B-I | Qwen3-8B | +|-----------------------|--------------:|-------------:|---------:| +| **Tail layers 4-bit** | 0.6058 | 0.3798 | 0.4536 | +| **Head layers 4-bit** | 0.3198 | 0.3270 | 0.3196 | +| **Ours** | 0.6148 | 0.4058 | 0.4862 | \ No newline at end of file diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 4b5155e6f..69fc21217 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -9,7 +9,7 @@ sys.path.insert(0, "../..") from auto_round import AutoRound, AutoRoundConfig, AutoScheme -from auto_round.auto_schemes.utils import compute_avg_bits_for_model +from auto_round.auto_scheme.utils import compute_avg_bits_for_model from auto_round.eval.evaluation import simple_evaluate from auto_round.utils import get_module @@ -54,7 +54,7 @@ def test_shared_layers(self): ("model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"), ("fc1", "fc2"), ] - from auto_round.auto_schemes.utils import parse_shared_layers + from auto_round.auto_scheme.utils import parse_shared_layers res = parse_shared_layers(model, shared_layers) self.assertEqual(len(res), 24) From 4d1f8de28d92a126fb33d87a99e0cab95d0ac3af Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 Oct 2025 09:22:51 +0000 Subject: [PATCH 76/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/auto_scheme_acc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auto_scheme_acc.md b/docs/auto_scheme_acc.md index 3a284d73a..b058838f6 100644 --- a/docs/auto_scheme_acc.md +++ b/docs/auto_scheme_acc.md @@ -6,7 +6,7 @@ to stabilize accuracy during evaluation. All other settings follow the default c We ignore the scale and zp bits in the tables below. The accuracy may change a little as we modified a little of the implementation. We will rerun all the experiments. -For mxfp experiment, we use fake model while for weight only model we use real model. **No tuning is applied unless explict stated. +For mxfp experiment, we use fake model while for weight only model we use real model. **No tuning is applied unless explicit stated. ** *Average accuracy across `lambada_openai`, `hellaswag`, `piqa`, `winogrande`, and `mmlu`.* From f1ed097cc78c250191a8077c49e1db972eca734d Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 16 Oct 2025 10:54:33 +0800 Subject: [PATCH 77/88] update --- README.md | 8 ++-- auto_round/auto_scheme/gen_auto_scheme.py | 11 +++-- auto_round/schemes.py | 24 +++++++++- auto_round/utils.py | 12 ++--- docs/step_by_step.md | 58 ++++++++++++++++++++++- test/test_cuda/test_auto_scheme.py | 5 +- 6 files changed, 95 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index f00e8d0b1..48e8f8f6f 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri). For usage instructions, ## 🆕 What's New -[2025/10] AutoRound now includes experimental support for a fast algorithm developed by the AutoRound team for generating mixed-bit and data-type schemes. See the documentation - for accuracy results and this Doc for usage +[2025/10] AutoRound now includes experimental support for a fast algorithm developed by the AutoRound team to generate mixed-bit and mixed-datatype schemes. +Refer to the documentation for accuracy [results](./docs/auto_scheme_acc.md) and [this guide](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#autoscheme) for usage instructions. [2025/09] AutoRound now includes experimental support for the mxfp4 and nvfp4 dtypes. For accuracy results, see the [documentation](./docs/mxnv_acc.md) . We currently recommend exporting to the LLM-Compressor format. @@ -67,8 +67,8 @@ Support **AutoRound, AutoAWQ, AutoGPTQ, and GGUF** for maximum compatibility. De ✅ **Affordable Quantization Cost** Quantize 7B models in about 10 minutes on a single GPU. Details are shown in [quantization costs](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#quantization-costs) -✅ **Fast mixed-bit/data-type scheme generation** -Achieve automatic configuration in minutes, with only ~2× VRAM overhead. +✅ **Fast mixed bits/data-types scheme generation** +Automatically configure in minutes, with only about 2.5× the model’s BF16 VRAM size as overhead. ✅ **10+ VLMs Support** Out-of-the-box quantization for 10+ vision-language models [example models](https://huggingface.co/collections/OPEA/vlms-autoround-675bc712fdd6a55ebaf11bfa), [support matrix](https://github.com/intel/auto-round/tree/main/auto_round/mllm#support-matrix) diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py index fb430e3fb..fbf83a866 100644 --- a/auto_round/auto_scheme/gen_auto_scheme.py +++ b/auto_round/auto_scheme/gen_auto_scheme.py @@ -102,18 +102,19 @@ def fallback_gguf_layer_config(self, layer_config: dict[str, dict]) -> dict[str, continue # Skip non-GGUF k-quant layers layer = get_module(self.model, name) - input_features = get_layer_features(layer) + input_features, out_features = get_layer_features(layer) + if input_features is None: + continue # Determine fallback quantization type if input_features % 256 != 0 and input_features % 32 != 0: new_type = "gguf:bf16" else: bits = scheme["bits"] - new_type = f"gguf:q{bits}_0" - + prefix_idx = 0 if scheme["sym"] else 1 + new_type = f"gguf:q{bits}_" + f"{prefix_idx}" if new_type not in GGUF_INNER_CONFIG: - new_type = f"gguf:q{bits}_1" - + new_type = f"gguf:q{bits}_" + f"{1-prefix_idx}" if new_type not in GGUF_INNER_CONFIG: current_type = f"gguf:q{bits}_k" new_type = _gguf_type_fallback(current_type) diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 789cbe1cb..5129b4757 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -122,6 +122,27 @@ def is_preset_scheme(name: str) -> bool: } ) +W2A16G64 = QuantizationScheme.from_dict( + { + "bits": 2, + "sym": True, + "group_size": 64, + "data_type": "int", + "act_bits": 16, + } +) + + +W2A16G32 = QuantizationScheme.from_dict( + { + "bits": 2, + "sym": True, + "group_size": 32, + "data_type": "int", + "act_bits": 16, + } +) + W3A16 = QuantizationScheme.from_dict( { "bits": 3, @@ -221,7 +242,6 @@ def is_preset_scheme(name: str) -> bool: } ) - PRESET_SCHEMES = { "W4A16": W4A16, "W2A16": W2A16, @@ -231,6 +251,8 @@ def is_preset_scheme(name: str) -> bool: "MXFP8": MXFP8, "NVFP4": NVFP4, "FPW8A16": FPW8A16, + "W2A16G64": W2A16G64, + "W2A16G32": W2A16G32, "FP8_STATIC": FP8_STATIC, "BF16": BF16, } diff --git a/auto_round/utils.py b/auto_round/utils.py index bc8b02dd5..fc4a2f054 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -1997,14 +1997,10 @@ def _set_config(config, target_config): if config.get("super_group_size", None) is not None: new_type = new_type[:bits_index] + str(config["bits"]) + "_k" if config.get("super_group_size", None) is None or new_type not in GGUF_INNER_CONFIG: - if config.get("sym", True): - new_type = new_type[:bits_index] + str(config["bits"]) + "_0" - if new_type not in GGUF_INNER_CONFIG: - new_type = new_type[:bits_index] + str(config["bits"]) + "_1" - if not config.get("sym", True): - new_type = new_type[:bits_index] + str(config["bits"]) + "_1" - if new_type not in GGUF_INNER_CONFIG: - new_type = new_type[:bits_index] + str(config["bits"]) + "_0" + prefix_idx = 0 if config.get("sym", True) else 1 + new_type = new_type[:bits_index] + str(config["bits"]) + f"{prefix_idx}" + if new_type not in GGUF_INNER_CONFIG: + new_type = new_type[:bits_index] + str(config["bits"]) + f"{1-prefix_idx}" if new_type not in GGUF_INNER_CONFIG: raise ValueError( f"the setting in layer_config {layer_name} " diff --git a/docs/step_by_step.md b/docs/step_by_step.md index 21c41862c..69d14ec1b 100644 --- a/docs/step_by_step.md +++ b/docs/step_by_step.md @@ -19,6 +19,10 @@ This document presents step-by-step instructions for auto-round llm quantization - [AutoRoundBest recipe](#autoroundbest-recipe) - [AutoRoundLight recipe](#autoroundlight-recipe) - [Recipe recommendation](#recipe-recommendation) + + [AutoScheme](#autoscheme) + - [CLI Usage](#cli-usage) + - [API Usage](#api-usage-1) + - [Hyperparameters in AutoScheme](#hyperparameters-in-autoscheme) + [RTN mode](#rtn-mode) + [GGUF format](#gguf-format) + [Quantization Costs](#quantization-costs) @@ -116,7 +120,7 @@ AutoRound supports several Schemes: - **W3A16**(bits:3,group_size:128,sym:True,act_bits:16) - **W2A16**(bits:2,group_size:128,sym:True,act_bits:16) - **Mixed bits Weight only** -- **NVFP4**(data_type:nvfp4,act_data_type:nvfp4,static_global_scale,group_size 16) +- **NVFP4**(Experimental feature, recommend exporting to llm-compressor format. data_type:nvfp4,act_data_type:nvfp4,static_global_scale,group_size 16) - **MXFP4**(**Research feature,no real kernel**, data_type:mxfp4,act_data_type:mxfp4,rceil,group_size 32) - **FPW8A16**(**Research feature,no real kernel**, data_type:fp8,act_data_type 16:,group_size 0->per tensor ) - **FP8_STATIC**(**Research feature,no real kernel**, data_type:fp8,act_data_type:fp8,group_size -1 ->per channel, act_group_size=0->per tensor) @@ -271,6 +275,58 @@ W2G64 Average Accuracy of 13 tasks and Time Cost Results(Testing was conducted o +### AutoScheme + +AutoScheme provide automatically algorithm to provide mixed bits/data_type quantization recipes. For some accuracy result, please refer this doc [here](./auto_scheme_acc.md) + +**Please note that mixed data types are supported during tuning, but cannot be exported to real models at this time..** +### CLI Usage +use `iters=200`for tuning. +~~~bash +auto_round \ + --model_name $model_name \ + --avg_bits 6 \ + --options "mxfp4,mxfp8" \ + --ignore_scale_zp_bits \ + --iters 0 \ + --format fake +~~~ + +### API Usage +~~~ +avg_bits= 3.0 +scheme = AutoScheme(avg_bits=avg_bits, options=("W2A16G64“, "W4A16","W8A16")) +ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) +ar.quantize_and_save() +~~~ + +### Hyperparameters in AutoScheme +`avg_bits(float)`: Target average bits for the whole model, only to be quantized layer will be counted in the average bits calculation. + +`options(Union[str, list[Union[QuantizationScheme, str]])`: the options of quantization schemes to choose from. It could be a string like "W4A16", or a list of strings or QuantizationScheme objects. + +`ignore_scale_zp_bits(bool)`: Whether to ignore the bits of scale and zero point in average bits calculation. Default is False. + +`shared_layers (Optional[Iterable[Iterable[str]]])` only supported in API now + +In some serving frameworks, certain layers (e.g., QKV or MoE) are fused to accelerate inference. These fused layers may require the same data type and bit configuration. The shared_layers option simplifies this setup by supporting both regex and full-name matching. **Note that regex matching is applied in a block-wise manner.** + + +```python +from auto_round import AutoRound, AutoScheme +shared_layers = [ + ["*.self_attn.k_proj", "v_proj", "q_proj", "out_proj"], + ("model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"), + ("fc1", "fc2"), + ] +target_bits = 5.0 +model_name = "facebook/opt-125m" +scheme = AutoScheme(avg_bits=target_bits, options=("W4A16", "MXFP8"), shared_layers=shared_layers) +ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) +model, layer_config = ar.quantize() +``` + + ### RTN mode AutoRound also supports RTN (Round-To-Nearest) mode for fast, calibration-free baseline quantization. try setting `iters=0` and use `group_size=32` for better results. diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 69fc21217..77c6df9c9 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -3,11 +3,8 @@ import shutil import sys import unittest - -from auto_round.testing_utils import multi_card - sys.path.insert(0, "../..") - +from auto_round.testing_utils import multi_card from auto_round import AutoRound, AutoRoundConfig, AutoScheme from auto_round.auto_scheme.utils import compute_avg_bits_for_model from auto_round.eval.evaluation import simple_evaluate From bae03541398574031a6f8ee9365a6ce90cd15957 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 02:55:29 +0000 Subject: [PATCH 78/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/step_by_step.md | 9 +++++---- test/test_cuda/test_auto_scheme.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/step_by_step.md b/docs/step_by_step.md index 69d14ec1b..a92811d3f 100644 --- a/docs/step_by_step.md +++ b/docs/step_by_step.md @@ -314,11 +314,12 @@ In some serving frameworks, certain layers (e.g., QKV or MoE) are fused to accel ```python from auto_round import AutoRound, AutoScheme + shared_layers = [ - ["*.self_attn.k_proj", "v_proj", "q_proj", "out_proj"], - ("model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"), - ("fc1", "fc2"), - ] + ["*.self_attn.k_proj", "v_proj", "q_proj", "out_proj"], + ("model.decoder.layers.6.fc1", "model.decoder.layers.6.fc2"), + ("fc1", "fc2"), +] target_bits = 5.0 model_name = "facebook/opt-125m" scheme = AutoScheme(avg_bits=target_bits, options=("W4A16", "MXFP8"), shared_layers=shared_layers) diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 77c6df9c9..55500979e 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -3,11 +3,12 @@ import shutil import sys import unittest + sys.path.insert(0, "../..") -from auto_round.testing_utils import multi_card from auto_round import AutoRound, AutoRoundConfig, AutoScheme from auto_round.auto_scheme.utils import compute_avg_bits_for_model from auto_round.eval.evaluation import simple_evaluate +from auto_round.testing_utils import multi_card from auto_round.utils import get_module From e014f417027747e284d9d2ddeddfd51d730e7e5b Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 16 Oct 2025 15:25:48 +0800 Subject: [PATCH 79/88] update --- auto_round/__main__.py | 15 ++++++------ auto_round/auto_scheme/__init__.py | 2 +- auto_round/auto_scheme/gen_auto_scheme.py | 29 +++++++++++++---------- auto_round/schemes.py | 2 ++ auto_round/utils.py | 4 ++-- test/test_cuda/test_auto_scheme.py | 7 ++---- 6 files changed, 32 insertions(+), 27 deletions(-) diff --git a/auto_round/__main__.py b/auto_round/__main__.py index a9cc01985..a3ab6f2e5 100644 --- a/auto_round/__main__.py +++ b/auto_round/__main__.py @@ -63,15 +63,11 @@ def __init__(self, *args, **kwargs): help="The batch size for tuning/calibration." "Larger batch sizes may improve stability but require more memory.", ) - self.add_argument("--avg_bits", default=None, type=float, help="for auto scheme, number of avg weight bits") - self.add_argument( + basic.add_argument("--avg_bits", default=None, type=float, help="for auto scheme, number of avg weight bits") + basic.add_argument( "--options", default=None, type=str, help="for auto scheme, options for auto scheme, e.g. 'W4A16,W8A16'" ) - self.add_argument( - "--ignore_scale_zp_bits", - action="store_true", - help="for auto scheme whether ignore scale zp bits calculation ", - ) + basic.add_argument( "--iters", "--iter", @@ -144,6 +140,11 @@ def __init__(self, *args, **kwargs): ) tuning = self.add_argument_group("Tuning Arguments") + tuning.add_argument( + "--ignore_scale_zp_bits", + action="store_true", + help="for auto scheme whether ignore scale zp bits calculation ", + ) tuning.add_argument( "--lr", default=None, diff --git a/auto_round/auto_scheme/__init__.py b/auto_round/auto_scheme/__init__.py index d6db762ef..cb1bc4056 100644 --- a/auto_round/auto_scheme/__init__.py +++ b/auto_round/auto_scheme/__init__.py @@ -39,4 +39,4 @@ def register(alg): return register -import auto_round.auto_scheme.haha # pylint: disable=E0611,E0401 +import auto_round.auto_scheme.default_alg # pylint: disable=E0611,E0401 diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py index fbf83a866..1e5499904 100644 --- a/auto_round/auto_scheme/gen_auto_scheme.py +++ b/auto_round/auto_scheme/gen_auto_scheme.py @@ -29,14 +29,15 @@ class GenScheme: """Generate and validate quantization schemes for model layers.""" def __init__( - self, - auto_scheme: AutoScheme, # TODO support shared layer - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - dataset: str = "pile-10k", # TODO use auto-round dataset - device_map: Union[str, torch.device, int, dict, None] = None, - tokenizer=None, + self, + auto_scheme: AutoScheme, # TODO support shared layer + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + dataset: str = "pile-10k", # TODO use auto-round dataset + device_map: Union[str, torch.device, int, dict, None] = None, + tokenizer=None, + enable_torch_compile=False, ): self.auto_scheme = auto_scheme self.model = model @@ -44,7 +45,7 @@ def __init__( self.quant_layer_names = quant_layer_names self.fixed_layer_scheme = fixed_layer_scheme self.dataset = dataset - self.device_map = device_map + self.device_map = device_map if self.auto_scheme.device_map is None else self.auto_scheme.device_map self._check_configs() def _check_configs(self) -> None: @@ -105,23 +106,27 @@ def fallback_gguf_layer_config(self, layer_config: dict[str, dict]) -> dict[str, input_features, out_features = get_layer_features(layer) if input_features is None: continue + if input_features % 256 == 0 or isinstance(layer, torch.nn.Embedding): + continue # Determine fallback quantization type if input_features % 256 != 0 and input_features % 32 != 0: new_type = "gguf:bf16" - else: + elif input_features % 256 != 0: bits = scheme["bits"] prefix_idx = 0 if scheme["sym"] else 1 new_type = f"gguf:q{bits}_" + f"{prefix_idx}" if new_type not in GGUF_INNER_CONFIG: - new_type = f"gguf:q{bits}_" + f"{1-prefix_idx}" + new_type = f"gguf:q{bits}_" + f"{1 - prefix_idx}" if new_type not in GGUF_INNER_CONFIG: current_type = f"gguf:q{bits}_k" new_type = _gguf_type_fallback(current_type) # Apply fallback configuration target_config = GGUF_INNER_CONFIG[new_type] - scheme.update(target_config) + for key in scheme.keys(): + if key in target_config: + scheme[key] = target_config[key] logger.warning(f"Fallback applied: {name} → {new_type}") diff --git a/auto_round/schemes.py b/auto_round/schemes.py index 5129b4757..cb7e15e47 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -15,6 +15,7 @@ from copy import deepcopy from dataclasses import dataclass, fields from typing import Iterable, Optional, Union +import torch __all__ = ["QuantizationScheme", "get_gguf_scheme", "preset_name_to_scheme", "AutoScheme"] @@ -295,6 +296,7 @@ class AutoScheme: nsamples: Optional[int] = None seqlen: Optional[int] = None dataset: Optional[str] = None # Import Notice no comma for each item + device_map: Optional[Union[str, torch.device, int, dict]] = None def __post_init__(self): if isinstance(self.options, str): diff --git a/auto_round/utils.py b/auto_round/utils.py index fc4a2f054..06f01eef1 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -1998,9 +1998,9 @@ def _set_config(config, target_config): new_type = new_type[:bits_index] + str(config["bits"]) + "_k" if config.get("super_group_size", None) is None or new_type not in GGUF_INNER_CONFIG: prefix_idx = 0 if config.get("sym", True) else 1 - new_type = new_type[:bits_index] + str(config["bits"]) + f"{prefix_idx}" + new_type = new_type[:bits_index] + str(config["bits"]) + f"_{prefix_idx}" if new_type not in GGUF_INNER_CONFIG: - new_type = new_type[:bits_index] + str(config["bits"]) + f"{1-prefix_idx}" + new_type = new_type[:bits_index] + str(config["bits"]) + f"_{1-prefix_idx}" if new_type not in GGUF_INNER_CONFIG: raise ValueError( f"the setting in layer_config {layer_name} " diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 55500979e..38e49d4b8 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -37,7 +37,6 @@ def test_gguf(self): scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M"), ignore_scale_zp_bits=True) ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, disable_opt_rtn=True) model, layer_config = ar.quantize() - # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) avg_bits, _ = compute_avg_bits_for_model(model, ignore_scale_zp_bits=True) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 @@ -95,7 +94,7 @@ def test_multi_card(self): assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 @multi_card - def test_dict_device_map(self): # TODO rtn mode has bug + def test_dict_device_map(self): model_name = "/models/Qwen3-8B" target_bits = 8.755 device_map = {"up_proj": 0, "down_proj": 1} @@ -113,7 +112,6 @@ def test_min_target_bits(self): scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) model, layer_config = ar.quantize() - # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 @@ -125,7 +123,6 @@ def test_max_target_bits(self): scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1) model, layer_config = ar.quantize() - # self.assertLessEqual(layer_config["lm_head"]["bits"], 8) avg_bits, _ = compute_avg_bits_for_model(model) print(avg_bits) assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3 @@ -164,7 +161,7 @@ def test_layer_config(self): def test_lm_head_and_mix_dtype(self): model_name = "/models/Qwen3-8B" target_bits = 6 - scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16")) + scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "MXFP8")) ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, quant_lm_head=True) model, layer_config = ar.quantize() self.assertLessEqual(layer_config["lm_head"]["bits"], 8) From 5e1c4e8ab76357856255e9809de1194bfc34ea04 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 07:28:30 +0000 Subject: [PATCH 80/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/auto_scheme/gen_auto_scheme.py | 18 +++++++++--------- auto_round/schemes.py | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py index 1e5499904..d0dc9537f 100644 --- a/auto_round/auto_scheme/gen_auto_scheme.py +++ b/auto_round/auto_scheme/gen_auto_scheme.py @@ -29,15 +29,15 @@ class GenScheme: """Generate and validate quantization schemes for model layers.""" def __init__( - self, - auto_scheme: AutoScheme, # TODO support shared layer - model: torch.nn.Module, - quant_layer_names: Iterable[str], - fixed_layer_scheme: dict[str, dict], - dataset: str = "pile-10k", # TODO use auto-round dataset - device_map: Union[str, torch.device, int, dict, None] = None, - tokenizer=None, - enable_torch_compile=False, + self, + auto_scheme: AutoScheme, # TODO support shared layer + model: torch.nn.Module, + quant_layer_names: Iterable[str], + fixed_layer_scheme: dict[str, dict], + dataset: str = "pile-10k", # TODO use auto-round dataset + device_map: Union[str, torch.device, int, dict, None] = None, + tokenizer=None, + enable_torch_compile=False, ): self.auto_scheme = auto_scheme self.model = model diff --git a/auto_round/schemes.py b/auto_round/schemes.py index cb7e15e47..81b1cc1f1 100644 --- a/auto_round/schemes.py +++ b/auto_round/schemes.py @@ -15,6 +15,7 @@ from copy import deepcopy from dataclasses import dataclass, fields from typing import Iterable, Optional, Union + import torch __all__ = ["QuantizationScheme", "get_gguf_scheme", "preset_name_to_scheme", "AutoScheme"] From 7715eab5be51a7ff671f5a322d35a06263d2bb4a Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 16 Oct 2025 16:02:25 +0800 Subject: [PATCH 81/88] support torch enable compile --- auto_round/auto_scheme/gen_auto_scheme.py | 2 + auto_round/compressors/base.py | 154 ++++++++++++---------- auto_round/utils.py | 2 +- test/test_cuda/test_auto_scheme.py | 11 ++ 4 files changed, 97 insertions(+), 72 deletions(-) diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py index d0dc9537f..7e755a30a 100644 --- a/auto_round/auto_scheme/gen_auto_scheme.py +++ b/auto_round/auto_scheme/gen_auto_scheme.py @@ -46,6 +46,7 @@ def __init__( self.fixed_layer_scheme = fixed_layer_scheme self.dataset = dataset self.device_map = device_map if self.auto_scheme.device_map is None else self.auto_scheme.device_map + self.enable_torch_compile = enable_torch_compile self._check_configs() def _check_configs(self) -> None: @@ -83,6 +84,7 @@ def get_layer_config(self) -> dict[str, dict]: self.dataset, self.tokenizer, device_map=self.device_map, + enable_torch_compile=self.enable_torch_compile, ) layer_config = self.fallback_gguf_layer_config(layer_config) return layer_config diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index 4c32bcf38..e1a1be695 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -201,6 +201,9 @@ def __init__( ... "act_group_size": None, ... "act_sym": None, ... }, + ... "layer2": { + ... "W8A16" + ... } ... # ... ... } """ @@ -318,76 +321,11 @@ def __init__( if device_map is None: device_map = 0 - if isinstance(scheme, AutoScheme): - if self.mllm: - logger.info("AutoScheme is not yet supported for multimodal LLMs.") - sys.exit(-1) - - if getattr(model, "is_fp8", False): - logger.info("AutoScheme does not currently support FP8 models.") - sys.exit(-1) - - all_dtypes = [] - for option in scheme.options: - # Skip pure BF16 option - if option == "BF16": - continue - - # Resolve the quantization scheme or data type - dtype = "int" - if isinstance(option, str): - option = preset_name_to_scheme(option) - - if isinstance(option, QuantizationScheme): - dtype = option.data_type - elif isinstance(option, dict): - dtype = option.get("data_type", "int") - - all_dtypes.append(dtype) - - # Check for mixed data types - unique_dtypes = set(all_dtypes) - if len(unique_dtypes) > 1: - logger.warning( - "Models with mixed data_types " - "cannot yet be exported to real formats except GGUF. " - "Please save the model using the `fake` format for now." - ) + self.enable_torch_compile = enable_torch_compile + self._adjust_torch_compile(enable_torch_compile) - layer_config, self.has_qlayer_outside_block = set_layer_config( - self.model, - self.layer_config, - self.scheme, - self.scale_dtype, - self.supported_types, - self.inner_supported_types, - self.quant_block_list, - self.fp_layers, - self.quant_lm_head, - enable_gguf_official_mixed=False, - is_mllm=self.mllm, - ) - quant_layer_names = layer_config.keys() - scheme_keys = {f.name for f in fields(QuantizationScheme)} - fixed_layer_scheme_new = { - k: {key: v[key] for key in scheme_keys & v.keys()} - for k, v in layer_config.items() - if v.get("fixed_by_user", False) - } - - # mainly using quant_layers and fixed by users - from auto_round.auto_scheme.gen_auto_scheme import GenScheme - - gen_scheme = GenScheme( - scheme, - self.model, - quant_layer_names, - fixed_layer_scheme_new, - dataset, - device_map=device_map, - tokenizer=self.tokenizer, - ) - self.layer_config = gen_scheme.get_layer_config() + if isinstance(scheme, AutoScheme): + self.layer_config = self._gen_auto_scheme(model,scheme,dataset,device_map) # Set device, must place after model loading self._set_device(device_map) @@ -453,8 +391,7 @@ def __init__( self.inner_supported_types = tuple(x for x in INNER_SUPPORTED_LAYER_TYPES if x != "FP8Linear") self.batch_dim = None self.infer_bs_coeff = 1 - self.enable_torch_compile = enable_torch_compile - self._adjust_torch_compile(enable_torch_compile) + self.block_forward = compile_func(block_forward, self.device) if self.enable_torch_compile else block_forward self._check_configs() torch.set_printoptions(precision=3, sci_mode=True) @@ -464,6 +401,81 @@ def __init__( import habana_frameworks.torch.core as htcore # pylint: disable=E0401 import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401] + + def _gen_auto_scheme(self,model:torch.nn.Module,scheme:AutoScheme,dataset:str,device_map:Union[str,int,dict,torch.device])->dict[str,dict]: + if self.mllm: + logger.info("AutoScheme is not yet supported for multimodal LLMs.") + sys.exit(-1) + + if getattr(model, "is_fp8", False): + logger.info("AutoScheme does not currently support FP8 models.") + sys.exit(-1) + + all_dtypes = [] + for option in scheme.options: + # Skip pure BF16 option + if option == "BF16": + continue + + # Resolve the quantization scheme or data type + dtype = "int" + if isinstance(option, str): + option = preset_name_to_scheme(option) + + if isinstance(option, QuantizationScheme): + dtype = option.data_type + elif isinstance(option, dict): + dtype = option.get("data_type", "int") + + all_dtypes.append(dtype) + + # Check for mixed data types + unique_dtypes = set(all_dtypes) + if len(unique_dtypes) > 1: + logger.warning( + "Models with mixed data_types " + "cannot yet be exported to real formats except GGUF. " + "Please save the model using the `fake` format for now." + ) + + layer_config, self.has_qlayer_outside_block = set_layer_config( + self.model, + self.layer_config, + self.scheme, + self.scale_dtype, + self.supported_types, + self.inner_supported_types, + self.quant_block_list, + self.fp_layers, + self.quant_lm_head, + enable_gguf_official_mixed=False, + is_mllm=self.mllm, + ) + quant_layer_names = layer_config.keys() + scheme_keys = {f.name for f in fields(QuantizationScheme)} + fixed_layer_scheme_new = { + k: {key: v[key] for key in scheme_keys & v.keys()} + for k, v in layer_config.items() + if v.get("fixed_by_user", False) + } + + # mainly using quant_layers and fixed by users + from auto_round.auto_scheme.gen_auto_scheme import GenScheme + + gen_scheme = GenScheme( + scheme, + self.model, + quant_layer_names, + fixed_layer_scheme_new, + dataset, + device_map=device_map, + tokenizer=self.tokenizer, + enable_torch_compile=self.enable_torch_compile, + ) + layer_config = gen_scheme.get_layer_config() + return layer_config + + def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: if hasattr(self, "device") and self.device is not None: return diff --git a/auto_round/utils.py b/auto_round/utils.py index 5c6dd580d..6ed9916b5 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -996,7 +996,7 @@ def compile_func_on_cuda_or_cpu(func): def compile_func( - fun: Union[torch.nn.Module, Callable], device: Union[torch.nn.Module, Callable] + fun: Union[torch.nn.Module, Callable], device: Union[str,torch.device,int] ) -> Union[torch.nn.Module, Callable]: """Compile function on the specified device.""" if "hpu" in str(device): diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 38e49d4b8..7948a3321 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -179,3 +179,14 @@ def test_auto_scheme_export(self): print(result["results"]["lambada_openai"]["acc,none"]) self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25) shutil.rmtree(self.save_dir, ignore_errors=True) + + def test_enable_torch_compile(self): + model_name = "/models/opt-125m" + scheme = AutoScheme(avg_bits=2, options=("W2A16"),ignore_scale_zp_bits=True) + ar = AutoRound(model=model_name, scheme=scheme,enable_torch_compile=True) + ar.quantize_and_save(self.save_dir) + model_args = f"pretrained={self.save_dir}" + result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto") + print(result["results"]["lambada_openai"]["acc,none"]) + self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25) + shutil.rmtree(self.save_dir, ignore_errors=True) From 6f61ee1b0b3a12932de7f075842e705798102ad3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 08:03:24 +0000 Subject: [PATCH 82/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/compressors/base.py | 8 ++++---- auto_round/utils.py | 2 +- test/test_cuda/test_auto_scheme.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index e1a1be695..1616d4037 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -325,7 +325,7 @@ def __init__( self._adjust_torch_compile(enable_torch_compile) if isinstance(scheme, AutoScheme): - self.layer_config = self._gen_auto_scheme(model,scheme,dataset,device_map) + self.layer_config = self._gen_auto_scheme(model, scheme, dataset, device_map) # Set device, must place after model loading self._set_device(device_map) @@ -401,8 +401,9 @@ def __init__( import habana_frameworks.torch.core as htcore # pylint: disable=E0401 import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401] - - def _gen_auto_scheme(self,model:torch.nn.Module,scheme:AutoScheme,dataset:str,device_map:Union[str,int,dict,torch.device])->dict[str,dict]: + def _gen_auto_scheme( + self, model: torch.nn.Module, scheme: AutoScheme, dataset: str, device_map: Union[str, int, dict, torch.device] + ) -> dict[str, dict]: if self.mllm: logger.info("AutoScheme is not yet supported for multimodal LLMs.") sys.exit(-1) @@ -475,7 +476,6 @@ def _gen_auto_scheme(self,model:torch.nn.Module,scheme:AutoScheme,dataset:str,de layer_config = gen_scheme.get_layer_config() return layer_config - def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None: if hasattr(self, "device") and self.device is not None: return diff --git a/auto_round/utils.py b/auto_round/utils.py index 6ed9916b5..8c8c9acc5 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -996,7 +996,7 @@ def compile_func_on_cuda_or_cpu(func): def compile_func( - fun: Union[torch.nn.Module, Callable], device: Union[str,torch.device,int] + fun: Union[torch.nn.Module, Callable], device: Union[str, torch.device, int] ) -> Union[torch.nn.Module, Callable]: """Compile function on the specified device.""" if "hpu" in str(device): diff --git a/test/test_cuda/test_auto_scheme.py b/test/test_cuda/test_auto_scheme.py index 7948a3321..96d335625 100644 --- a/test/test_cuda/test_auto_scheme.py +++ b/test/test_cuda/test_auto_scheme.py @@ -182,8 +182,8 @@ def test_auto_scheme_export(self): def test_enable_torch_compile(self): model_name = "/models/opt-125m" - scheme = AutoScheme(avg_bits=2, options=("W2A16"),ignore_scale_zp_bits=True) - ar = AutoRound(model=model_name, scheme=scheme,enable_torch_compile=True) + scheme = AutoScheme(avg_bits=2, options=("W2A16"), ignore_scale_zp_bits=True) + ar = AutoRound(model=model_name, scheme=scheme, enable_torch_compile=True) ar.quantize_and_save(self.save_dir) model_args = f"pretrained={self.save_dir}" result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto") From 7bd62734d2ec496fb5a960c9a9b6b36b32aa13f2 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 16 Oct 2025 16:14:19 +0800 Subject: [PATCH 83/88] add so file and cpu ut --- README.md | 2 +- auto_round/auto_scheme/default_alg.abi3.so | Bin 0 -> 300664 bytes test/test_cpu/test_auto_scheme.py | 32 +++++++++++++++++++++ test/test_cuda/test_auto_scheme.py | 2 +- 4 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 auto_round/auto_scheme/default_alg.abi3.so create mode 100644 test/test_cpu/test_auto_scheme.py diff --git a/README.md b/README.md index 48e8f8f6f..a28083a16 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Support **AutoRound, AutoAWQ, AutoGPTQ, and GGUF** for maximum compatibility. De Quantize 7B models in about 10 minutes on a single GPU. Details are shown in [quantization costs](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#quantization-costs) ✅ **Fast mixed bits/data-types scheme generation** -Automatically configure in minutes, with only about 2.5× the model’s BF16 VRAM size as overhead. +Automatically configure in minutes, with about 2X-4X the model’s BF16 VRAM size as overhead. ✅ **10+ VLMs Support** Out-of-the-box quantization for 10+ vision-language models [example models](https://huggingface.co/collections/OPEA/vlms-autoround-675bc712fdd6a55ebaf11bfa), [support matrix](https://github.com/intel/auto-round/tree/main/auto_round/mllm#support-matrix) diff --git a/auto_round/auto_scheme/default_alg.abi3.so b/auto_round/auto_scheme/default_alg.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..052e98273dba44bfe573d8f404e196d0c4c327f1 GIT binary patch literal 300664 zcmeFa33OCd_AgvOB0&rllqe2Jkf1>n6EzwdORFTnjV2lEB{-f!=-@2!&~_1=Gd-+F6(YklI9y8Czb z*=J8@yaU&bEF9IfOBavxr<>NQ^7`M2sA zRWEO-`gQ7c1xfXa7b^bZhR(k&;w7PeVwLCR@~C63x$IRGQ9aLH_@Csm z{wnvo61D{Y_n7NDe%Lp!9E{8T{<#bMxZzb@2ps4`2!N@|4;%y ze378tt_k$~2khIQJ|`yde>j1jrzeQ-&q&<;$)A>>-PaSyFG(Q(j|BK{3H+9q0KYH+ zo|QoUCkgDjB7vXZOc0mpJ@)UP*AwK)!wKZ{PmmA86UaFi@!j8Ga7zOHPfQ@EB0;;i zC&-`83F6+Ez^-Pkv1x|K0@gErR_0`Qgn3a#|AP!`K9Id^~}iEeZ5_CV{=Z z6STW1LA$pk;D0hfyPqfU=jH_Y@J<3b7bWNy9SQ8ZAp!r61nr)cAWxbT|HlOStVj@-%M<8%MgsXO6W~`Q$g92y z{MX4_vQq0dL{7x3D744 z|C@h`6U2Q{0zbzR=xHVB*ViZL7vTi_KP13MLGfcuykV_`C#udnSRN-URr} z1o`7lAb&!FI3ANg&$AN9`4)I@&k>%Ce?uVtsK0T}fOsA3>E$VTlHvFmTT*h#tU0qw z<^{{<21`mjB?aS)N~TrJt+-_7ykN!L@kJvl=gh7cUpBR}LY3{mq@;X)S;_R7v&$-H zUIoVD1tsU9+PL7{>hhqcc)|HoFRdsKmW-@L;T1CmIxpQW@vPrA+%gd|h&aIdxii``+se(~+%YrlK%q9#hqi0@GF&h?D zR1sKI7A&7pK}}NGh{}qxx$NfgbE_RUlh;R;RnBv0*X;%KhznIE4B}*In<6rEYIU%a zBlgoSI}cUIRn8$#6wIocgNzAGn^rVuS~aqxv$f)L(kZ{9tg>=WIT#9&Qxb2!L~JCK zid119S8;iD#q9Eml6a(5I25-V&4StRSk>GL{5>v+01!VZwja149|e>hN^7`~^FwJ) z8Y$0!Ng#-#HUWswM!<$q)w9bf7JveCFQHfz&Y68lNnjrRfO0%Es`^)U1@_fB4sIV=atRsl%L2P8emY0NR*%-&!jGRZpDHt=ggf3 zsCrh#>>x$R>4DPg9dg8+Syg2)(QS9^%<>rmo;_y{^<8(g8CN;8T;rrnh}Vm!@P5Kj zSaC%~r5mTXFDJ}a{*oRMsH&=%O+$gxDV!FW4$NasP)v>(TRl5Cb5_N~vbnR>7)k*n zYUF%y6qQv~0WKLaqXOZO{3LhBbI{SkO+3RT6s^Fx5d{TqKXW2OW6s2ivdhL+OefDe zf<~Z&lS4*0-Ghf<0HoP@G@#O);0`EM;0$UsIFlzTrc(B(8h}t zOu@7y1$bWdtf>_+0n;CRNaKqjMb(wTnN^hw;z+^l;!3y~gLPaOOsAng&ZJ632F)1x zGq0G*ZjXl+MmS+c#m=~ldX%LYqA*1}=5=BlS5Y|~RcW@vD8iE}`!_!HkpM|^N5ZJe zIb}g7uS7$!Ni$z@1?J_?six&h@d5=m zd~kNkc#4}H=g_L`2ny#z-xv()4S^eiY?eBX3{AgXQ=t!jpwuq8p8@^ z&kPzmsD6`QF};KeJ(V-3mJgjbXXt63(S-$Pk0?2L=*dHK-Ro05C(W2MtKy_9D`wBA zE-RmLQrYaX@;S4om7Q`@#q2BQokVuC*(X5`7Qf>Q+yc)?$1+r zwt;7e9KV5kgumFpTTfMTN)5bS?5#5J8!g3OW8lsC3U4&V+YP*{#N8wJyB+^$pQ-rM47^$N%rNkl(TYFYz#AoAegm(O zymIcdJMC@|yGjlI-F_v%%ETT28~8548x6cw@FoMV5`CHtynUoVc?s@u51JUP;kG2kCk{88+h88N{>k^KFBN?n4ZKF= zG#PlawA*apH%R_O4E%b*TMWEPmka|hm2p1X zz$1Po*KgoUrQa1Bc$KtUYT$1;`ETIG!e3+H`NH35;0whMO$Oc~c(Z{wNn9cZUM&5q z#lRbdztzC!JN`HDV#yQFqR#&Ji~KYLuaWUB!@!q_KeG+IM)dI;_T zIsP~BY|*pEz-Nn`Mgwp5D?6JEyh`+GHgLc2M@(Guu*JYPNV}~D-tPF{#LrUtc&_j4 z|3(=n(+oV@nSTxZH1SWifmfZOz%vAIH}Izf_xwZ2bJqWjqJNr!XNW!-2A(E( zwt+t;^8E(hD&tkLfwu@=YTzD`UuEF!62}??zeoJsXyEM<-zEd!JxbZ%Y~VA+&k+NE zMC@uY@NALaYT$bu{~LIkAEWtAjJS6eW zHt;6#kKe#+g}>Oq+hv?7HSj9wFI5J`F86zT(de1OG(gl5OA_g8L1;Met$+PZ9a0 z2EI@9sWR~29RC~mMv>oW;Ef``$-rAB?#%|iTju?UffqaeH}F-`ZmWT}i=OQUeuu=x z6YlK)TLe!t@XJL`hJn|JUD*cSB6;gKaF6I;Y~VGrUMe+k+gbk@c(KT@G4SP%{|&rV z^lvio4CnmKz*{6P5d)71-eTa}#BZ$z9+5n6H}Jlqr{~7b{%;okGy}iP@xOsr2%c@= z&CdL5;CbTDVgt_-ywt!y7Wq{M{=Dd4W8hCY{x|Rv$NvW2B=$BN_=UnBG4Mx4ev5&p z3EpbpeZ{VJ1HWJRJq?}xpCbHe2L8LmH^ab}h@5N#|HkQGw|AEB5jw-bOLG)oo`L&? zwhX+PUP!>7Nd{iitne8I-o97i^9{WCd4(@F@Y0P6Ut-{`LN7J&j60P6%M83m@C^pu z^o)}4$aB~K1C<=x;4ghn@jLA}{PzgI(~g5bF8E?;-@zmQRdN~)JZ-tcml}A_45d%A zfje@R8Mq^-#lW8veKr|*_KV^N1Fu=F@ZAQ!SM>4Rsq}K}a^$2KxFaXSz*mSqSq7dh z_fNA8+#~u}2L6V~DK_v@@$)1DkE~U8RT=n)B4@sVJ926a+>x`yz`qbVO$P24yCMeu zz3^`^aEISE@Lj^+Zs5ftKkcs0ap{(+=)MN-@aGx0SNQz~UMliS4g7H7pJCt*|6&6_ zM)(^IyqMn_#k-*fUb9BUtJ%P_Mb0tx`%zi8>@K&L113zr8 zlE2%)_kH@=|o83tZVFI3>qd;_nN@wv&svrks~ zEi>?l&>IZgBl>JI@K({sHt-hFXSabjiar_lboS4D(I?Nq{X$y??pdSkon+uGk|#3^ zyj}F4Z{W?2e+)e0_{YSB-eBOxj(-f?@A${S8y)``c(Lf8ac^h;TqxsqmVq}#@!|^p z_znE70jj=b;Ei7^zm*#JB#}SEz}u_wVh{e*76E;}~VHZQ$AbB>_~pud_d!MV}M{&;Fjpdom3CRpHMv@U%9? z?>F%CMIXz+{Z}acOAXwSKf}PYa}Rs(nJwGF(vo8r%q?~>p(C69`P@>`aMAEY4uJ6pqhX}I%!6O!ZAaKFY+--vMj zSQ?(`fcP!eaC*nf`7=qwPjx{2mTLIv8a_kA3pKn-!_U+3`5Jz{hSzBL7!6;n;iVeh zsNt7r_!14jRKuGzyi&uLYPd{kT-vPRL5+WzhF_uK5e=WO;Ttsk8VzsJ@I@NFNyF)yEXg{4e#5ivJ$VA;c3+Pp9~FGBcUqG(r`5bD?D4n@v58q zCr`sEcbz|e4X3u9KbD5OYa(eQy9 zf0~95((t|-eyoOPX!vm&o~7Yg8lJ7;gEc%)!;jZ+zlIOda7)8a)bL^rAFAP#H2fqD zFV*mD4WFUmIT~K2;U{bOd<{QE!)r7=SHl-;_-Pv6sNs1UzC^=^X?T-{|4qY}YWQ#s zZ`SZLG<=zc`!qbF;b&_21`R(;!&@}mui=|CJfPvN8h*Bh+ZsMX!`n4HU&D86_(%=+ z+#1jSQ5v42;iEM?O~Wk>@2lYj8lIuy=V*AAhM%k9*&1G?;dvTftl@qQAFJV(hL6+m zVhta!;gd9cf`*rB_yrn1L&GO(c$J1v((w5jexZifX!u1MzF5O2Yj~rEU##IvG<=GN zH);6aHGHXtmuPsihL>shG7X=q;Smil*YFJ*K25`0G`vE?H);5E4R6)(85(YD_)HCN z*YL|Ue7A`ovJPlu< z;eHLjO2aJ;zgokKHGH9lPtx#fHM~^AuhZ}u8eXH}RT_T1hR@gVS`Dw!@Olkjtl|IA z@J0>4LBp45cu2#WH2j|$zEs1*8s4no4H~{o!*9~?h=$*+;Ttr(QNvp_{1y%0q~W(} zc&moruHm+Z->Kn$UH;{PzdZ1l2mbQFUmp0&1AlqoFAx0XfxkTPmk0j;?t!1YNB?fs z{*YpYlV_fk=^Vle!TrX#LwpqJEL1WIyxGerW+eO zqwl$NSEg6GbT_7-acMeQ*cpAmrIVSy#ib8mdXY=h4UC=9%UwE!=`xq@!SooHrW+MI zqXC!h$#jlOAH?(kmriB6w@de8x{FJDnf`H~V{aPMTV0xNT)=*prW+PJqbpsSZdB}y zKI77Kg97%uG~Jlk8NJ1&>4wD4=pvV<8xgSIrRfI5&S;rS(~XCn(J?MfHymKUOVf>p zozWbZrW*`9qXS%;ZY=DK_I7Eyp|CUB#ii*+!p`WAdmZ~TnBMBrbmIW_yYw+kuXO2w zOh4n&nM^<6(u0`3#ifsBdXY=h4FcHj(pgNGx%6PB$G9}z2!Q=AJ%s5Tmp*~%0WN(a z)4g4qZUDf3m!^Y%*uTfIKbz^TE=>phu-~QWfFJR9X*$?P{9XD~rXO%=I^akAU78N| z5r3CHo$1S6nhy98f0w3%eZ=3T=|CU$yEGl-BmOQ;2l$A;OVhzU;_uRQU=RCUnhxq= z{~wP1ex|p&G#$*tewRL*>6I=$g6U^mI-ltWTzVwax485urWd(19ke6204eJ;~EE?vm<0GBRey0=T8$8;B$KA-6ye|PLJW_qhjk74>fm!<=E#NVaq zpdImdX*ys>|95FRSV#O_nhw+vf0w3%blC6GbbyZdyEGk~BmOQ;2j;NfrRks?@poxD zAV>UNnhwShf0w2Mam3%H=^z~O|IM+#lG@2TxilSo zBmOQ;2i}OkLu2-yavaUx=yx<4gI;UU zD-8MtgMP}OA2I0r4Ehd(ZZPP2gI;LR)doGwpeqb|ia}2>=put2Y0y4{&Nb*04SJA4 z_cQ2pgHAQ*WP{#&Z0G#=#h|wv^tT56g+YI0&~F>`T7zC;&@UMDQwIHrLEmT4cNlbo zLDw7fLW8b0=vf9`VbD_ydV)b08T3el_8D}pL7!;QgABT#L8lvZszE0k^xi>6{0(}$ zL4Rw|Ul{a92K}}{uQliu2K|CTKV{I581#JxeTP9e7<9csFEr?CgPvv36$U-UpeGn~ zkwK3%XrDpn8uW<!l0)Z^aO)0GU$;8?Q`hB1%U|#py z3qpIW(61B5*{@)mhHd+TA+gNDa8*iC{WsOeSsiPOLO)x>z6egVLOTjWZFVLWv*4=! zd}Piyv9VTYb|)J1szG5lM6$wQvw$UrUd@JdW`)YmL7DJ(9P=bT(mkc ze!@5_^aqX^imgyEGs_B%&&;+$g_(JUq5MpLA+D@xRyb)8?dvQ+eRCpq?XMjj>yt85 z02PGRSPl7^RaBHhR~dAb1{H#1a<+&2WY+$E(v{v&OYN!@s~ZYJeKKRctS~bdr%-D+ z-biCdY_mGL+RNYzD3=P`;SGC%-{Xm;z-893SoJehukSQ#h|(sQY#)zhpjEr37zR#; zg{uP>6omE#CMctRvWH`rMD9d@ z@5TkHPtK2rkd}#*YG~35?Hyz1l9i$L_6iDY?e30XjulSdfM?gO+TS~>dsw05^f{F{$D$fqi0(i%ku=Ahp5&qE^`Oj&eo9eq$UPqagb(R@71S*dE%d9f-_M$e0-kcJsh z_ad!dP^bM6J$A`2oDvXIzdbmVY(9XKqz&EDj$tz%2pyH#v6}6v?qz?fI+OSIi1$U# zjMbD7ACa`5DLUiESmEnZ?1Rxc3qxPQlYO7<=JCGL+JD{!MfKaOlX4=lGPX4L?kAz< z5vp}cpwZ;t$0?iOE3`caZCkZ}bOd`BVCYz{r2US##168@NtgaM)*JC1bR^;%&O<+) zjAoJg7pat89hf*Wrz3DdQD~ivQ>-EN{*|!^E0#T2cQ^r{rZTvZat0^3y1x2?Z9)xp3kPj>EgRGU&WUC>VX;}@| zWcsHM>gZM&_FXWT%8-Q$u!b zs~1R~eIa?S!8hayIR6V)YCYT3hJ1aEa@PI?A#qaqQy>(UpH$kTZ`+%2eq+_{?Fb$T zF+V;IG2e-8eIcm2PxKMgv7aRjzStb4S1GLUHJN>p3mb_2xS!(kN1-s^`_lN#6Y{+; z!p$SSFTLY^u`n~+`{MY_JT%k&x62{poZcPtqg=V0I(%d1!}!S$-y^7KN#E(mS6M}jU4ZD%jrc_8S@Pvg1Q3L}}UFxs}y|B<@n zwt~k}&ChOcTr`eS+DjnwlXK42}DN5;bWq`hoskS*@egWo`g~ZugDq#prn$ zM7(w1;0QbHi{&JFditXHyCC#ULC9XTtpi8zRyg%q9Nr`Stz`j^-}@2_wN`FB2qXA9 z%IoDg=f1AvI)FXaqiY{?0_xn%lU6+oAe=svgi|{#_RS%n27F19(MbJJP&}Q7lcRE-2E003;eBzPBi9+YUW@Yx;?J`0 zp@mE+xsRjOmj{vMKZ3WXuML00Y+b85625Gu$DcumUeM|TlH_WY+exd}$pq1AoV1u` z4^}Ncjx7=UVHUS|9k;mf&n@0fBH`%B8MJmmi)Ec!tfe-TAMAHAcEXlPj^smF@*tKx z@XwO{BH4cy_(*d1PLh8+J}$XhNj|`l{4#EVgne(K?Vi4k_}lQ(-$@X3uF0pPNJxH) z-0OO2;)!v|M=HtBV}ronC}GK$v*bCh^Pf1zB2Ous#ur`qH_SWvV6 z_v2i>`(qIoys!!>1f8#xQt zr1?FfA!*(w*QEK?QkOKpM6OBm)xzT^0hWC)O(vma+8?m`t+@9U_U%N4p1yYcZKyv; z)UQc@L8<>Csp6{tC>6WvFXNt1%jR%$oj8otFP3Xk{|7OP)Q?bXp%tm0DQc4X%_4`? zPwK4x7rfX=z6o!Ivibv9{Zm-|99R9GzQb{i?D?3|WLfuaHAg+7XOFj&5nEXe9#&ox zP00f8uh5PxcvQ3|3y#+;xKpmlg051RELcbFqApo*j#%l(^~Yk3h3n%5FDAxGb~k0& zt9ZYWEgOyAANE}cSD-K8ZzC`_QfteSy3`EBs0vflhf_q|z@(CJH!%IBSu!gos*zbQ z$#q}5o)-kkqny?^;+ARHw;A<%`aZ?qsw;Q=+7mtvtL9no#?5t8J#H&INZ_WqDw zMD5+L=m)WSuG$EHL zvbWT2QsnP!XdN#TAbB~ZE+jXr(tCwmrbs%srP2sUyZvYGt1shI2%Hk9;=Xp+Hww<} z>C4C8PEf8tz;jw}OzUP)7L#3WQ2s`{Etp3^d6ZjKxqXfnlours8Q}Sw7L@PAH58Qh z<(h)>v0VFceUb1}P^OEMDJUlk4+Z5=X{!|1pGg8xPjQ1N2&;W~5%E5WA6@B%Yy2y~>)Y zNR^5w`vSj)l~CQs)*^Mg;1ns3@KdA?5}6dKEQvlv>SC!&k=h{F6saerU5ZoL3#6N zMMhNJd|{@bK&vlU<+{ukMUIk&HYw68WUC@~3u!Cz0U_HJ`MBu3Tahubm)^!jL*s>{ zfA14yKhY{pk%x+?zKXm{L}e(lR7$fH`J0qxEAnoUoTtchCF}f(Op>~mBI&%9oKvjG zn?%$kMQ)S2rHX77a)u%w6USC5@>U|lsRPM_;q-6#e5fvSzA8+Xsx^u{TFhOn$m4`; zRAj3JVTmFGVo;MJKaiu^#xW<@?P*+fXsM+VvCd=GD_YJqFMn{VkRekOUHc2hLQsg=Vfey0&)!HRP zaO!h6$~E=52s;*wM#4ot^|{@oIJy$`xhDmuK9?fbrMONK9cXl%FL)KM-{OOl)COuM zoIZ-(f+@>hvx9x|$Pw(5;6ZdM)H;N=da<5#{UNStvxkl92Dk+K$3A>qiu1$JH>kR3 zO$%UkKsixP?r`Y(6OLT^7KTrIZ75}b=!{J4U->}P{+eoGC!JK-P}PM_t5$7`AGc&d z7S0;ZzaNJOL%%1@=m67Ow~wW)xBp?s4-|d{Mte(~S)3Imm`*pUz4kHRsRlBmA5pCi zk5xVn$6~vParkyRw(%~eaU}Y0D#oT6hcR2o)W=v6bw~QIa9{U=n1bvOI zyL0^YHXjn#{*a;V;E{@S@YfNXh~qmeoI9D5>;t=YFtx+y#HY~tQ%cUpSFlh6hI6=S zSjFNn93<$AGgIte@CXsK!tt2>3M$49wx3h+tR7--{FVbr2dp@X&HW7hq%gD-irf8L z*-Se4=Js$T8-rhSz0jeM&W*R(d%h%=`8cYEqWUStz^@R_+BF%PsW|LYnTC526UL3P zvsJ{si_b#(;GiaFd(P&oI3-yC zt%kAUZT8G8lE(Q^e>bcTA@{xX(zEiyo+yOw9TG_maLA|(a|^&Ax+fz^s6H07Pv-yJ_?LcIzQs*d}kkf zI_W`%*+Wo-W7l9#L3m+`y%24~vLn3L&m@`HKP97sH{|;6QwQUzTlsL?{^dAUI*2~d zX4@brG^215YG1vDTIIZmA>VKa>a9B#veX%NX> zv~xoN2TSNfgP^&`+mjFZ5GvLTE^C8($q;w`+o2^YuNAJx;+D+Z8$ki z*|G@#<~iR_fHyHjMYm9CM|UgaI}3vBi`f0iKOMp&o42lK1{Krr@^Ad(s-|1w9v^SH+5>SDVR zt-p>nohRpg%7-omp$IG~eZZ7`6(^v&)%okSzT7w z{5G`uKx^fWl#W#$?^eHQh5G$=Fg2UJlYZdwMLRv71H1jY%H!$isP2oA;z!sIOO8?6 zR$m((K+1$xE&7UBtVO>Q|CJw8-IX@ZW-BwZ17od~->1E{0h%Fffn^8c!u#U;g$=27 z4|nstjKe$6Sd3`*gDAEdlG9--4m<6;C=<~ab2?r@#87pI{qPs;fKU*r5nXyT?A-+~ z7Tp2g%tR6Nx|Q^%LHwUi=}A=j1oi7s@-8neI5x6rQS|dDET912`t9t`&>Sp zhQ!K$jyP~>M)0Ji{$O`|Dl$8EviGHVDIM=v<-0Qb2h(u>BWGhlxX&8%0is zPH$TBPs`z)*{ZzFS*e^435fJ13T}1aJixD0BfSQ}X>$D8c;Vu{%`cmN+FXRN|+SZ&k@ZoRSx*!~+%VH&sc{DY+LV$b{(aFz0D% zsUdv>l{MsU0R;bGaIvSNqhZqSp0hdn5X6J0N%5>_o;Ewc48bo^a=j{fMM{oSC2LVK zO_e+$C8?_936zXdB~4QD6FMaZtXoi$rAq!KC7*JMw{9MdmJR6#sM5u%G-tE@vMRr0 zHH~tSW6-hQtUeXyZN3HOwXo?tRQh0F3D;V05B?0Y=muBv9IUSFf1*~bpIuALP-0ci z_P`2?UEo#vJK7gr*iLdSRKRhc>20@u!eI-A4)&I z-G@WSb4YC(;fjc7BScSa;_%tWiy@fneaE7^+W#6zxu5(BYH>y%Ni{NP+$5wScQ99M zvvW2niGS=v`6|@*^hWK;s&>$+{RK)<_T*tGJ4=-nJ7p_XnIGEP16A1xPT3=>tPrE2 z-JQy)b2TdPY1n1&L>!>zPc*_%SrBEoH~iDSZl2e>Ard(eEa=p2JNwQp6vE{7y@b~y zUcRIEItCfA+Cjzfl*4f>IAHPwk{-^T4$B$6*Smfay`i)Zby6q?p=} zH!SJ9?qFJ^n7TSlrzoa39HsyqGdf}q zt+IEZb5f^^9ZZ~U_IW%x=3at$IU(%N-cmvq9tO6suaSh{afL<<;?50hAKn2W2?r?& zgCK#H!$l+GD%^S~)y_STk|FF{eV|g|GEu=R0^Y>83n%16IK}OT z&w2gNooWxu^u31Ikw*L6O@_B_2#QG1RV*lXIk^!EF3*oE{REjE&TXQiurEh6`t(!f zlqQstfMH6&vm)RsO~9o}z+Fl}JK|4XT_OTXM8IyEqR@0Z{bF55L&mgI2vcr7M>%FA zEv&-6$3@CmkusRtK<#I_cJ4eaHnkjlCFD@$i9*(Lo&7VG=}v5Z@2w<+^z7N6Pq6qO{dvASf=lI(P%0( zaxA{Tj}o-yNN)Ci_I#V&9~wEeXHc8r+&U@>`|d!mA=mmuqbVpQP3}_y4&Z!lv$vcb z7jT>s5K{uii-0|!sNfzY0xl*2@a}m07KT$7VtB;B98T`Td6$}ukqTM(ci6zg2Q)ex zZP`7vSuFx%pyE{4VV!*yb~la@>#zVK>AQ|#W4eQdv_2hbIyoGD*?-?`LM521oROB6GcE!3CM+gq`?cCfFqQEBTxhZLqxy_ zA1g;4DgsVX0``aoK~2D?suyfk8hn3%GT~Mc@GaghBOmNMoDDq^g9mAFiY8!*65vsT z(_3&-bYvpK#P=kPi^hRvYk>pba@mnaDvDBGdg zQhz*Rqp7iQF3+=J-_fGVx{s7I&Oj-ta4ZQx&7+l|(`i!eqNbxZINoXR4YteImpqHn zsT8U>T6PtqKJUxz{UToJ2UVQfx2gpGOnK!t+KeJIhKnj6_EIX`0|BJcZTRiTT?>KZg#3ncRLN zGp#0(&HnDpxCWcS0u64XqOk8`5s)MTo)Q7mlz<`; z@Q5bhZYAJkCE!RAu=)dKLZt{8s07^38PaA?(*#_o1T3JUuLE&rgeU6Q$Jz8t{!;t(-=!Yz3!d*2&yT?xQ?>;H>pa`yr9rSLAjBH;%Yzr-u`G zXtat`q!JGltjDbz!0-#*w`riiaD0n%?<_*_yfPM?PD7vdaQ#;CU1i_Dnd?{kYT|-^^YBZz0xn~^%`WHKz7tT}ehL0_GIJsKNH;V0K%A4A z7qc#Hb{31;W`BXZBu+JU2xsQsS_oAc=c5|o^iSYZxHg-dNlztnX1>U`LFj2mZ{6R( z=Vs5?que?T9uh&QBIk;564fOnFCZSmtx<_nHkti``FZ1*gv_G zQEDFz%^hn#K|RjQHhbHfoSF04g?NtkS7gp`By<77!Kt}!y=v|7R$bE;0TE02L{UiB4h~gZw zLVwDfLVDN0eiI8Nl!v{#0502HS(@ z%cc5Ij&Ayq1c!4g$!OSfqm-sm9Kxx0kmK?E6%nC=VL)|s-=eh3<-p~B$J(v8 zcgT=?9;#74Ri*RUvNpR#O0!kzB31eZjf`mGSt&hOmASBwZ~bMMz16Wf-fHsJ&j(2l$%E-vdmxysTHkQwj>lWqfFe-af5or2ekxI6Uk3FQ z-1fn}?PSU^`eVm1C}M|J>+8ZbZ;$-CDS9JZDmrxpR&)hh^b#CuGbHpxhsHh9zzWFs zMDra;=u;hBp%qWGmjemC4+qKdwvo7=Xv~2EE12Hw(A+i#5>Ir40|izv zy(~^+kn}{CIy6^-t31&=94N4Y>3WA|7UYa4I@f^$E0~@dr_r@M(XkHARTw(UfdVTq zLU^JlIW)7NPkW+AIZ$8)(+4>;vtYjRM1RAHv3Q&5s6(?Wv1sr_KaN9iCH4lM=xRWC z*a8&=cvb;{71&VW{<%YC6LqIUT|m@gP>_wDX z2>`E&#H(&1wkJAC>xzLOx+D&vO@`_Lspj!!QqJb0&^CHD{#sW$Dwc$FuQgYWMEpaznO&LIlyl*C^Y2O z(Y^*R2L>a!j~qmi>R++*R40m!Qr#kUrpo=;X)3{ELsizsj#a4{J36p}@-KF%N~&0| zzzWKo*a0e@?2+ixICOVzzfo-Ng)0Ipo*{>%pc@8Ou)B`LWnt)BaZ9R0F-8tS;-U_x1(*KtSr(NvVok-KkFs)cDd-*Hb-Y0;ZLn zXbQ94w|K-Z?A!Mp`b6{K z>xTV!&E~+c7VrJ5@I*%I6o($*zXnOw_Y8QO^pEU_NeOGNPAHixRxP*^rzw z5Ag~iRa1|ci>rp*pR4iX9k~H<%4x+Vz3UpC168%gz5V?j?GZS79p4K5G7hfE(jK3} z@dfU}ezOeiYbSAx_u8{5 zVtC_q*A!=5;(TrJ{epap7Nbe4_M|Ss1F3VW{uYhFc1v8aihq zvavU8ZM15;cdXReg3zwOgrd+|{yxA*;|fD>@_-Qbo$&(&1$ktT zCEKyUsEd*_==t89jeMR&&b7aRnY1&7CSX|ymH;+DGw|yvzy&sqsD#HTldam3^weG@ zUeE@fK{#Dx@kQ+cU5aqZlTApTQ#6S#icrH}Ky~SZ3+VCPAmrjq3;l?gOC9Da=VHFP zsLQOO3;Zcquik=it{WM}4y6ZFr{SsdW0jKU$ky0bn!jXxa_5TxhKPLyp}m3ea2TFv z_I>?5MyKfrrKDUhh(_l`+6AVpp593F#OEsSZb7@qyJXvfXYJL>z11z?=nf9*<_)>M zXE84ZoYj<^&oxqj&=4N1dG?oXw1>oV(DaPxUK&u^z$49`B+b$nE-3DHxyauJ`TyCh zH0XE{GC$`2Z?9Hfz<}R|I?+rX82p_7`^~>p$ve(4Nml6x=hJ=^qHBIPV;gW%{WQ# z@XPK=kOY%1C=8#GnX|bd+?@=(3J-E$H#l}K=7qx0Zx%00R(G)mM+-tpy$ZskQwnO= zV{nZG#}p2WEyRO++0?6sNzZx9?*9T0fx!=fpywn*4w}SJi1mRx;~#}B3p!t3eBPw_ z2p{0M_5;kZRggZx8h%1%wKqmMJ-ccRJ0Uapq7@!VhdqViEIa{kkAH^sZ>`jgf8%AhQduNt=5|AY}VM(0tT za36eRC`d1T{tdGZb&|;{j|+!q2RB;bv#~(LS9mbZ>?jJ4N+}$kTn;VBslh(cFCaPi zZ~-PH<+l!e!6tkb&cN*No~Jezd}VK^fWqX@q8qWD6A>8@;jODUV^#45bMRu< zBNvfJFhLg#+ZP-~QX7@j&+WTJ>W9&Jkm^<#OcnSg3p$6x%>Fcz7L9MIwYzxh)iZ`a zMxUiB;p9_3CT|v!r*h9{Z<1$uc{B+giXINOLL6F#(dc7PH7$59p0r0!V0eiHZzdPS zdTTl3t((sd&2tp6bI>}%=dHUIqOeq~elNNPM-cX7&8)z)AH~D@XeAHG80J4mbJ6?2 zrtMGM&%^K+qLBago#5h6ji9s9$=^Eaolq-)147CnS|YBq{ot}{XVG-KhQ4&6KInw3 zqjfS40gIm|6J4|hR?>S4Q=Hj~<_$I9C7%do3&W%Fs5K_w&}e+CDH1%Opf;;Z@PHg% zZ}FrAckTHvjsPMw`KWfJ{`L?5hxK3GfBmQar}eMdUwtgL`a^qs4dno6e`Q`=dn}X0 zAK}~rv=Q4Res}CS!H~yq0EPbOxQo0+F8@27)$l{tTj5R&k(i|-!AVYZXtle63hE;$ zn;-Drz9#VN=Ny$_Ppp1{^Sbg@c0& z)Ei%P*W(Bp7Mxx<1Tm42tN7vu$V%)-aT^G%H`B- z%2%0aVzisYTNtQv$I~V^v=-mC;_-mqkg~I>+|gt|A!Z1P!3Tp<>{G#Ig%84PMel^xQ)j~dbZvEA>@h7m)A{@ZI_|aS@hH2H z*Pp$t^&{y}4y__3ForKodF>pK_P0V4 ztNW^=%Mb=tjL&(&JG@_^FV?9z*+WPL4Cf3@Z?uo3c3*Y(x6~i@@MAr+FM+Heg7p#- ziPWB|wuv%8a{oL7oeQgpEIdYlFMeU6a}NeRHz|*#v_Kc(MKs3&`Oc-)-%@w@_fl5I zVL`mOJYEO;BO0q5x5! zHFK&t^Fla%4A^#!0r);^0$8mDNuKz$!I+cmtrvyi@!6@L&r<9*OzFEU_ zKYY8JCwL6K*o#T(2rGPTihVd-L@SPr;MugzjU7PsayDDx6?|$Jx=$V6bp%zwQq^?J z0mDA;YBU}D!Ho|rEI<=5Z09sK%)S&pRO?raaX255H8qX5-2C0}np*ITkPt0_LNv$0 z(ozd^zB*&Z`32GfK9yjp@y!0?Q4Wz)fc7`O#qB?c_QlIsnteka+E0D(M~NRSmqSrB znQUjl%l1R~O9eCFy<-hk#XRavi}CS;_an-=aA*LYCZphAdlf>Y>>7f8hYfr;bc`w4 z)G?GJX?#*?4!_u^z>e5<-mfeD4;KF&FPgIt;dO2JLA}K7pWUhbENLGD=>FS(?h&W` z(3|YPRprVpkBfuIezqrJ#iw zqD4mBO9#1LqWqmldT{<0sP^8M_AVtekm+vzsPQnIJV5b%CVXeRdg(JPeL_^eQ)y>E z$YXgQK4PR(qrjBt=cq;F6^0JG9oKRk<;Q-62Jn6a)t-=v)z;eJA%%P}N#|nj{7?0k zL9Scmu0yuRT3{E>+rqggVR54dE_)C4M%vDPP18GlQUDL8e1a~*UEN#vJj1z<;H;NK zzYTV!lDF<(Qo~;Aum>+k-fX&%S?Q~Q?;>?Dc^twT2hY@W!jys6>%YSR_p@NfWKq{d zg6XlV;0bENdV}oU{(#t9jpyIC*;_zrQm_uP&y9Ozd-Z*>PsLvQ6n5Trcoh}m>oNOC z;DzB6&-k>PhmS*Dap)3|C*wISo^OHnq!0Rg?nv70@^X+L@uAh=F;wiV2C+`yQ%V}k zRX#xZY-nrZYQCidO&<0e>2wBP&*krzVqU}D5;~v8jvXtIe!pVZXn)2|3|;|u06KLO zpwqEAD8R6cNZ3PYpWTpp`6O15W+Cie`HLZ2(U%+6;Ukm?N_8oUc~)Kyas1?du)7M2 zkCvd_*^*#ja!!Qv5MM1;{-7Xiz;@bc?p0i)xg^zG>}HCJh1fwz_3~Jxk@)jK>w@Ch zsn(ZrHdahs?_g{Fh+-rXFD<=w6}VkW=Yvj>yXy4TU8IV*!&0q`c&+nQ5zhrw#E0QD zD0}PjRS{29RFoaBHB1%dIc?>|i-xKqzf(_eWf6C$q%RsLI7@t;}$BDXD}xX+$Y#E(-R*s#DgX z5B-}C7Ko~$V*57G{7)>YINP;E--~1tdLGgLiPPmo)7g)dk0W|ToE||mJ?10jrx5)? zoE}K@mNALwMP`h@XcV%xKJ(YtYE4qMu=FSh)zRX;7(2n&PHZKjubmkPAz~bvKLB<>n<-9%+_;cJ^%NG`f1d^JXpheA0nQ@$T)>DI1moEiJDs2}5pC=@LOXfu zgY%KTbUsqeyCEyoC)0i!ZNUU~N-aC3Po_F9qN9qO`+0U56{+w6GE>r zab&$;>P83ncLx)jvz>sGym+>4m)V}GlIp0#wq zzdd*%eb?nmI#san&h+5c!q5p_tndgNd9S7;25}$c+a${P4;dnFY=#oYs#7 z6Eb5xB{2peXWw$XQp}}KCK`G<tk6hVCELM+pxp|A2o@7Nb? zM&yRA^xo8udhHTxWowevWq1DY(iCs~fq;-P{9xoRUUJ%}lR%7X_bn2Q3doV$K^eP5`+v(*Q2*5)fJyzHL)W!QE z4m(JDgbIg@Fvk7}LJDYT^w#|jedwv=0}E=0f*b;|Mery{S}*-T2B2GGZe(tv@mvm$ z{J#nIk~S;9FM& z`*)Kq@>a&}P^X}LkKG5D^G;JETaNYg-Gr%8jQH#}IL?_L$PRjQt0DJV$fVh*>~`h2 zEl?MI044Ldq|F{JC2PnKyjRwcTSj`M!Aa_z##{F=+n8HH+uIPFo{#aG+UJa9Q_=*qh!PY2g?>7{jN1?w#-^ zLQsvx-Iw-?+bBd%?qjm1b3f_;AKr=z$*gk?T#C07zTi@RMj3Y4wWt}*hFF!SB&>>{ zSidc ztxb5mlYg>=wv~1V`3k30w72)u^HV!`o{atn5}bX(X7EsXO>i3J!dpW6gA?$M-O)6R zs9pM5bjyX$Bo4<2%VP_MpkN6N6ax$3^Ax+$t$U$5;;_#|Lu&1P9Jjoh&Ih9}b@NO? zA6zsRrxKX9F=0lNVF~Y}ar0ywAB=o`3+EfyU_+%?MC^q+v`NVaH)_#8rLYmj(Fy1aNN9c_QM0&F z&3F7SB)P}tc6TVICY<157_`a0PAew#Te#){=mBbNq`r&nz1X29%%(C5+9@XPr^xTl692U*1dpLdjt-VpZr3ui)X9dIZ;E zkS32>`{?j@J%fJ-MQ~Ob<1tq~a4yx5dd#S9o=ZJE{@GJEcXX5l@SRwGN_rn2cNtGBJHE-LzJdGNUTV0(H|hLt zwC}S0yAZAifl?QsnFyHor;bl~ft#B8nmz&EY z@)vpQ?9#4RWS?><@_e%$=|7)u3aG zu|NDF1!suKw?B)hHl#n3j}F_AIsrfKzvT1?2#k|sX)(m(H z?MI!^iRMU`Jj+*#jB<07$Sj1fTcS@;d3X*IJy!8#_Pv~%|Cwc|-vT*WMN|V!7 zt4F!5o{UzV2wwq|BK-HCVN|m+st$7betq<5l1cW5QF<#Y$Vtq(%`aMUaA+-rGmPrT ziT8Q%#+uN4_!+kLheEd2ev;k9;fgk3Ibcr)pbePL{=gY5|GYWiJrE|-XAw5)>S^Bk z8}Ne-LMh#7qBO&|yRhUxu?eR)^eoyEo>J7SqLs9Fr$>tMy*acTTFuYEpN0yuxE=@w zdU-I$GW&hr44HhZZIcXMk5d6XwuI!i-~K207+2BT0CT^fM|>%G>h??6_Z9c=N<`Cn zz|FpL3k3=0#q-MY+npko$*v@&74c zJbv1@K_4}TdFze?%9AzqSS;4p+vym-plkiNu!IgU_#g7Cq8=4mE4q&*vv~`YhbvyY zx8Xu`??ad2Tpxun?7Q}9iey{Su!Y#r6`(cy!{4}hcGm{Q``ZS%D_Pw2VTf{8b?Rj( zmqw@HvM`)`Eac%ZzvI6UJM0&F7=PFo-rMOx{L$9%6Fn#mo=GY4Ul@shvJPY8DXdrE zwZmW#wg~t;AJ6DD_-;IhG}~@x{>VoQ*K_a4N2&M4Rx#x_F!Q@_Rk(mnWf9Go5EQbz z!x&Z;`xn};fawIs7*eA{lf?OV41h>X>rCj zbjHwH%AKbmU0m@#E@{K`8JG%ilmIIVL+#NBuhA*P?e-hc-=`{?#++87sna6QiEbgf z1uJUMI7f;Oq%1_eWrWjr8iCtG&_ihOyzM>LhWe{fhciAMIz*oXCnncA=mFopXh$dW z5UXgCH-qFI!XWE4iRdpKJ$|({Qm^(BHa&ZA)j+EuKybx602u^`&`sTZbhF3W}gWHi5Q)W=Ixab!_N=V zI|9Etw@vxUwm*-`?owyD_;)eompkOCrOy3)M31EopCUehaFcs3tRoG$tKP)V*VBkr zNVR#5@jKXPse}8mP>4cTcLNvFIE$rC5q;oN4aaKIR7;(6eDS5umG8%wI_S&A+B)Y( zNPq2MlGosiV0A>Z;H2$v&j^(Vq(k*pXdVAVTotu1=O0Oq3Oo&rq7F<;5bmYU@z_9f z6~d6HHR$N2ZXP=8iTuYA>W1{-85lbnQj_uHt|umdz0iD2A)2*4p@I#`o-2cI&*cKL^-g+N72HE>2)Do`f zV0KIm1kgnK06s}w5XK|dc!Z61w9YT0Ul?MswD(gRKbS7#(;x64JqL%%q=57MTXZhV zR^uOK5emmi~bd3qK4 zwV&`8x_tTw)#WRq@WtOL|5yil|4Zp`4VHAE`6=V*!NI=uuvx~BHAoi9 zi~a6Lc0-KpNBX?2_9OH0)c4q0y5;#^+zPALo3AC70X_sr@x7xim@ie}C5A-|zEGoZo+5y?VZT@3q%n zd+oK?UVH7^%s6?MIGNu#xx_sZC+`}PtLnbd_5Wq601FoBi_xH3AacjUkl=3|V=J8Q zF6gpB4$>H5t(TY*?uvL~mh|Ny>=0OGa*O#fEmh#Xbr?1ui}t+>a!604rPnEkE#nHDA24@)Ee0S}sX zNHXgB%{5;B7+H)n?&d24OFJ64_g-{}`RZ!oB~?9;(!al!%`lW9|3ku+Q`a}fmhPiO>-F2wNa|rF zm;s5>!aZqR)f%NSe7PLoV zb#9N)$H%wPRZB$opnXsQ8kXli1LNrGWj0jJCSFv}8>^me6NKxZ6RakOW0+6-A+al8 zen>n+O)u3{CDf8ASOCMh`Rs5UjDlEzSl^%f?eg&6}jaYwY z;m(DBL%6pEOxv5^e!1J!a`>VBpGxvKZSF<1DFt!06;uWd_%7uIz4#oT)wv77x-ylS z;ITHcb|Ez}^@^nBGTj;5usw%CB8Kz!Kv*O1t!N#rTju^@(4Okf9?{F@URmPKvm{^F z@m3+0LsFUH#y1YMAfweLB|i75fOu!nwQxC1;WFD&l-c%bfUm`;Vg!boWx3tfoIx1f zYu(A0E6XdPdcE_N-qb#|7QPht!Pz%?ISDccjazR$I~f}L)n3K9Zw5SPAnhz|BIU0A zTe8Vn%4DP(l*1k7Fu*~$41`JG1H+EXy^qyYXU=lvX`H22fMGrbF^;>Adv&%6g16{} zaQ$u!fntrJS7@riax-tNxj!lIQT1G;FDFk0n+$Ht06H z>Pw7wH6Ku7Z_Rq%dkkoN_&OrZkN5w> zO^v)g@c)0mI&W{6nPL^j;QzmqoU#A^mDvA(tW`>?j7jT8nlnZ_K#sDp2yng)gB4Py z(Nu6l+8Lg-;hv055;C@hjD~Bx{=FNylKvh1d8~hJKFJAXmMPHLsmxhIrcD^x6Zvv8 z@K8lZ8}YTEK(j4(Q;2JNM3N3Q7CUy%Ttw%X?R z;DS3vB2~xbYD=eyL^l*9%Ey)fvj=&qufBWm9t&tAxX$`WTtc;g3BHo&1GvsRG0NW&gy%1M|SexxCr&|TlD+RJM=2kP9 zr!!4&`UmA0l^)VPLS(4MV5o@pur^UaRwp-}^yp4_hh8*I<}M<}6j-#zqEXyvGj~D~ z{h39xEl|-_N%RXAJ;&`SjN8E4>bf~@$2b{P82t_eJTvlayvlM1Bb?=8yl>rFq{Rl9 zhvAsNNzRsh&teRTx2vCF3Qm^+Dto0a5yKmD1z~CR>0q#?7y&a~>QmnonFfp+b|-d= zUNX2P?pENU_l(*lZae|0)#$$#(d;fJqA4d?IY(dh)5eKYIr5(a>u9)#|7i_(>yDCA zUm1E-#cpwrI!bCz$23yeZ>;DguB%GarGO%HD+R6%rER3Zc?9s;yQnu@i*Ythb7vzi z5}OZR1fy;6O=V}VGVT~v(W#!IEj>kTpa@Ey!|JJgNmP4LdSeVlJf*_pA^b`LsKQ%q z%w8eG?0O&Mxy zi_`umk+b?-GQUh_?uW7zPkVH^$N>4LN`H=Yh0 zv^_#1Z;~*a0vAHOIupp=yTvNlxIK(DB=zZY!RT@61v2Xa-90xR^uRX*41I%jqhh)O zXrtI?Ib6lRVKrR>fGX>2x;}9}lMm}%B)WgJp2c%}rPs01BZ!G9{OG8k5rU^6+xD~A zWlMwVcujOroo#@S!-ZE}uPRaBnvE<}F&<}T!U!U8^|{;5_CvJx2aj8r8n=r1>UVr! z)mxi)GtVn}Rhv}yN3MBv2vuNxisq9@uv8!!Hj8F1WbynbD zXUTK!)E;KS{~Ji8q(9V|c`f6*zTG$7jM!%V*r;_JxI>>m(7MAC*8}de?YSSpiNfz# zUa%HI7p!f%$LLf1+&I2B@e!^c)}#Dh*girJL_?!tT8_YmdA(2X7SiwW>2pk=IUCO7 zZmg%fy;PyPfjwdcnihUtIUCjoj!JF_(I$Rv4eO9QXwhwTF1d%jcD_;Q-40g zq$IK#hCjMjd3nPom*j%r>=$_Z=mJ9 z1M|AJ1*OJ75^J{b4Xl%6+T4!Mz5)qscC(6;)wvI*RCCRVmln~pGq>f-loYQ#@s z<0*GK6XuioL(Dl2laa+l|4vg*H?!PGECJ()&=k!N8*##ovUz5XohBG zs)m#NZu@cUrmx$g=^sh|t>(7AVzm|;T;gTJgMQ^}9 z1q_HIck=A`jqE=C2&IGYlheSi78Lfk6*!K7y9vOUZf->Ivj2|wqfIW^HbzdfyHgof zf=aosC!lFtyxadCz%jtbyd8Z2ggcL>kUSuFII4;!i84;V4W}fcc64*#x1ZYAxBIcP z5{tI<)W5#1QO{z|A?lQH2|FjDE<>4PCPaGO(QVSQj6FgbKSLR1yzb7T3>|fAcTX3t zy}4Tq_!yhlHoPL2-amO-YI&nh(sBu-KS_IJ)z#nI`c|F1sLcg_%1k#M)9llj+APye zORM^w9Db*S-$s7j1?o6b-O_2OUw5@p>OBUp5Qiu8B32WsxEHYml=`cx%XFKo-(D(2 zp^q(8EH4S6g$j*pWUG3no3n2>THinT__6w4rSGD#QbUAV2~V%lAtVnzy`bh2HQs$E z8-Z7zjsLwkb={@yfUGY)j5U}1>JI)$^E}**yRt#Ij5AB!mPI4?^V=E|mR99FKMFjX z8|rbFt7nLNEuEXDFML^>`m#1`QI@=9emtUm#JSAnC7`kehfKFwBsSnGFo=g^h>wIQ zKr{-1X|_RRVu&0UM8^>7nhom*h?8Q7egzO}L0eMM7*-KOJkM3{F^)$B0XwafE-^%} z0y-uOg4!Fzno~kuzTt-X7{??MW#cuwFXH3}3z8=)Z+*K~op65EEwq#F%d6_WxK-8P zeR!8N8c1Q?i~K-^$2*soP}oqk68E|7_e#-9$0LBSKJBB-k|s#S?-*%L@Z= zqYu6wg5o-<_YkDz*1M|f!Q)<9B>IR<;r_<58qj3f*@P1z20l z7&Qu1C?HxC=%IipQJ}X1jUiB_fT&OkN@RU*AWHAb!ccYYaVxYsH+Hz3bT+M@Niz&$ zgQb7Cb4twIu_0PKUoC|Mt}@q$#*>qx4Bln^10E2J$00NxDZas8bsH3d9t*bWx_7lL zu-T13&lnPV_@Hq+G8xcwQr?q3*}mkUyp}IBg0ELI=hBDbsg#M|K<;>0ada_n_-3vH z+#Nxg-lR)qww8G5SRi@#z7w6nV0b_2YV{>rNs#$-A`!BE%GiXb#`T*+{{65A`_)>8 zM)2=SqIBj0tH0{H$Gw_CT`VLl1EfN_!-W6~!$HN6(p!O5q`1Sq{zzqBrBcjzn37>u zHEnGDoO7YS;g{68P`l-82EgKTv+-mmYkq5s#(ZjCYCV8dt+fo7GB(rB;L+uFJaC+6 z%JKFoZl1`OjpY>dUJV}UU+)oZv9X z66@~6$zPotg*fBPw+h}#qB(oLKrc8^BH`CJ4vmu})V6|sqQ{s$-Y_D{L5>;KwPyRO zs|Wz)07!aim?iE`8U_*c5_>qL2H#6R`!sL#wtQAj??IcJb1jl?$fY})3^%*ghJba7 z4NSsMm7JORvq37esZ~AQ45%&5%H6#;m5|)-oiAWJ1GBC@RLnCTs7qt7iF*Ys_e&YsSNLTU@?M{rQAEB54NnWO`yq`#7XY6A^t$h?YFY%I0Vo!~(;Y*MO}bFHhYv zgGtsDLad(*C|;P#E=IXm_fJ2osLUq{pzKb7B->7$8oxaV%rMNHKq+hwt~s5eu9rhv zb#;qOR6S4jx7J9%3A?1Y-n1m|1MHNbX)+l|x5c?{-V^*Ep&D;kGaE}6@(J17K$Xcv zbRMI*=^ISP6<345z*>k7_D-%(3gR1S!l*0pvA=?Rh3OiDqMvc>6{JUURRz2nu%g#n zre9Kk{}H9%_P^67EB*9>^u*|Se?%}8uCswzJ$4%&9 zQH3S}<{a}44Z&=qZA@j@7@Cwbkz@_{!X^mrRsrH+l|!tkP;7<8_6xD1Ua>tambdN< zR<%}aZ;L%B#5RW5DvRY2Z-bqs*!l_HXi4lVRwjgi)gp$iam^a$%dfvmg0~@q+Dm4X zRM)Yb6rDz*zmrXiR!G#*fHC9PWpn+cckGB;RU2125w6r`equ|vQ=)mi&Nv=+#g3Fg5(b%RdHvD}v_g)S~ZL zilKm_Hll^mzE)xW^gZ5xa&U1V=MhYwYu3wr?r2DngCH!ps`?n7waSOTVyv!&D66V} z#y@<!_yNzP*B%cab!Vl-h0UaE#toy{aHp>|$jrHSFX{@nFr!oU=0I3nW zC9a7g#o-^UF>di?k>3&<5F{L6Xq2(GR+rh^pkDG&c7&SO>04#zT{1xfShz}AWZ{(= zK8CHiYu$Ggpw*em>D#IrHZ*Cmdjbt14K}6|zAb6DR{&%Pt7#jou&V)J{Y-ED?6S3A zPW|ocCvKeHVwT*KZs#u>m=9~O}KR=(!&iGMXV7VI% zRP;Ontn1cH2-hz$$@w{zNJ?t(tf|E!yuNffvDK-;jX}y*Tpr|V2Jy(*EA3;6YOnqm zB#JFXL8R?Sfhzj(Q51U4FXopKhh`~!WE`I3rYhVo4li+!D15dLOMKKz^~c_)81;cn z#e*JxsbxO`UY%Zl_BvGT--JBt?QwqtZn8lEw`z^2e_epa7+}5tXi0CC46558l|3@} zCw?0Su?G@B_IJgFSY;|q>Y>ihH=-|C(ID~F%i_cW%@RH46CWx_RH@PgRnZ)u_((xw zqGzJJd}6L3adODn*Vc~d4a}IMt}wE4`Zgp5H}~!E4)2CK!@dx2OTH%)7uE8gVtc4c z7ZSk6s+?XXq`|<-9xk~3_%A%^Ko0&5iNU&TlpfVzG!Yn2O0;Dt>!qO}ycK;T zL3vVvn=HWoT?KBpz)W|m0w)vrEBsSD`c`2ra#6Gt0slYRT|CrZ)CzjE!S-qOj$Bcz zOHHoVYloa0!{pxUDP|0y{VS($r##2mXPG{CvCpph9LOj4)}G?NKavZG_=r9Rx5a!K z$B09f)4K_LE5Ow@H~yrk?uu$d)L&IjgN6R0Bee)QeS3Y*2a_7=Do{uBj79i!6?oAC zYu%{|+;0KyKTzOXV_wnDm#EXCTIt{KZ)Ejdv8kf8k@>LU_{Y}r7yU|MDxtHozDHIs z`Oyp3Y|XAn@y8-ra_p=+g1`Y5SnHY;=t-b*`VOMCXQ1_c#s8i7fvM*fMSWJ93`T2z z@-6?YQcty9vU@CS&9Usj9W0+KcwG1h4Q`zC6m9)PNit`jv~AgyxxPAcNiXDRPfbh1MqyzPJsh@gmXOzvy=h%qPH|_0=*P+}Fb(feAB{*_nPatJ7j~Q)KOk zMy{xqOsa1&^_87-6n3ebMxfx3_xBJ1kwiM%<^$h0%BEYx_>Y<#9LhY z%!?K`7l4hA!J^O6yOq;>soh?~$aVX|{t%X=R!-kVN$Z;kHoGC@i;8I!-SdBd0=nN# zV!o#WTbP(z4y|2>%x8hO;LOLL_Lqh``t6w4@B+Ur`rGd>m5_4&awxonW8<4-j8L<_ ziuT96M*l+sVzwg_(AgGGe?C+kvKdX-U&=ABQrRPR%g3pR~co}l~MjM%LbVn@K~c&Ve-=4oVWWb!c(7H}KbAPlKw zgK4t+V=H+YbLp#Kl*Wl>WHz2z9^#X>ydEC*`yrLtkD}=4Th?GX_dAe-QEyX9I2GLN zmKsz2TyUsz(yxGILi#$N+wf@!zs8_6i^;@4{*uMcK@k%4|cd_V10~ z7)*Y0+L=j=qOz}&M&J3JzRv{m-3d^qU0_Y-%UkBsRs&B*%GcoQ@v7Vs*U^%^+g|k! z*5N)(Mg8W{L9j4()V*l(fB~(o^X8R#i&PpjnCKP0jvuGMkove>qYca$j0^iwKxx{^P2+Bq4Q(FIaZz=Q; zgGpHbrd!(m2j@dR`ybm-GW#F-$?tz8fVq7{TUAiQ^dss+%u)KX|FIfOwja>?xH`T% zdc6OUoQx$h{2!bY?|-~^Gu2TS?0+&>3Jxn6Chqa?wv z%O%3L=nD&^aC5bMF8zqSJ4oYFLU4Zu_!u(DZN;b_hpFHv@tKmPWO8vLR#wcLaWu& zuH?>#iVZ0;{JF6rJ*7@Mh48%s<2~>5Vjc565icUF6w>VK4$fEWE(vG?Y+3a>DNfruDrbafnQLrcB7`_sntkoCVcnj;o&F(iCACG?GaFq<6Tzbn|!~{9| zoMgkcofReCqMj-36gq#dyt|EPa`aJb2iN5eH3EH2OKKR-Z*UiiQK5ykR`z8hrh{Qt zuLDJycHU#39hfJiQ_?xV1JcXS{YZ2?Ms4^f`RPWllKfl?yt-boglMI^!i0`f9$E$H zk|*GK1Cxu;Z=>d*a@c||1a|1-G+3@fA998z0`-pdjOuQ^lM^3ice?JkBP`UJ6Ca3H z{=~;niu9sI`8ot+weDzy=wLzt>b>E(zp^#94BQ+B8Ea1*2ObM+GS7%~lU4$E5I`l^ z<_ao}-a5q8i4GtLyLr$$49Ih?z12$N?pY#DBnK9hBrG`HQi7QDTw4;zduWxo&93uF(?ArwlQZ3aj55C&%hx+I z?s?@OossW(jZ~S`&lHt;S^?&XS;z`K??%I;8a659GfaVJXL__{pmlc|khe_tR({!k z&F+@{#Tl;+e?p&oUOoWDLB-|%q7gPUb@rWmexRpwFUkqZ@zg>(cd zs)ARBjI9CiobamkTPqiR%-zgUf5=JfFA0SoN9@2+wk?U#-}z5N)L~R2dY)-sfqxX? z?STlkiZ=U6K+Wzu%JfZBC!b|NW_yxeW-iE1Y@qx>g0+>Q7g-0q&r$*4m3zN5uM;rZ zOvkBP-MQIJkp5H?ZZdt>g3_&IEY$Cllr8A@>>(sdVs56aXkQC7<2rJ?L*{9zU5eVU zbIBZ9m8m;lACMa8q6?^qHHtSwXWGB1`98D|5DZ;ZvgITIJ&vq<&1Q?7LtqT~1J@Rb8myIl|DMfc4MprDRbhk5cqG z1j2cP+kT*o%9ys_)j+&Ad8()F6ID|emMUi_=2`JrO z3*!<=KZ;b>{WQ4lSFOWvDtt+lP3BdieWbFM{pc%lqypVM?e0JY?jR6azBBoB#3Ex2 zvBYh!^i3=<$Nf!#pDgDR*Oq`83(@?5Yjruv8v$EYH%DrrgLA}qn3Fv|AN^huy&KWc z?*~6Sc`Y_PB}DgIBNhE}vdQ$KUreU|`A6$8QhJ&G7krO)Lyu8Qk9$ml`z^?#F`w}W z9`Fc`G6V;~Hjm&mAt(m{$|_Lt-#N;Ec_&7zdYVIS#4ZT5`oRS16uQRoyr@1pqI z5Z7;Ly8bsKb@)i*??Il_)vH@}&W#~B`U-ihw5f9R5uenrtIxE#&-A)w(p1{l z@9$`iZQ$R`q zA1w^mdp_Ukv;D(oE4OT7sn0f8*>)fsS~c+IfkJkeAXy98%rI9}?a}r3=$8E&WG#>J zH~Ln>&8T{{F=H#AtH|e?N-obVPsA*B$9q{N2pp{9lLJ{HbiS|0`^Q`LU;V+VKL!*D zh{tk`unYjWz*{yKysC1s8r*-i2Q! zZx(KGbK$?vwU6jc@6~{{w@&;bfIuUw#2y3Ki)dyBC%u_wi}-<=6s0 z{!M=a`05$rM0MS&f)iZvdaC*Jp98O_d_rF+5A%YB;!Df>BC2%Jb%8(s3|@@n9E!Q> zPN%NSaz4FCvm88jxDg&66YgJlUTJ#aV~N{%AC+L&Uu8ZJdvfW$0Fy{nSU@FXQ_V$baJ^FBsky42?x%q+>jFYJTPkL9ZC0-`46dP ztkSo{0c?|z#2KXiB_r=J&7*aC-9ih2?n=1U5 zeyZOll^IQfxg^#CTb*lnx0sMCKbH~Js`jX{4EE1)^FSXr^lP4FYps_qaL-$%?Vwwx z9VGkd*#Na(3q~EJLhAHndimp&;C8jzZ0Xtj8k^V)7G4sc@D+=N*-q7ABf zfZcfgqP5S4#wl$Puhy%#s;b-Ac&g2*#7L?h2ZOD#&Ntb@3KmW7h){^VkPG+yny)rm z`s(!MYtXoz;O4f*G$xw+iQak^*tBc}PTROt{-%~5>b{hVjODwUZX@M^9OYVHhh%8F zEvizPUWU$m45jud57i$a^lhg#o2qg9bi>Ed+L%qF`E~1?hPP=|(+(&2pkvy!N@XY5 zG7fiTZI@R2yvm6y4ziZ1-Ew>IT9SIsb$jG{TMg}#Z1g*q1POSV@$hvedGB7cJ2E8U z!^W35R)UBV%AAX_^GO^D4Lpjcn^4qGtMPN`gXsGjC+~&zs%@-O9yMCiwZ@>Cu9oE3 z^>~Jskk&0=wY8LObH2q;;bSy%>IH4*>ob1uwrQt8#4Wy*?I=Y=bPPmT#`9T_LUTQ* zv|O+2MhErSMbZ9xnkE}#ytO#bV=BYdB+WV3lDyC6&O8Md8yoc03Z*^XCts)6!0P+; z;@x3{_;@f;Ep(l6)SU*iP3|P0+kg-Glmm0Qdx=&J5=pPtI?_OrL*2a$nR8q+WF7f2 zBTT-<@-mtsxZ03L4S4Y&>&7t7>Gqofzhb4T5jSmTLx*nLPz`Th(DK{QDn!RHq^VMI z9_O~;^z)IHx8A&@USf!${(b_cqi--#>BJjZT!y{C)$Hme(YvnaSt6=Ri}+-%szX;D zW1Yi#Oh=*gswFh~uA3{yG+ere7e9KptKZF!JW-6WK=-|2afOo8&5)$4PWO_t(7KHm zh4&3O{X1$r@m~I5G=ots`yv3LS?(~{PTi{dcP=01eVBZ1&S$)OaJW=s|IY0dmGOF# z9^44A!HGH8B#$eP3c5rU+vGAkkYGN|tE`Zs5fD^mTjZo53M&&CiuBet@GN&*g#L_D z^ke_iaS;qF8|)%+@G~bfc{EHE^8OS9E<}H|x1BA+;mb8Z_XhGp&Iw;cVc0`eezZ$`+>j zGne$Mm3{bfXYZ^D3}3-sb;l`U)z5NN**%D1y}jEJ$YFiyC5u{Xz5Uy7n1T4S&)ok< z9FD}VzmKP&eGwEDIGpwNd%+a1w-3=qo2r8K_MYSn>+O4760f(PY{TK%A~(KoXbDIP z*4qsqx^qfTNZk$}1@a|l#z??mfApfNh`dcydb6fum^b4Tc)|kn-9-w#On^x&ZRAc< z_=o^`i~_noqq=TBT9x~6h|t_QTKl^qms{lSailKZHoJ0Yx89_bxWADiW285EN}{Vs zh8Wo^K6&*{g~{hxZoE=)@@FlQb>o2f$>QXf{*(+sQq+BYFUcdi2}n_3q{Lk*=|ocG zeHCiilcJ~0D`EEm_08it&`KpozFw0Iu-Ww_DLPFJ1joB5a2SF7`T|8%sgFhvdMA(zCxqx%WF=(8;@?r5I8gHO;+zA40rMl#b9yrVd z{-&=?xCcT64ECiCr_?_&^v}G+SLk({6_%79uk+8N-~%YwMaYW{F}0{0Q;5(+&LoHB zkG@%Liqq(GX~`It-`y9G*DL(&7BJXn_r$gESz=sD7~qobdkT}b?2jwh zY%w)2pShwJ25X3GtDAZxV(DX?RlaIrtj|WF5+$wF#W%%pkfR*jt~*f6dA`aLO4o%5 z$z$|1k+##6$c^I2TP(8Kt=gV!zgU3t6$*T5f#vQ?1>Ug0TKAp;TC;($XiMuoaAjKP zI!Fu+&jWHyO%nbZx4rm3vkC@mDq#A+MWgd#(hW?78E6il;~H{IR8K?>9;%khSNy=; zyUHqH^-nT-MjS?#Bq~AFPq7?_aVgc{MXLdBLaPDyAF!_y3(KwK{u_sN0o$x;idk!ljb;BLv4QE<*(KQu1@biUQwrGzp`AH3&1+syMn7pq{ z+WK-trl3Z=|F$xbwLqTC73qRMDbFfeJ`B@gvAs1DxnRUZ8yori*rTz2B>16QtfAR( zkidP7PEd<4Gcj1=W&;&nE+%Rt_)K)8<|@x9_7k|MNyNVt@G`{up8{TnX1aO>{v-CP zJ+4y79}$@2E>Pfb!!XC4NS}W79Av2m)nCTY<3SId+`E)e{BXTu;g{%CDVaO zHO`saQi9E^HH|np#117kw#XMFrh=>pkl2GjULIn1BGyj-nSba~Z-ZBn7_87v1_$?a zO0_sT1ofG)rlAG>N&Om$!#mF_*n?(1+G3<~!3GMLmUrzCEt1c)gC&v4Q2&6F9$@U&;l<~%aOgEXZNGA7}Rx+7fC7pP2Py;J)=G%rR z_evSjJsG|XSJEPRzAlO$M(s&E+$PXt0L>?cPT4T3MZ1k5L(+=AWemF=4Qe^xTw-&j z^Z_hIh`A3yqzx95Ro!D6phxPF?^Nam6_ZPUakQY9xG_M-uAu3b#KeUODW<}wmL!jA z&AIDLV`{2w#;w6QmHor)f+g<7ZK&G{0^IxJo>1Uz0#>gAo&j8>bs!|vTp~auVZFDFAq>Frmv%GB|K~VFy`4r`gFd7yy*qsADSzA&~ldEEa3X| zfl1#SQIIFvTw@D;VF%S@V>+q^hK>p>UWQ1jEvz|PI5?v9%=RhT-_zD)Hc9DD6r?sa z1<;Jj|8rx$iQepG-@7b&j(eio&jQkC4-?(b{y>v5w=GN~BrKQi5F^}%1^ zX=681Pu&S+q&>RBTLIZN1A`!U;qXF%;|Md~HcXeA9_`BG zdRr=7uw$X|nVj+A!d=J;RK1RkLqx9hp+1g~41ATL2g;>GGP1SXyba~7bxYE$B zw!Y^o(Bi6QMZA)B41j|)Jn8-24tYXy>1WZit|W%XQZSx9sY6D&^zb7|{0Y4$?E|t# zOFT}_P+LcuruEEAP0pC2ROVOEd!O~Was>7--?gHRT%ddv z*_U1L{pm)Qwuxdg?dvj>1y#!bQ2l0jj}})q3+=B zGCkqN_wCMVVlYY-uwKUn{dKclM>-I>vx5T{I+%O<4|joja^0(5goauB<8AaLc2E%h zj`AGaNs^^x}B3?(HFR;L)%jNW<`@#(FpaTP!7a5q5VQrdZu_?|= za~xHsB0$?ho@U2jVpFrIkhk_n>knKV>N*DOEdS9aUc9~p%(L!E`_eXwNqp(1A4r|Y zjcnGx>A#xTmgte_x?B7ODY^3TXav);@r^6i399R!NyePX8Y!L{6{{XUwX`z~Hnps^ zq7vNC*1{g^6x!hpDIU9Tug3?$@SO zD%+Q4b2n}!`eynC>C+?xs;Xa+{-spn?Yr@DBqb}-pOfH4Eo`Fbs8nxhHoJz9iXQac zY>jm@DN@sl-zXT53?dT!j+RR*FA7HGd$zAr@&L@8A-4_F$~c@Ak@!LCKGfvaO03DL}Ud{i^zmWLOOo^7eF-!3a~P)-<-(boV4G$@8*o5chPR z5q`}07Ja4zPGX>TIJiFWS2Fp#4JM_>Y6S+mMtKWpTN9mUJ@M;KRw>gE@$jQOn^@~Z#k+LTT;7A}Q-C|%+slMQR`%gBL{$MqExe3PLbq!S zu|WdCZY^&}ZV{*Lq`7Js{xI~g_PF?i0bDO*#b%%1I9B^6#(&+XXY$`Nck}VKa|{Z{ zS79Dwr?$DWL-frrT74RzaQsfzT%uaOljZiLUFzGl6T15Ji|?^^rt5~DARNCrRl21T z&G=s}!QfU9)yzMY#qF)Tu2;h`Mal`>`_p?_50;Q?oZiy4s;p)J|IHVfc&x36W0BN0g-tq)x$Q(zfKOtrM`W^Wn{T&eF`foYkJiPtY_ zRk?Iuo=&EwuklmxzSjB#VZ>8&<0@}hP-WeRQ&`S#e^@z8HSPL+Z~5lZ1FEsAwoyJh zAKJ^Sng&~UFZvgyV~6q*{1x_PcIZhPLS6szaf-VBgJDRPRX zbRGi?BVC0SMPjW3o6IDwp_$FF5&@Wzt^85xM_INHgY4B?=7;!gNcYz0RR%=QhtHdY<@X{FAGUnv=K0?M`J3n`&|+P&No^ z6fJQFZI&+y!;DLov9N89A!TF7;Ag}jWY3;1Ho_Rc^K7MSALF*8#M;R85A8o*tCNk$*kf25E7Wm=p|u6>J(#Bq49~K* zRv0*_wa8f_HogLUVB-kg4+tvvTS~mNQi!AJjG-_8iJ=$LA@gwJs;?!x9p{^XU|U$> z?u9@mE49QKE#9{Hn@NMJHioMCJqX8((^#y<2(d2bgPU)PKe=9au#{okJWfp30o%3i z4^YQLX^9O1>7IZEi?zgkX+?VxWB^NLW>`J6abk@LGLi-A1>|ujS%tin0M1mVSmXt6 zzAoofxLL)^;{yf`YBNm5&5yyD2qkrw)IPDPxMZbZm&M$C8d6(|XLsXz3@2{BX7pju z(u6DBn3y=T?-DWnZ_=MmvoDRs7xHDj-gdN|Yg|ze=E}H9D6n9dEn|PxLWX-?zWRvt zTI~3MShm=BPlwy|1g}BJZS$hUnz%sy9hFgKI#D{tE0*>uJXR%teL8bzqtrH-Y-ywl zvzP^USni{N{$!n<&^uUXTlHzvDwTbbI=P?PN;d}UY>wr^oZ~5iV@Zfjv$mn5`>ROZ zewO673C)o|1j0*f{B_+7ov4763L~X!fC!VSuS3;Fsme?fPGZ8(l&5K5F_EVv%D>`0 z&#GLzz4zBox=kfNF(=%uM|e%nJ$2q1+F7$P_ElEU4rA%6FM7YuGG(z6;%ojUhQ?9b zxhwx3G`h8ic{-9Nc_}nW1%PxEcv)*=CYh)&q}|(RTZ9Tjk@uH80YxqBvm1esB;KGd znJazK7A{)YXXgcob38=xuBy2p68mg$fH=e;u+NhEMJF++>U|8UrZYKisx3e@W-5g$!6Kx6S()mR04b0@ZPqRDdp=c94>)e4fo1mp@pyxq|P zZoCJ}!wrk!%pQ2&T(j(9@Our+;w)AcbHc7|7_K5{NLnO*53Ru1z8wRN_R+2@wasi> zThL@DZ;vlCkYtlrd-z$fPfM??8J`Ps3@{LmmJYH_<--g_16F}~n?EFesFLf;XTHVG zuE)-^on5KSm-ubDT4^>v;__}As!eL({<5WboVyZ?rt)RA8>k%|s=cZis@@p8_RYOy zb&ra$qC{#%l7W}S1DB2sT(ay5yX={UynXHdy#eJu0!?xIFL9F;HI69kGbpXHacdlr zHa*48x2u~xytYP-68tVPJm;|#)=guUpZ8)E?&8x~MA)aATO>Ik&)q39FUtH)R$T#t z8~bPw>#c#Whg5NyY5%AfMqEtaOi++}gmMp6Zq4(*g%RW?qvFX;M&cvAYGCsBSDB{# zk4#_xu1wy3WvC-7MU7BVY^HOU`dpNvD2nP3HCl#RNu%HM1*oR}hj~z-2t|(qH4$VX zakmzUlMRLXIds3jKja z6DEX-`re|*6H);Xojo}fzOCY^?5G0RJp=404+{=I%^g7eU~LBVO5k`4CR*g&fT>W? zoEI=v36sX_|1lpvMsW4zx6MRhT*9)m{IR3tkL4s1z1tO?X=BCsOHG;KP_sLsZKPp| zMq0$(kJgUICg` zwQP(M<)oX9R5i^#ZOx)K8{)2@ZcG=Cf;?t~_r9TtVd;kq_I@tB3_^oANUrl@c|25T zq4MF?0uZl3Ugh3xMKGa~58rNlgfbH{tn{@p`UdG*FO^9dLaz-UVw6U;RCWQ%NbKvo z9ulpcD{dEk4RaKVpl_={32IYPzoh~ZTb@k*V{iCPnu8`}`7W9alcDb!EeRF+@I%F? z461HA6^v38C_1Jn@Y50t_=(;<#7J?LL?=*msE~>Y-7Y^IMQ5{E?apl@nezr;Z4m#8 zM2C1Cq@FoajSeNn{N~j`bPyz|;9}PuL@Jo^lnD{O)>X8+6ps3Iz6DnzQK@WC>-V48 zC7`N@Tl)vw#H=0*yuI@gF;dONQBUe&`5Szbw-5uG$kP~2$Nuo{w`@;qrRDA>VeLX7 zAv}5rWE#3JR?;g_q3(620*?`3Y=7B^nD35K+^q(^z)#W!5F?68q8g&zvoxZk!f>S;^-4)(M)6*F0TVv1E7*^n6(_ya{#Ubb?Bd=9yE080(Sp1 z;LQeTqoJ3P^*jtKQOHR&&XP4K7P}?fr~om=(G{MAV)tPja}_au>@0STapX;w7mr;s zhkl48YaTyTzwtwh@hs26q;vT`rA?BOz|+VceMZx{5&Bxl7oAuQTjCBj)O@9irUNX+GEIS}D6+6?(GBPr8ZJ}WcgPhjp?R`3AkVMwOA^sNnd)x(|x6{1qN!H410=^NLOdpPFzw+%kPn!8a71pWci%gqqsyO>|OZ)WLv&(;VoK z1ZWeY-9Ex{gTRrn64y(?^A)6vlqv8ZL2ae4tNHSpe?wSmT!^g$_J^8vV4d#^#%?!M zvR*KdX3i&8Wd!fi==FVP3U(as0Jqsn^+SIu`2ZyYZH~ASxAGSiD9>)T(K=fy^S2nn zeX>qs1u1aDrd#@rd=l3r} z&R@XJK;u>NEy^n5v8PbZ8|^-W=q_{`MC8GO(4h~nQO+QX>%{pf%VZ=|K(vU- zpL@1{C^C*Nw9b+6sGTXHqHnj(Z*V>F>Da7qk32MC45=^pjy_=mII!Xn*y1XfWG2TD-t;oC*%>{~+-F-$eaK@$L+o(5> z^{X`jz2pGnqZ;URNTIR=I@-+Z^`~0!kgu8c_6pC*BVu1TI4iJf?ig zS3+Z3KXdQHwYYC)kHuzT7_*+~Jsxfg5L;z@zp!2!gjJ%SFYB5nm4!&PYPw0W){Y=) z%DBVeX>{#x_^y)sKl1NU4l3A#IjGLAMbWF2dcv{AHq2FE)^n7sqiRp(WPbVH%whOP!9k4(jD0t%nOu{ zRjoI7-?aJ+)jrFH3xu)0e52=ahMaw@nt?6!{!zMurRPJ~_`%Y%_*Gr^LyMbJ z2FS&s65rSAOnvEmu8b}$k@WD?B=RP?S4S9cH>zt}n`d%(b!n~dFM_DmM!B%FU=lQj zSC`hhW_pU@Sf+g%0YZ&pBY{l84`Ybx0*LW~kdO-E=@{b70y=61A<+`VjWNU|)rwMx z)ZPTPD7Z1C#r^z@kgSx^Vw|uj(8Fll-*(OUy%CD-ZLw`btf*G3zenLCwHO#wTgCP@ z*vTPwvSN8Q2$7Iui}W(3XPclH)lj(=!WNiepaO{AB@w@H!vASrMpzCM1qI*<|8I{W zS{3lF1^+LOAw4_w8+A1;5lyg|!Uf(L}4szuP2_VVv?e#>g6fG7LZ=LpgR*1M|n!Q(Ffel+m=lbr?2F>ST@JC=x^ z!WV=?J85Y|NeD42P=VeGy0Il%P1Dn$dX;L3-Su>*v9L{e zA9Ny(67P||Fr=u%a71UO+k)1Vu%1YQUkh9vT6&3VCZp1%u(V%mv6R^n>V9>vgONm*DFS3<$F;7ByT#EsfG}l|*+rE;+<VWkh z0c(}9in*5c2e^aJ$lpC-o6|8Rd%TR(%=G&d!Sbio=c)Wkr#0#qsNM?93IVnY1Z_@A zUj;r6DP;;QB;W=u7aP5kuL;_yik`>Hkk+czp}EJ`s04qbR2_28HsEIWEP2GiCs>yp zG|rv_-C)f7tmCrSuKE_ieaQDrddEc3EaOw+g>J24i38;iDP3~~xWf*L_f1L~O@jT6H|bV_ z_nSJ(+|#7X4Wl8#4jw;)T>BN;AGND5SNaOzw!cFQj%9A%_acBVG9Z3jqa`plC8`V0 zbVn}r5;gz^tB;yk{RjP#Y~FYWcz@@TwS=y9vMn3^V4;bts@K7949JyVM!T!KOp&Z0 z`qpjg<0p(cX-5BY$CqRDTG#XD%GE}>w16pqQ3}QiK$Wc;ECnVgb~2o3>`U)PK^FZC zln3VBW^{Yz9bqz%er<;W=3PX3%slGHi-Fd!w7h+y31jKyU@c(YtxfqpySGF_j5NlH zgKjH$tM2(5ZD7ok59fS1Y6?ypwDdMj*pE6>CFmx-Rzftc4Oa){`NrBPJ1q8v*s8j_ zS}@P`!JHSfhUipYd`ZulLKX*x5Ky&-7(b3>GYM?kTMdy*Zvsk-o=bBDHk}8B@pg(| zeMh*9i(V2LJB;)eZ0Z3oy^Nipa&x7_z*@kjbHB^8=_5nff=$aw7Mm6nvgu&SrN#lV zX*565g`RbGCf zsS_D)vG9tq)pZXhjnar@ySr@KV_;HKe#`9~m6soJ9#=5dA&L*;T^r(ufGDtaBvH1B z!4?ur5RgH+1NC~=+ile@&2|x4E56x64EgSEC+3Aq$hIQJZ};9n(b5YLhU|eRjK$n< zy~MYv3{sOOR9z}_tU$c!-0Wtp1EPU3mDvlB0%g}#&65lU@TQnE4z~5cj?JaJV!~85 zTtUsMa$`$e4HQX~I?(=gOX+R#JOy@CXkBwV>+;!$N%9*QPWm<`G%kf!(^9c|=`pBU zY2)m2?JPD{=hsH3JMsX|YOtd4a@t0W7Q%uXPuq@3k&$y!(!>=mmT!Up9=az_#G+lnEQtmRcn4 zev*CJBJmwmikMM~$B5Dj-n!~+^*>wyg2*36w1DIFD0o=Vl8xn2)# zg!uMzR@3sa1X8zHO}ko6%db>SW-F`dTGs>lqmo(X-bY$?dW>IU{~YcmO(wHNb-oa} zg@%pgIf3G0O>gVVUt_k!uD6=M>8I$I_DXDb=+@W(FZ9I4akT_9Zv2?ot8Mi_b~1KL zT&07pi_y91FsaOs5(SM|Xz59S_~9;0Z>9)oeKN-7twoJ>7wjnqkIYch;dxe1qmCja zX@?3w`swDy7LA8Z&LxPWMS%^EYizXiWei9aINta9eYF4aU&0h*-(RYTo zZ6N2OJkxfM0^s9y;iK6~ko#Gebu0G5VqYdTP{V|~Z-^X9*1xPKQ+#^CP(zdXy1CzyXTQpL|CjKZ@`7f)_LJJrI; zrk9fm7Vb37?{XVuUhffX(dA9!N1(T9e>z_UdZjdo*gB%W`qlRio5ZCa-PjV=`DpQ!W?Sp6u-|7cM@eZ0~i z`rqlbO23G7@8JtK0`ERaCoeSoR1V`{#`Xl|?+{+xTME^ux0tL*c|y-vMAgHA41YRd zY{IY-c$Lbm#m!%p>u={T=x0r8-Z9_0k_EWAzl`_UEqAR5M_&TV)LxGXKmSja)b3yU z;Thr7-HoCz^;~1+FnkQx6J1?Dptf~168~^qe@)J&ehHq_jA^ahzZE!;fG-El8J)%o zLLPpJ8!wPlfWJ(EvlDpr;n6UH^BvK)@w1e`?!Hyv0!1H(B&m|G z?cmKSlwfwW+`%@s+s*pYx(tol@bkszv<+of7|DEirIt{Wj3|B#+4LKn+M3AuHF-bG zP*Jl8cy&ab?uzw%<#D7Zo6jyIPF_*00a=Z^2M{U9`23h%$5&sz<^6WmfU6mZgZyY# z*kikc^|4l}q%MS4|Ajk0Kz#I_32!U+jRLa?1bN97ebqWI=jta?aDAf#kRRpG6|E?Z z(}UG$ouyrpesw{9^%Ldq9nHtx?eiC=*M{^S(S4+g|3`|R?V(4M9;m>{A<$2O9A0q= z#?A`po_BfCwoyQj78g3Yy&%0#wLy!X@8DK_X4^eFL{nM6vY5qj%!_f%2XV~#am=f6 z%!R}x>(MGn#Ou$Oqz8vZ^G(3(YKw|!9|(8j2u2(56&s16)wl4i{9v!Y zZZNsLk)7RuJvyrK-a_=N)~k9W-I};-oW5TYAIo2Ga7fTZ6All*@##*MDWcc-FTb83 z`qeo4^Yq^d{~GEDlFPp;EhC>DXg~1GeW2mFPod!)Z5n!9UvIJ4ri~;ss2;}8nT$^AzV$i!A%C3CXU!d-k9QfPq` zM8Cju+sEvF`xr8sq)AukYevwdixjv+;x-}NNi_Qa?!TXXtP`YW?(JAzTPZ=ML#l1h;wHOMSJ7iO!s#MZV4p~Q6LH> z9iYH_1fa=JnEDXo+0&Vlljx2_L$6O+X}M>*BIUl>+6v~omGk4Q8i_o*uah{vTOqE_ zO=7lD0den)PJwdwG>Op@;^U+GYSh=g`qEWte$w{28R|Obi^tgx2@jFjNhwq%eYHS^ z-!FTJQicmkvNy6IksGBgab%o0svz-rv!v4P+xhdsM(Zd6y3dgiNe0JA?EZ+F&SraR zudT+Cn$9ZFK%;FSTn+NcM`BJ*w^23cq>vCeZ!uTeE)KH zT5gyey$kQ%1cEl+A{h;}lp5)UFNFLM9xQngvelbs(wn`lbA`wa17a%Mjd3oFyT!DC z+fT?=8=2OHt|YtLNrY*pZC&Ut3bhwfkbW^o32L0i+tNOq{r7u)1(qhl16 z4Z;~6CDPyfkkY=k*0j@5?PyM)F7e&nBny5BY6mQbkhLr7#A*=qycEV}B1##k_gns1 z2dS+x#KmY#gdrMCZZ}*gbq|mzJVb$mLSSD7?83*GM;R1I(y6!>urYejHs{+a(~9vwWOHxP!Zwgt(~{GbELXLY~YzNmZ%L8zJcol58PWZr{t93>ymc?f9WoF^sM| zF?Ud(|NQ+>MAU&8_YOgOrOLggFI_t8UeK4u069G$S70&qcNb7=cejGOn$aqPuP5jg zQJL+$Vrh!2ZIx29HCb-xm=II5pV$Gbx83l-lT^ZiA1Qcq8y?>fw)D{j~RG+cZycW zd_Sc3jmtU2Dw1H|-P{Q2?^9MKe@Zqkz`ZdSt6G`o5FRi=}ggWqN<~uK4LsO{+y}cZ2k>#lX@q&jq7`i-!lz}b7+z=rC z8NBGd6LRwQwH=D^kImVai?;nadO*&0;a_M}q3 z>)=IvGsY#T(i19&OC3EdR)q};UjvR_XnkeJ=x7ip`oiwXXv2;MB3s^*(fE@_#CHF@ zRzK;Rt`GNQWC>SJ^`=Vc%Str2%se$QEjHR;lEDMIT$0f~aPLq4tIRt<2hJ{H{@b9Q zf!p?(oBVuzMp|Q{zqGDklhR`yfVz6dRDjprq?93!#9UDyksT|zXd?z6@T=x%;UH=gLlqw`rK~t>Dy!`SS`Ot$5<`B=ZR+V>Mi25 zn99+pHPSR-)B$Us`@L62dstGSj}9_YgtD)%P(4Po7<=UhAl;8LuoCPA8Z>%HHVo;b zm0v`xlb#LfWB*6GUNv-^ysaVl6g(FVqK}#7!l=K7@gLt(0^K>%jmPnL={_SmS_MGp zANWmXDe8Tq;M4bd0u9@(hW2E6rpWbtmtp^5{!{13-WGhzK_woiV5}m7RxUrW4Nhh> z1rp3D|B4dniN3x8Gu>(xk2PB&N3)HXjE@tEluvgOzpmpp-g+v~b%_(GIF#2HKpaLK zqdT4+!Lz4YQo97w=}oqz_5{zcrLxOd({cx!Wa?3$>wvZK#61VojI3B0ZwTVBq_*5m z3aLzuz4IZ$AstrJmb*(ss^0;&NF*8=Yv)ROWRp3g*&FXi>L$D6T|;ka@*YRPS7R zijselh!}-oc_L$?l0MmrcYA~)D~#f#NZWA{bHR!ntZX)@T@rZd$;tEBo|Iw_21h@K z`U8)C6xNe;?xmTKkt;8SyRd&T>m7!5Sc!Q1hZ_tTvFW`naZrB^9+$8}-oy0@WNIs* zC@o869-yY4KYC}+7RTLD^q83IhrqQKBi|<^s;@jEc0zWUxc`US4LS6yUbaxjxK-x) zmD@}kI(k%%`;MJjn{nB%>Mb5`DFkX@D%5s;Iu#n7h-EGMQh|TVEnuWjiJ@Kd<6dt` zMl6AX2qL0;sS+WSmg{X6T_+PHzYujNjjIc@WaC^l+iEucGA-K)+BeiU*3#wo!EAVb zftDvI%QuRoNbA_*z}(Pt%B*(kxhk@ZHuwV}kx2gsh`4!nBbQ#Gh5_)#vGBTKD)Tu- zSrcfMQRd!#(+jdN(jKYIoun5wh6g(cx+U*rYL;C9p}l>h+)Z0qif+`IflW%jg>{oeb*N z%mZQ8?TX%L;$fG52NkC>wM4)<^E9}7-^hz)MY7h{ zZ6jdg%89s&f}Agg#0ta6!c7@8CGQ?t7qfSd9tMu7x7K2`Gvg$Jet$(Yp8e))@GnU3 zR%lyAZymnx>&Ek@gV?{NCd=rx3N0Y@?$c;f1+63Xb=?sz>q$Roz6oU|wz0xGx9fnd zq%3xoQ4h_H{aZ|B-k#?zrpuUAdvZ?U%RGGl(#*7fx>FS7JN={6m&x~PpSDUdnWYNg zJh)$-1D9YOms%|*js_Wj62B%eYMq`6kOG#)JX8BYfw3k~Ixg9H^a=f~mu z858tUBI7VovEbScM0C|Xc-pnasaBa|$$g>Jdr{cvDvee*v!JyX1R?R_CFR|qA;P-T zSMqGc6!>Z2q1Jc75-;7)rRpu-=W}lXf^cMYH$>P~h+J|jnweYf@sQlzT0&kgobWYd~x-RYHW#1MES&a@ZgbOZ;?LXTlOijI{jIUqPM_>MjOOr}7GKn*a?< zd9v?OIm7*oGFi*d-l9ls<+~cOm9HZZ=35QB!CKmfd4b46*C|MganR5nUF)D2RjP0l zSQg^?7RXy`Bcf%W-PGtz)7WCM%2=P+@no0!u;W z=MnNP)+*{Pk6$$&8NCLhd^*dJ>@CrleJoJc=ja?<5<^t9(s0&_Q+c@EfXjn?Z6Fl| z>6^Dm_tmJNURnbUMJ;bAMDZ#pCNlDqUgkdVNoCPAUmv5WWeYB9=%Z$$RjmzuBS!}V z&p37$FP!)oTDu^hn%&QRCB1Q8^|az>C2mx*_S54i_36zoS10dl{IHUuXGvGP^o6$Q z8o6h8;Y&!YJ4SbEU1n<~-j4IxZN`_M;OQQWgcw9Gp{HdHqA^y$OgDeJZ-U-YI~?zB z7D1UDeWQ24``3u}C}c&nv(-eS04>zpJ^zv?)2qsuUQtm&Hfkv~D21qALP1sTTNBL6&in~wi>k1{o5&kV|h*@BLTr`+iM2n0U za4&#KAtY7GgG~p_H}kuij^D5WGh~g3b=&Y^xT9xD&`N4w6@4>Bi@q|qqfrVkqs7EE z3aVCGg2*E)dph$f5vpAhV*j5^uK`lk^!!rv1&0IgCe#L=OFjwwQM101c= zm5S=S_YmW|?72k$rKH51ii*MF&XE9<)elw^rR3xG^|_6Pgk915w2qr<@<=8zW9uT7 zZ$pgPlUcoAmSiYE68%$o&qgc6Gl(P19+Bp?@v9O>gHbgx|Fd-}*YgEmB~hqy`ld32 z$)}=A)=8@1`OuK`%hZ;+*p;hCij?b9l9J_^rJMP&)=oGd4gqGxWWQz&OR$H&2!}rm zfl38#v>gcAU$`4V@w!K}V-nXTiQA31&@4UNhDr1R3FuFO_#V+atUVVn%=f#f7tND2 zptVADloqn4kuNHxb=_0?dIJqk=lDXZXR;#n3ZCz5!dZqeN*be46k>zvbBy&f5kQQ5 zxLM}TB9kfk$0X2XF(n~3jJZ@SID~ZBE`6h$z-$t)7NY(n`1)@`0_ZGSO;qN#iqn*{ zEE*JYN*4r0@TPk6c@kHqHuGDHSwO18Wi@&yw6{IiIBwqq9NldIlUfgChu5 z_6=Grx(EiIHqqrsH?*e_Rv|r5@o#)-+cyHAQ1Yd;kQHS$N=lEmvkqF+1^9Te_>M6) zF0e6f2>l@XN@(j#7mg;@zp^&#JU&wcXKT)pEBfXOLiOoeE{t<^RIY!M%l+^?U;Whj zyruVz0(v7&O#g_0$f<}@2#lMlv|a$Z>H4bB*VFnsLadUK^gRfB+WN28^TZeK;d&(` z`qdl~hZGn$+6<^#50ShG`JhJ_gE@#l|Y9JREGPn;xLRI9-aXEf84zZ ze3r%a#|vmQYVnN=ifhvvm8#g#HVPt|sA$rLid#iR#k%2I)KJm7pd>XhMybV$JGL&U zxKZ38MqCg#aH-$|qNvOxYTR)ta=*VbGtc|HA-J^me?Rx;^MUu7nKNh3oH^&r%$d!G zaduDpX2NO%ZP|F6Vrc}toUuDX1Eq*EISZbf6wPL=WAd{_Rw7J>`cbyT`dmbSUo*PN zcBXS!5BE26swGQ|(Gsy=(ZGG$j>>IY)a`GyB?WfLUEY=(I6b)^d*z3j`vx+6Z+;ww zxMS}D+z8a)QeAWzZds%u~?)C)}?OxwamUaf>l=L*fHqd)({5baVsO)=0 zVST%%>687mzC9jkDp=ocBtoO9`ZBFZ>rdZ9!nb=vJf+A+9W+tOhzP}=s+OYT+98Z7 z;xJ;Xdsk|7LTET4^RvGRI*m<_pRAU#J7h1Mts=&J#e8arO%lIiroN57bjoKVxikD_ z<<&36?Z6nSjq18}*@u;woV8{5A;+2B!oofGCp5p!9V9Z@4JG;AuvqaX>-M4bgH`Fh~iv> z-RCRwL()-7ve-$=-`u-_*WkiBdVN_5`Sqtp>y`N~kKZh+lhii^T{UfFOOIMN_m7p2 z1g-0Wn8hWHI?j3bVSILnqWqP>CviCZLCj|tcdAv%7aVVZIYo!dn|o}-B8za_t?0$J zN}B+!>mQ0`$BxGVU^j4w`TbMb#68V1Qt>&9Gqp`P#TU7Gs6ouz_yGj?Tex3@I-Ba( zHXUAqs%c7-=L0RQV9!ur^%4KNUCtdHbq*(VTRf`-cZV_0vu|SN``*m=g!DUZ&qkL= zC{h^4rm30DqyYGZJwAqc$k1uY4h)VlzB>fT>=a3`Qo{UrCbE}hp#q=tp<{4u`o(rt zy3cMy#`^iHT&68M8`v(Qk!!2TE{D|_eW4PmeByw~*};X8Y

UBY{5t+kK20X(&uqiHq879 z4Rtvx$w&_?bQm{y$O3GPe*G3dD}D_YwQGO=|_PQ9D};Xy{92-A;E>3FvOLr>Qt!cHxhICVOI+?O$tMkL+39 z%EBlW5p^|V9_?yf&Js{$P7(Y|l3q}QUA0Vn@pIj})U8q7pTctuMLz|GO~9Lrdt*Ld zIoGfT98Q+5w>B=MLs3=uJ&PsMShsy>sPD^a81DqDz(Rdc-Z)qc$^ z?r5Nsr^+>281Y@hRPSSjN3pW;7zF(1<8K!(f67yN-ynK20T0ULPOQv4# zotq_?9!+!;Z$e9{#_aLUyJzzyCjmogIK;ceJ$g$PiC>lfs#HjdHhfkihyfqZ){@vG zs?Ts1ylVP!V-Pfl6@i(o@Nfe2ZGdw1Qx(84jHPw_%OqR=L}Fx~V+d{{GteRY=Bv~Ra~7OKhfj22AA}$ySXV% zYnxuIg6`aK`+6=m?n6?Lgx_I$veX&M>ZVBr+g7a2bi9%c=<~&-VTu{7 zFP49#lYHU)42n&lI{5@gRossgNp#P{Z!;C=YY8TF%tK(dABRqf$+y5ZCD$~$>ZX_I zJjLL_z*V=53%^sm$zt$*toA&ehU>un>Qnal9Z+?kxI4vAeAV^rURdN;m6sMBEKfwk zDs)RqtItrXZQjlMb8~dZNJ-buu1j*?iW#@-{!8e7Gf# zD;rHMvjhYPrk1>JX$aVx(;0$YELnF&X2{Ry{-q^r&6;zDSm#xwm|=Zym0W$9lC^;7 z58nE)^~ssS(&fN++fN3vhBN53OG4y6il$T%6E2o3Rsdf3Zzn< z^8kM(5EuBze>&yyVFoeI)U|Uu_5>m`;2Ufi@bzRn;5voKGCgWtdUWxErhLLFApTq@hPSJ*Oe7T0dWA)Bg9HYKb1Qy>P<})olMjoVj{|vVm z_m5-B8dk;6)JYK)Ht0!LaT678i(&2qR(Mtnx4XuI@GLh< z;rTIrjvKDNhG)5Ig>`Du=$qsED!ic&yR8-YIe}Q8C9bEU1yRl)Y71c6I61r4F-V5VfhVs%DxRxS;RA| zQ##(f24A-?+Q^XVS@~bF=mQ`1uF4dP(Pf2|ZiUh+@vsQ=aLd<3v_EHg zlMN7X-j`}FI_VL0k(tp+sWQgViH-Cg_6|&j0L4nvSfIpz{C|G5lAOVsL*9O=YNe_g z9O*ODkIt;le(=VKEvJ)%FkVYeXWb~;YwU#^awR5Y6O$op>Qd^NN=W_@7RIuX zLnaJb8CU@1{~uV=*QOA=<*UsaiEUdcrGl(eW#$igc|k;y`P!~?w@ZF&D?H2GkFJ21 zWt2pt3jL#@ZuA}^SFT~(%oRrbGBaKrWb%|Ak+)7Hv4H0GQaOi3J6q# z`db7~$9d-GGrGxAFQQ5X#4r8j>OTOFR#+9Q%KO--4ZC*P`@MYmmbpPG!1fKWwx87Q z0&4yfb^6S0&mp6@P|}rpgh#rEuq_e4kvTwrgkyp`+kT3tB)MD-sFxt|*s&NCd*9k zm$-E=MejF#N1Evy5+4G0W` zy5Ex;KrXP5ZDvli?Y93WTLa4Z2L&kbUzd4D+0HDKNp_|oR$O4U%8Noj#iMT-u*Z|y z`MC-@IK_nBJ@bXPH8O5-+3yM5YoZLQUTCc!%Z&CSes*&5D_O?!!RyGTg6bLtG9yihJ-s1(EF!mOvZ)Ur3cO zAA!xWn{n;t+{IQFLgM#S=+fJ8UUg?XGQd>^QX17ze=I~3?VF{*Xe^;Yv{SIqXcQX` z4hk0-2o|jrrUw%d2gO$vr`ne-#Ew|9}>b05Ld}d21Lp6c!4E}jrghF%(>=yw72VpwE6PQZ1faqSC&nnw%O9c#24y@SZV4; znk%&eGWI@JK)Um>2fJl*4JT^iiGqy~zs$+W-GJs{d_>A+Q+Uptj({?GOmJ#qBsI%M zYE@{i;f!v|*!y$vVjpYzaSKkaL94m$1S%rq);LTx>o+%dVeMvoG|iX)eYP=7C*l2q zP83tlk@DC#HvzsWnc%)mCSle6oRR#(y&`!Mb#^CRA~poO+m{$g&Jy?XI4_Jwcjyky zzD}l1g6Uo2rUWSUB2ffaHv_B-sE(8vWO@j7|M->iZl_VHTk9U9F#F z^gd)o26~TA*Y>swlw@_X8>xZRWo|VC@N4up;1c(OZ^z3s68+7-losShK8HKh#2NMI z+*7il3Ct90U7eF^cLOi>OjVZ|7jf>TQq}!a%T?QdO4e+h$$kmWXVl#zvo%5i+GbOl ztg2~-1mbr{o$b`Ee_oa%!|Z|hR~m9^fezB=FCSKSGf~x$H0J8-nTks$M<7w!@8Elb zNMqkEgr@f)P?1aqQuDE!?#86GRCL=Hd;|i9nr1YJY(~{d_T55U6*vAb}T)I9<@_B#c&r#fUkxmDPDK9g+2A- z=i9Z-*ICEn*7~cLf7EAVs_M5;EjYpxwN@Yy+455;x$vcGRNd1A(sM6T!9*xdQLO02V#bsXp1pq3laiSHmDd_&R;2XRP4gnoNJY#TB*f>=T~r<= zV_Ni)K-lSy_%In@g#0 zw!9+rZ&Kp^N3aNzq1H~a9+9rTq!^k{t zi|Qv=V-|v6nYR91lGOB?82g{>`EVala@-GaD8m8?Ieq8MMl(*i`tCIOubanxBIGq= z9@8d}nlWFmJd*b|xX%YDdoV|gzWa4whe?{{*(A8(d9@R88_@>IV-_%1hSw9kX|@O* z^(`1BuOfx^D5_Gcv+8E`#Dq7iLV0~LH96PpA~|&85Q8`G|E5?yj0Z21_gME(3-@e- z1-F}((DBpLOa}d7<1;nPY%XfOO(Rj?Pu9$Ml+_RI4XhHX}5IOjWQIX8${y zVmgd4t&RpsJN6GBFD%4 zd4V3+S8BY|{VzM`>CPV?xb9XP4DGk864Mzzm3jUGkZ%j)Om$Od&y`jJHp`qy$&o51 zm`P*~qp7~fkBhP%z1C$x*sQ`b6_G7TZ*|ShMH@kgzcr+#;?mtbHII`?W}{7+jl~0? zD$01rK1DV~klDUP>=l+|216ZnuJ5qbTr%{+pQ+xzwf`NRnxVj40vhVxufP-waQ~wM ziv?z9FM1pR6XE5+o$>_cc{yZ~@p83Yl$fRZJS<+qg|(a|a|h9AHXYH^2-QuGL5vJe zBSgBhjky|BKaQEP#AorGF|ul!9`>VvSp7&)o{_DKA4TN6&R6CpjLW(zJOt#Keg~vIdP#ozo82an)feF(+trnQAQR znl(vrd}AXX<7#ANTBYPF8!*(HTA98VBJjGOM-Wk_2qMPha1bG}DyG=&W;9|~l0|le zZ4t4<+_uCby2Fy4pxPXw?iFN63X`3Q{+yY8q3>uvR6-+_i?nN9o5ZTafpg75`-a4m zZA<$U@8Mt_54Xlg?l< zc;J&%{pdTS?P*(5nX!aTw8W)S0)&Dw79W%}I0$LlUz1QMP^px(z9$4xc+wU&iA-8U zxA*)xNZm&|F%$v6O&8n2`d{hdt`sfj6EU{RGOJ zfk;~NZ|gqhxJyP;g@eUC>eHye6;_|u+@BOEB_LW)v_R;)4^yDVI!xcTgPIm&ZPj5? zxWuCreUj%Qwn!v6^rYUT1l^5lWG2#7HCgzQN(GG}G#i$0T`vZK#gR9$fRt&AC!o#A zMND7Rm)w3f6wiSCyur{8JGdXkG_tbwYl$2piDx12viBw)STA}fL!a9kE?}J8KEi#~ zi{Os6O@4(>B-fyYQCD%U? zP873F6o}eAXNKKZ(dZB6%~xUcT!nVQ^8Z5iWJEp+u0IF)&MDN-Ere zP{2AG0?)oomC`CKM|!eoj&CKKtB3N;bAT^aXZ!KiVp%%2du+bgzCjXo-zPq?5oueJ zw8b@g4*ze(7i`}8*{k@emIpUqHt7RL!E`!g6k4IUX zib_c?BxhJ38D|w(5$#5EN`_r>0jaS@Ru7}Qpfbrll00a5>I`Vu(bka=Q?Ko0kBmX{ z)G+FlqZp@{>Uig-JKTFL`+<|xWrtZ z?e1XGQWww&?zYrWrdLQ@K>KU3c(2J;evK^I1P7e^1ti9zzJg2Y0B*A!JV{!gF1Ncc zPhGw$zcctHZoZ}Yr3$;Wl|JwZ;!=|4&Gsa%gnM5d_|}K6Dr_c&qQ37uz?jP_!N@=^MSw zq*o`kvsL3b3Gj_&Lx4b#4Dk;_lUIuZnrt-ENdw6#6E6kpV^B=@u$dU?Ny3HFNZz&P`J&2sOZjF2^j+4v+pjp}sf5}9m;{|5%M z&=yHn9~{~v_VKZuk{`dRs(Brj+F&Z(Bs*#j;fK@oI;uDA60qW97FsQX3 zD`hw+WiF+FF3A(-{j%whwclbeM4o}qUy#DV1HX2$i^$B1^Ilw6k(ZoWyI-RId?P*Z z${Q(GzRXCO2M4yKLYxv(1tFP^!0HwWscCvhXKF^8B-3YD!<*%0A4oJ^ke{c-kIs@N z^UJ5FAs9$Rr@veB&$Ou-sF?meF1kf;m_w&rN|i9BTP3vtE;ZFkP~JL5DM-GU5jDdL zl!imGrLehPqEuQCFzv}S8{KNNttdp1L0OI!j;)i6C`q#dY+s47CNtipP(J}>ATYrQ zHU-P?<4!40hGNxH_RO6e*b}G&3wA5d8yo*eo$EMr3$yOXBo&bka`JY}?AP;;nSptT zCkW!iY`|6>*Jc*Wbe$+mBSB=GS!*U%+*Ds{?m?n#D@GIuZQfvs9y#}|7idaW)exn* z=ne{$MH_KKuxRpVEj&>cjj!)FJo#LF{eh<*53nF=RD9oTbusD=^tZ}yM;y! zoof9hHg_qSJ=VAH*yGf`b(_0Yfk{Rwu~Ub3vS%XckH&@SMM3IlRAL92Y{QT(RzP$V zX9kFAg(CFw&evpBY8i$Mod3JiTUI#}^yik`VTAu7mSBn;rl6+(E5@7_o?Q1Z>zVH@ zzBeXKA@I^gp3d zmev+Rqb)QdU}+?zW8Hn^8DH(_=!iM%CtMQpquhD(btOPck0X9TJq};_=IXAqcUra} z>|1FW_!Vo@tLDzT&jX#}fzI_ny1g0u(eCPw@hqzEuk#Fc2*ppZT49+%>>VLa)Yi)Y zQ3_$LAU25d2(b;v6Cy-UK}cv+%9;^k7;gIrAqKRv1IHlV9UkadDFaa~2xcD!@kE5U z7P}sCbP)u*jtpXAgqV?`R3^&y&xnpcM~DgpOE~O8P7UyB2sMv8B0}_!5Hy9b3&IIv zmk2Qv9vTj4h!6_|Q5qr64iFjw7y?O->#Ah1jV4g7k!{}qrpVcfq>mBV3XhFIv+bZj zxWx!NR&H6Z3bEppz#Aj@5g{Q;+4z{SwGz0>LG8m7edPeB1{Hz{5$;LVA3}O5p(Qhr zwQ|RXR8`3K>?`jb%1f37`h~>43W#S)oEQ?tWrgqM*R67dSmFJGN7-^}kLG{Gi^TL6 z)JS**iy98nPOatvH(Vu+ecxTpus-@E)y;`RSn z5e~nATfvuGNumj{l~%9g1xB|qT2wX<*!^0D2x63`gMPVqk{{AzO!_K;KR22YwvB}b z0;jp4U1>6qvdSpIcC3$K=d^aLv$=jN9+Jw{|63NNf6+V(v=b1)GfXl(JND_V9IJ4( z+G@ar%}U`x5Z5(%hAgQP-+p9JoBaPh!MqkIwV$6ncX!J;*S{! z38FcfWC+fT5G!XO76?L(Du{z3#QhSzz%hxtDYM%#ChG};m^_l3TyR8x55Vn?YP>M5QOYzaXzk>F7wD6SLSIRt|)>qmK| z%cPzYg3@4u2U>9H%DfA>VHaP&6pkAyNu)=^>;ZFQZ!MojkRW?H7m&cqU@enujW; zp9z7Fa;$Ax9k11Jje&EAA4a}@^F}W8{Z{M6hIqixE&cr=*Uc1Ko=SxJ%#DrvA5QV- zJTj5k!W$bQyQ|0!4JN@!xZOv??`*tSm9XZZ_(V^e1$Z$wxyi?0#==dE7ylF1Fa>{1 zCSJ{y9OUDhGI+%EUJ-PlaAZw?Dx32wpwz(nT=I2BTCh6`oEQrQ`GF?kevmRq!~PA;LH;`wa(0_w~$6>n?`9XPTnA(I^&t6#zr3K3nd zfFwtWjUln1Lnd7kq4WtMU6P~BrHQRspqh+Si`J)#E!{s|FIBzVpvUi}6{LgPkyx7> z)E2b%qcL7j9|IV*eRxxTOhY#~T#WS^s#ooNpPuZ+vR68Kx54_%s%e)9HL_zc!IZ|2;qhZ|{1_5HhQ*J*@uN6?RL74>`^en2a|A=b%r!1$7EG@t`h-@K zoUopdKYukMJ%2TYv(n%n8ouk}Pm-CE2|f>2Z{L({u< zz&l$)xEKa$SqlgyBg=UL^2u)@OsO1mG|=QYpM0MsbK7$E3Y#!=;HIFKkb77bdH$&K zVIlXB&U05-vR+rn-0~B0bEVB{0r~LGk*n4($WTo2B3*9J9OK`DmtWZ@Plfxl$iZ-)E8gLO$(`DQp5 z;pS^KvbkdRWg4gQ-U{X{nyOVaa%wIbFj`t$vEFz=ugv^ezwM~^UT#zF*4-nk;!r8W0O!bs z1gxE?rPXj>V$}sD$5}zkH;U|~{*udB2~&6WQ{h>~?qVof1sY?Qoj0WW$aPOYe^x#* zYjqQI?Kc;_o=Lay!J8jhzL}Z$to2Ur^64khNU_jHD3djp)lbgw>HX4lbqOS@e@J%s ziTh@dXZw+)gmm3g(}`RPPJhYYf^V#mJ-}zF&L{L6-0VKDOL1U-oGd$6GhU0t*?iP}hR@Bc?%VK$oXG}Sf$muAS26>fH~;S{zMHIK+w(qh6WfnEHchGn0!iK>K@-qY{3(tZDw0vmJX5| z%2_pzR2pHpAdE^QxdW9y7^jw}a!;s?{;f{>dyPQ#m%MLh0v5Sbe@hp=C*tqs?XBAQ zt#z+)fF(>ipEVDL4S)u_n`(XanZ&hG!CZY#eIKkCTadt=T0L<4D&bTTTD4xGq4)XM zhwSPc_uNZm=-A*{95fN^;z7`Ik_8sI`3ijOhsif{p-fyB-(n3M4&Dry3*kk`_UsL< zZo>$P&kx)DpK-S2`!(?8s`U>}$B8=Q;!k*+<#KI)#oJ0%r$I?B#2SIbJDD_$^cNI^tKE>VdS#ZU$qx;I(I+%~#<2*0CTX?8nI=M`kZu!{<6O{1AH1Mb75u zA+x66?szaS$5pA(ZOxuxWVelcmzp3{_nV!&@S>dw=|cJ??#98=A{_&Zo=ob+6+!p{ z`Cw8t9^bM4*haf`0#q&oN7xcprq9gL}FD%+e0m(5#A+ z)s~62rEXMAOI#07`{-FcY>#m5oBI^3alvd2V&!2RO&#n;0pEDj-3YW9rEfT9cNMG$ zUcWTItY*vmhO|MXX?roZp)7H?W0%y7{#rdb1^fb8qO}P<+VszXs$l*Z-EpdCbH$7% z{07av$DSL=FR3!}d6&ce80pMtk3^Au9HnZT zhmp3f;dvxn<+UYwJZX!;X)J$`4@lYO4@OE6KZE*x!hH~f`n?<8cvwdslKz`_BZ(Wh z6`*-0rOk69Yz)nF1h**ZU57OU)Lh{j4QFyrs9vvKm6yro`)-p=^)B5dw$A>)$yaZ0 zVhgJGGKs3*FeT7zV`;Xy+7w&v?f|nRFewYxqRE)V1Qx7Il4rqExNR(Gr8UfN&w}Dp zC&-oK?;;$1q7?o!WXc`4sAG{jHXuE4a8oBi}#$24>4P z;FWBv!xcZnU3U?PiNO&WVlF1eAIa^tyAf_k#_S}r?Nx?wjoamrBz`OzAe=oGqj<4+ zdp_#)kbb14bNs{K_2#B~9yPwQW7=*f@~JV^5RF@g)o^4qS}PeEUhce9o(XN25#!rn zye5?X_~;xx;+J9|zTpvl2rqT|qZC^$fG2lW=957j)AVh0A_r9szlBhnv zRJvRgpStb9j(2Sq-a+Z@scaoJ5+mrVY9Cw5n&DO#EE?a9UwZXj#Wf8zLS%6;)pvB)@SljfHjaAHO`hz&G zYJ9*Z?n7m{F#P)hB4ODl{BoavuKtN%s8&6O+Bzn^6U2djv+X{)`eUU#cLaGa z7t`(~S%3Ogzo9?RY&emQJ=ljt563x4d6~FMQf!s1A(4iF3-J1=WOyd7PY{xRyVC7l zs%R+PyIK@7X6tKk;U@fkkLNBt8uL`$^o)S=`WbsZPm{kNZ#@9%1?w0%FLT{gyritV5cXPwx1XkybrluUG{ z|IM?%C$$#m&u#W{csjq*4CE!siA6-4?bM0Pj{{g=~Mt1vp0@cl% z3?fj|?@-vapHivb!)Retvmpjl(bL3=_oD^`P$?QYC2 zUJHl%=p8LD_e{3C1H67Wu1ku%rOiq;4ECf7sJX z8+ocSy`!FI7b95QlyMy<=Q^=Ng&eH=S|N<5N%abe=h$Me;WFW@0 z{clG}+Xs{Cez~j4*Ok3Ls`B0B5_b(v9c64IfI>gD#Qm9sIG(w_(lhYlMu6!NtJ4-7 z9bz=S_icKj#q{_C#UhG5ab`-u`cky2bUULhKLe*bt4`i$8G5Yx+P;;{(oWWuWX(lG zEo)!*W|i{P6}c-Q#jPjeC05&UL?ce1zI7gNGvq7ae`>_^K@<6y^7v62KZeJTZyWR{ zy}7K2mXq}u>_+V@%TQZ3IO)m6IX`hx*Imj4s^`+Ku%oEYk=puU&(8ji8mnU{^|L8s zO4T%x8r&l07A%S6m=OJ7w4dZJAX*!8LTR4`mkakm4uvwca^K~>4gVn*VNu2r^9lD7 zJ~M@W9k-H&0cTK%Hc5a5d?B_F_GS@nsq#&y|=n4y~8BV->B zAn>Xj7u=SB%Y`6-agp~h9C9W?29FQ{=@J@k>hLJKUyoZN?Mf?1$O|dHW#xVH{ znh?0hy~p?%2U2P^^J9e7@zw8W-P z6?>Rfp5gX1Jbt6#Si{rqo~%~`S4_p0>P=xvyUcUBaSt-fWZwh?vxMo~a?2d&1V+7duY0wXt9!9{c+d;u&CTS!4d~} z5AQ~_t;$Z)S98TAv-PjJ3o<}IB}uJWECTyitoD0x_-iIFSn(Hd0=PZ@oX7Yli&ww$ z(ip%|#lfD)=8AI9=4$#Ew@n8B>o$X_s4OGw$y zr)->0`AbOI*r%+XPdPQDbhDH&0Bo)pW@6FqhJ@sWl_Fzb9UZ860a_%eJBG9wq)AU< zT72duF<*itoZXpq#?$rLJ#u7aN7) zE&wW&;5VY$#Q+@#@)~h!6sEI{)%?0&K?3EIhST#QzVqQcSA(I^aMm9sLns&9QBF@`gn=mccaU@&4DpTZL$)NQ{EoQQ=XM8}D z+VLNx>zVvY)`HR*R@uP^^Mcmz+FvYS4DJ^8hY2uAwVQH8^S5b~Wb1t`aW+ zj==YK^au1$EfrOtbd63?UK&TrM;#J7EbhnqCstGVI3=q^l$2$8J}t*0C6&ShCNS-r1Tf`pGT|j8k?NV8`Q`yj+Au%JGLx|NjZIh_KdSYS zlv94bPwyZfUc>S<_$sZqF;vj-i2qw<<@&k#@t~1_L9juMY3A~zR86}Q+$n4H@q;Mk zCf}Qm|IN7c5-ugGZTcU}TRDY@bN?sumP~(9Rcd!@{|bFN)UFXXYmRO4?Q5n!Sa%i= zr_jI>h2Zl-M>)(W$5*7-tV1UvAdOUXsiZIYjTQU3bR3TjU55_TN_|n|pr~f^DaJIJ zW#xa6?^{br3wY;Wb|Amg*6JX*p<9ahy$W^SEg3cWS~O5dzW0ly=h?)2FeBE`p?;A|-=@dO>Q=NDMJtd><=v2eLx`u=_bN<6Vg;?dPFe`wm8)^` z^}GHB79B^Os)0pHblLKoz!|3>Jy<*<>3Pn|rPGhQ`)IV|hoa+FQb)x3_tO;D?hYK4PA%xxZ4kGJ zD8|HdSsAeD#km?W{X*Bz1RNy-Y_3>ePC`bXTV{$MC;o!=ABLl19KQB2PdMq9HXCc2 zt2O`cZ(}q8$hYYVq&L-$r=EjBkg;(&T0fUDoEi4dwMtjw#aB8RbZ@!AK-IMGSK1*~ z8Wt_qD=(`y+Tg^t7owo~$m;hiVCl2S=^eCKY~^Eo3oONfq!zdy;}#p|Js711{)vfo z#oDZV;9`@Qv?B)GY<8j;vl6Mvem1zxa7A=y7~|=VuK%iO6G0uuyWeO;(qW z!~Tq&wn2y=An-j!KqY^7=8dM<|A2Xgg?tDl6lPSVja_ z87r#CR8e(|atH#dtn5ft)wBmlr-MKhB85qJpyBvVdy3=TFVz}KOOJ)G**v@JTbg}e zS2mh#0Tca+kk~F8UsaV9vAOLL1znw>9284g$)qP&K#nKG)zhO-wgfG;93m9W-Cx2rNiCPow_0xWrmy8# ztC#r=`7Hfp(kzOq%v+@(R~k)i!;t zKJ)<-bQ?-FOtk1u+B8-<)boT`vXP6c0SG}BX zh7^n&f7~4>iqK4ClDQunK|Y|>XCr!)+^aG$wN1~~oPI?1G{8nxfm1~;)3uoEIw3^>& zF>MKSySKLlu`Q;}gAnaGS?2a8&X4ZujtvWG%iI6~+Mv5`KLHIofiPUjFQWAiG5JNb zpIJo%Z~gpN>G;7#y!$V+bH zP9xHR@|gvqvnbVJ2*lL&Vc^uE`k7bAo{S`8EWjgX>DzB2@dcc%?J=iqS2a!LV40F# zJLV_=2f~9U8yOr&Uwqizb&N!hu#kMsJx2G_P_z^=XjXfTEK9|R*QIqT%5O(3rFqj# zGs#2^jm%iD(YVY*>)S=uh8jeEo^qmyh)#hjWwbs?^ak2-ZnuEZ0PXIYS$QhH7sITjl&qr3m^9 zsFfIr0(;%6>9+?99;YfANglI>u`k!SCW}~d;j7H*ejns6oKWl`UN;V9cQJaxyq0ND zBrT(AAMBwy=CbTUy6w0aT}BbwVxm>+hd}QYE!j-bp7RN@sj|%`!LM4^9RWTLl)HJz zm4v<;OQCHeOWc!+p+>4Y&F>ei?zYH^IR{JEUUU0n)W&?yZp#~~tj1)e5G%7|{1Wq$ z=cRjbLQ%;Day7v{B$oM`B#-Cr4J_@bs%ei=RaG^1>`+ zr=l9F(iFyWzr*cA-;}!p_yd5u!6wXNRNW(Lvf0BWXzi}okMj&X*P654txH^-oAwNG zU5IOy9%_cV4*mdWYkU~F14(VuLN^+G#@ys;8%tE29r!oLYY_$WSXUAGn=3XRBG%4u zQ!UAlK*o?Hy7~%FbH(c@NxH=JNU15Jp*YCk7P%6mf0-M8bn5ZYS!JR4lqhp`+%0mO zEAGO9qm~?$p=2fNaVn0LK}AmMe{Yl(9Xn84ZxHSJY1BK|on>xUBOvHE3CQULAX)0I z%8h%Zwtw%FEi`>#?(R2>r{@~(XGxaWI>pxCssNncR1&G(r>yi60N$MX9j z6nJhtz&FDFfT%6kw&b|?>BE%yb51frp?7pw9%aVKjkDiH?gDM&j^F>+Z zT?4dglr~tH>hFfo=Joey##t`{|Dw54{VmC`Tw`wG*qHLvVdi&ct{!6YJ2O8&(&T4@ z13i}xFb3Ez$a+=jL5Z+CWn=Nqv*kKs?WS!J#2WYv_ZCINkp`_T#%;y27EgzG+Zbe4 z6ZOW9eB{>osrAQa2U8|-&1!YODtpm7$ znFwAX}T|XIjJLeq~SXGo|TO?BEE)JTn!+G8+XLZnza{e55bc8Z3B=D)ad)(_$v0 zBmrUxR3yxnqyYVGan2G~K&()eJOCRiWjlb+(b7cpXEvPx9i%{U>9 zf87r)XSi{Pgt;^Ow;{u4;L&yW+F@a5z#U+dqCP`J@>da@kqNb;) zL2sEDF~$_Z99v;Ui@ZGyw?|0@{XvtJ=uO;0?45eg#zzFAxzC#FWc=xa-8YrUWrt>A z?!%G(PUk*@jM4?5a_?-Y1`MOJ65zd6--qZh_&OuC=sO8k`w#rcukQCU={3W|Tpu$C zLQeYiOxG}Tt=<1$*c@4 z#C;$`Q)%*FHSHO)Ca2%e5P!_dW$ZDnc#OA^g?!UZREahlb$kL4`-G`7NnCp?n!?7F zD1WuoOFc$sawrdnjbhJd{9S7`;y5bG@asd>n(UY5SMQhqR^yZk0?99y<;fq^^o7;H zF8wi|4ra${u&*7P3^kbrb?Gc8x$(XN1T{^u*0tLCf)dJ18m^TI1b&t*a^8F~9J3&hxRC*?uD~ zMtQt5yq)5jV7O2yG$MZ>VsoZ+hQbIkZ-v#*-Ph9#G{a=vm_={&mw_pFe_u})w(cso z0qHUxe>oW8&>kf#G?_#xnc_n|c2R62GzfXYH82PYN4a> zD2)@zdI+G~P`Q{1Vms=uD)ytWzj57(#Pt2Oqmpx}6lsck+2Z_gt4oM`jJUL`dWk8m z4AJCOeTEv4C99}-y9(p}1A<882Ol=$`TTB^h>!bq7edI zA2<`~Pc8!7A3|z4+V+dKyBpSl#Jz04=xbHWUi79i*YGw^IjZMlAcy(&BE+oFyzaqg z;*NvA@>*J?-+EQ=pA;)PT4P3qyJOOlH@qv=~Zcr6y) zC6;}=pYK87e8;ip`f9G})|8`FM03TszUsoB=I)bs6=nmIMEh<4B`Fi+=9nmxpr@>6 zEmrqEAAoAEQgbIkahwFxyC%o5;$h{;EZ$3|{2IpoS`2|vowyp^uc@K>57oO~B{UcP z0ivXGKZot8++cSDMI}d*Xoj0JGgZ@6fJPgs)br7cI5U4e<43tZlG~;TEgJ&VMCFU8 zMK+onbOMP%Z}CP+)KZ*Dy)woUVy;Jp@>o*bG9%p&P}Y+y&}IAy@zp@(`J8y|XEm#+ zPay6=i}O{zLIn|b3vsDxv@)ZG?So0EZCdJn4z6S#)y8ilK`C)9Q5=k2dQvqQuA(Qm zgJ;3jLp1T;1(X!i(1V)3>gW^nLZp#ztY36#CByz{&aN=@!{1-vLZmJtTpbZ!LlM~XJ3?I7Cq*3Ob$gFCuP{0o9@GSn@8X-yjldl^9^PAnN&5x05Ze5no#dU# zk|+$Nc)qIIK_=y5hH|>rv{5wndop0BN;huf8t7~0LjpNg0^v(PU zpWrMrKvplSOqGrLtQICNLve>5@;fBuG;30V2+0mX^R((7{9;{2ye6{3s-9D?UZi!6 zQ?;kq-?7PVxOmhp}o5Q#kc1L;+r}sqw)RQCDdL6UGv_d z_NI4KDHJ;sAasLM4{kh?cvP-^rc7p+qjK9mRF%8pk==3w7bNEu2>CY^Y(3*haK`a8 zgGr4^qw$c0n8;!iXo@iU*@6rP$QYb-)zJ^c!`HUblJYrX{ACv2DKGun@1$yf5#qsr4nR2Wb&0F zHB{vwWrzd0u$PcJ`5rAvaOUPyy&$GS%d*~owHxn zaIv!aiyu+CI@wVB$iz^%FRHMFC6iPF*9d$dp&1soRc3ZYgGv<|qIQOhe*2OxJJ4jV zYT9+78h({(`T7uJ0{~UE6xc3FgO&Kj_bjpYzbJ7XEAh+kS>nF`MTytm@*i+DwpItu zb`X%1p@>07QhrW}dFfxVfF8O=UO;p8x2yH?0<)Zu{$1&m*nx&Z5t?!{z3QomUi9U@ zjR!(f`|+NV+9mEctK_FBr!#g@-1fvppOf4Mirb91bR4!F({ILM_-{GX^1izfT*CqH_SEG!N#>CnXTPHi6&5mtLTpTK##VGe zpxf}*or}DUyHrgx49oMZgC+jz22-aiw~{(pol&Rvi#o^s4SmQz4Nn-S_X=_k!9k!9 zK=m#0pcm}4wgq9nBmB-Z??G8?JenGuTH5-B94CEWLIOWtHQJfFg%a3Pe&Z$b$*yXA zKS#XG>`%M9hq!33ejMWdL0oEo?mp1FNHm(K1?0`&41CE2(`j}+mZ17d#Wj#*haJ76 z7jvrO?--Ws>du~qFYtkio`v0H!w9$o#AI~kq1u#+hCiG0>U-(Rwa_M`^WAjiN7rlU z`0*h8%@A)p|1z!%Tl?W0pb45b?BX&;#+}inDtk|{#tIK zHF2>^ogP;^cHRwpdZ_7lSL-rJa!ZgkS>r=7710_mloPJ$dr4Rv0DLzIyZP^4(7#-^ zbXl@BeMo_zp{LRnKU{|8fvT4V|N1^GK(ho;yO#27Nk0J46Sp4X#k7OO_ z(0J_duc_5UYW1D3Yp9xbISBqIM4dGODZLhoU~#uH@2{CoFbsX_ws61d0%KuYNivtI zT{=j*YH0vUZXh}}f1aHk4mC<|ekd+JeT|67jOi#bZ~MJ|rwt|fG>w+%l?O7s5&>hT zrE1z8@HSQJ6&aY{saVNNkf4uMECvtxVYD<}C?dumtg3HZ?HOxyM#kDUp5>UCytzDV zkQb|wQ5mxq3UTtMkk$Cov6xVHoVBlrJ==dPj&I;fvgPu*v%+q~$CKxU6eO)z$vT`- zR_@NgSg0BOEjxCY)$Dw8(fy^~Av}>*!WshQz4A#vGcDW~=5`L5`rx&9kdRBVP-P8M~AX3MIkwF((w%#GjvpY#>YG+_b5<$+H{DCyA| zNMwPv$v5y*Ny9<+O=cVMhN<(6m@_!=2Vx7G`ikok~@MWd0Qz-cOGM2v4a{T6bi= zko;UMaNHD{VvJAABlhBtY-Hx`r&=-TEFxb_7?99M{1C;IP>dK>E z1+Nrmi@%*WqF6Fx)TQ)z_2|GGIj(>#B}bQ}=`the{*581rQ)wsy7KPJ*ZFqv^^}l# zra-hIclS{nBQf$1-w&*lY{cR`DJhTmC^T4kB36b7H|KL1t#-F_rDh_*cr`q(X;nB! zYAqsGqlgfEC&Dy||}!ZyB$;7tRt>)W&yT*J(ysps4<7W)~(AzUIL? zTV)H1?_in=g<@@rj@2rdr>o{LvS1*Vp9$O;s^)Q9m;McGz4cP+2fmV6Ql57hxeV_@ z0rcMAuYg(#0C!-OpHPR!Sy2Hu6f{n(K?h!?EwuP=GI?Cq@VUATb|FOLoMltmWTv{` zuLkE+0Q$J~hH^NV;GmUfe(GiWdpNCk0Z+>UlLc<4rHU;Od`mt#0~spXzhoornjm z@C8>Q`n2WN;uTKGmiDU=qd&#T;ig~hqS7zDFZXLoB(=CHPo=H!h8)_I(NfgBweO-U ziLEZWVbVAg^jdf205z8hlX1k&%0XM2!+P9(KXhWjWW2N z`k&Bfe)W(P^PWX+$cO42<>EMNSoN5%R?D4t;g11p>9H!yJUrNXxqpi_r+124yNRZj z0psrL+Qr?Dnm zBoiXa2=Bd)wWnBRTgH6s*;c=CDX8qOP|s|8id7JYMTqk&@({f!(Vk*8`xGIz*gOxR zMzN<@1+htlc%&>3Atu=?j)M5=qd>>r89LO!_KKq*UWyQXRU;@3ZXkO}wHQcuPlWhv z(*OYzX$OPoB8W>O#FPLb<7~;F1M2=ra&$PIE(n(0MMLC^38OARK3ycReZ#l2Y^|2A z_b|eynml&N)t3e!@k-j+%Yfc9yq?Fpw$yNPf#wH6(1xhth3&ckD9UPCtXBni+3CRb zydXCU9uib@VK1-kH;i*8Q#Vs|#Swz)cM1`Uw)Jp=r_+^YOb}c6)Az*q-CQvWLKW2( zqxgP2^F2HBJvZ~cAb#sCnl^g5H8B@q^H2S0Mv&jO$Y$0De#T5)843tNRYXYSbEWFk zQeS4jY(AI%DH(#QNz$L&a98<2xzbJv0H{*72XR^5>UtO~S6}{9pCwjtkhuCuZ|z~% zV^@tE`($M_jKvaCiii=yS}p9#_a;3he4Td^PnL%*j=yLT>;BRyHhqlm9#p*D^ZN5E~=uqU~h-UULt;nB+BJUG<*@{BdPNtIfbk`=7 z*}rI;b$uJgI^^m%B7x!-;y@PRr*u-up0UKeb_N!5N{P@P`BvJ-=j$IzZzCzbR4{l) zK#Z5G{6wX9zipjD35xf$R!j?Z_*D8w9r~(7eud50`+g-Kl&lE-(F1)!$p}O@!IcO2 z3Hp`L+i#^#LA|p!rzpDfaV>>&pZzWwh;$?ni^BTK#8J?_p`bpW^=Z?t;GS@YYbVVQ zRMYfXO%wax9~EG)0I2Vh@D1Lr3T6+xrTjKF{(F$3VeDscFSEk&Ex*^0d+=% z8c$bau?1U2OLDF9I{Ed^-hjPhy@NJF#Y;*(k!+j1&eYKE_TILyI+V}qR`+*)SDA8O zX(B5EACJUxo>ORjs2^-DnzR-_c6w*}_7r}?V(Xf;`Bo@Iomg!*((Sb! z5vEWdVv*>`1Z548DbQ`L?Z&WK+2dK8S0I+Xa}!inbMMpmsinH(+G^5EAFtMRxBYwn zk)&MxUlhR5mvV}*s%eH;c6@(f@I-|R&nL?)W=bB&_5KZ0*`w{ByBkAh-7&(QiLF~J zbMHQqz2*U1p@zygF|f$$wsdzD$;vwS!Kj(Aqt$Aw`m^@6yq`*Ing?tbEaTj4-nq0I z8OiQ8C2y?snTJ3hj937yzxjLcdwFZPKdFUBXr3dL5*uzCVCdvH`r7rT79gxKwl_#C zX%Uq!-pdZ_{lj49KC1G`Do_=SMEW*6O=#huWg=Bj`m^e$mCC{_Eetteum1G0+r%#Do1%LcL`KRI$|?;DyQxw#5!FGi2~7Icn)?*cn&sK z-vm^Y2BZJ|A)<7fGOT$I8apc%_Eu-btu(rZUS~pSlum2k+jkZIilv*HZDxD<15{BE zDm*sj-zx<))vJUx#a+l_HD~NgfXbGZeK@l^eRK3_V@X9XPeitA?%hmrM$je3pNhJa zpsfOgS17+3b@e?xLB>QQs4)=KD-e|OF7pOyo}d~}(Cu4>KG?)po7>^K%F&-Y$H{lA zSbqY+6M#<4ma<5Jr|VqAxHcW%JC|;eZs)G3L52kBMXd}}yVX`y?oQ~;pkO#Cce={Gdj~6bfR%fb0d2BdXlP^FBkDfUH`XNz?Id93UrF^qF8e@ z6JjsE^AgBcYMu;pbwpoHB?XZT)go~!BlJM%X&WJ%{?6k#Qu?Z}EAE7uN&5LN2;Q1S88{k(l{pW z-H%{861ofYaGI&e68A5q2Q7@N!GAwg8`pgs*_lMKxY&KUNXEx+9DZqm2=Zf09jjSX ztzN!qx*JK&WL)nQfWIg8op8#A`B+FCXSvTPE0!?6?d~*=VSCN4^g2PP18-ojYMC*< z901ady zEBlY558~OY(*GX?#S7x$# zxKcCH)E=d2ORqEy=dvBxI&n_H25D+{tB1rRNu)^0();d6@k0GLXPI;L_XKe;VWA~V zEb?9C6UVCDAZp7=%OPZrouBF{+Vy80;n2tRkq!%Yw1RqXk7hFdD%QLs%O!DUr5B7$ zu4Z+^JdsxSbDmSQ=O}u43Eh$Oc7TESLL(D@^sFb>YGIwxx`!#IVm7m4iA1JIvo|T; z&;%N%Ry{(#^&J!&@#+%~6Up<|RMQxuhJc=8zHY*5caj6_zV zH>l~YT>Y?*4^p+SITvT!l(w zSpV_5H>~gf7RSwcf}urKyld)ZtNo%?Vg0%NUK5gfQz`u_!P?>CK{9$%O~2jGJ3x84 zLFxCX71R`;J(lwRx2-3fKt#eMK9(zU)i>~ua41(Jhw zGBzg6@0qEX4d0k7n(G@wn7DIGXS>_CETg6j1$`rcMt{||(}5S=R7eS`;5IJ+uV~%K zn&`oZ2 zm$I3K8b#&vz3)^qmlIg?gweD{iYK20*lgpNPS)l}ih@M!amAlMLcDuZ zw-haMo5K#ZZ#nMP%hp7IR_6NsD3+J~fl`|0i*0AVEEki#D>C&&-%#(A!pSl3GE7Ts zWIN!TAogh+_mWz`5fc>|>aiMT&|7$^x**6mPoZ7umy#rCdPq$2Zx`fL^XR3_TA!Kj zmaRo;bYHV5qz>gMXi%h;fY78x4^l!#25pM)f)y{D0B5ikGu#%If50AcxMiT-iIN5= zH$p}EI>SYI#WVO$<xmSfCaV#JHn;-Hen#rwsTcZ^}fd1S;N+2S#z?5q8;prsjpG!5XX*cm*^lypp^-%2UUs657 zEekjN9#%6QWWW1dU%iw8$4qBsNo6T@R!B`N-}sPEc#-=aBOKiww68qm7jKO-xg*S>Fqk_Q5o8zp~~!OkGUAWNVB~dcyvhb z|FFnqa}8&Xm$^G%@HVrCDynvC8J(huzHXFy7*$7&EdtTkw?G4@w!>R zGOpEX=|36Dk==A%uqHyBjNdjw;3&hAI}{GR60LW8OtE>unB!KjPS~?s+WK0f*44jrSO?skXq_ zg>$b$$2D@QJ;DAVD0UJ2Q3#5^1m~${0INX=-liZD_kl%!wbEJZCC8B=_hWz_?#f#8 z%DMUj4a1IeyNq014ONgm*E)L zb^DD8KwLS-7_bqCWh6?FWnd?r*CUC=5LlS>d_$AY8Sa?p*bQ~uLe-(t%(w99uMY7j#DL#dUJE2n7vOJn5BU1<0yHZQNR6%x@V1@O@mm zJhYWu&=8@rluuyB1@pLEyQO-zFfb-nQ2HCxFq0P>2a(vn``l0IUhAL!4YB8qBd!xEGVdj-W=vOBFE8*#LxNFe#a?CH3-fJoC`Er6&+177 zMAnRV6@*;bKfC*AZ&IJQR|wcSuW*2O3HKd6ea61!0`QKy3`I@|Y7oT|B&BkbT% z{io$jaCKwl{y!I&uC%NgP2z7f_ivraLB%F((2}gFW{|snvwc5XwvBLfWfAfx;9%@m zDoPk2nPVQ0X{e@Ed=2yK6kR3pZSK9TBoCaT-Kw9#FD9nawi131A+iYkinn4)3jq2L z6hO!kg`l~jC`h34xltig8-XZwr|s$y=bPqWi|KI%3k45~EYX)XPgtO+W=W*jje-Hu z1OLK5#Xn-zVKV&UJ>xGte;Ot$$vZ^fcKc%F%pq#rJy|o3D4R{*O_vdYrPZD|5u9bc zcb|Xwh}ax_W)LYT{n<8;iER=;+vZ`Do}%Zq@R4P8WH0KKtovHD<8I+o5@L@vX`RO| z%x5jedux-Wn3-r#tf!_x)xhlV+n;A_(B+m5ZAcc*mw|70jl?8XSVZ;9kRoHJzcl|1 z%HPuC*MHW~mi3YuZXmM+jR5_lX;=CVKK8lYt%FsE``Sygr2Vs5(UB~KFsaWZ?@X7BFh!hQ3VrSw9PU3*R*O=ke~&uG-`U+hC{H8({+1YDnu(Vl#`wXR@8ZliLT`UzoD|F^$ zB{(qhWG-%k0-NKorfq25aLqjI6hc~6iP9h*eQ-z z0)$$ia~zKu#5lhnso}Z09@Wk4HhBCV%2$ZvN`u%})9}s^oKJGLM>#Jb zELkDKlMTWz6nBpEU_oT|_8b^+c3C0LfdQwayK|i72H}@Gs#!{c6*j5~tYB2LQ7A>4 z(>Xg=38gGpVJROn^HVQ8+8{bBaz+P05@mJ;4V|t zD>AxULrQ%ptEBU?P7Y-;-W))mH-^Rr(5y|EICQJq zA1_zks(-F4yJvdc?Z?rde)NeVs37^1AwJuI4`b z)%A6a#vxtwb!|0ppFPZo*XMtI6mIC($u;~gs8NlvR0M{XT zX1R$9=mh{_yF!7knplGELIt>_&cM!5U~>!1adir?3D&@lR)CY}26mtVTzg4ijvJ%^ z7sV2I%~dM!hmhx&3Y!!ZWCg$G8Tb+4NIP6XjK65)$|S!Z5<(A>IiWc z5X^g6IU44sDv^)mc4bq{%`s+5j2Rze?u#*hjxnu@(3wn=0QXz zFNMHw6?i`csuWmBTY~`ml>$9OU`qvhTVRgcP=Q@SN_PeJ4=G&~IEH{c<{v*q;P?Rc zwgTq_u$L70n+4k4Qwm%bz#dfKwg5I+frl-y#9gnz+yFLSf%gJfvjW|8?;P@QmICWr zV3s>cfo&`>$NfL-U3oxNRsP3{V%nbVPRsOau&~m!;+JL=j7yIR%{0qaL_kFmh$XSC zC?4hOh1xbXWfPTCRyNsgWiQH7o0_zBEL&*x+|0C9Yb<`B&-a{j?|twPE47XH2b_D) z_T9hVv)pqE_>g6aox+EcO|if7VZ14p!G{`CtS29?;Y0my`<;g&17u`-%!D&=-U4dk zJoP2=5ObzLo7#$mcc2ENOiz|!B4`u6Z6eJkLN+nPCc4lgxK z_YKASh(<2-=)EI{rXt&pHf1zEdS~+5n=lx_9-iS@1<~%`elHz=VLNW_7vJ$gzK+NK zMR&Xe9m}9uodKDYd7{Oc9Ejq$WIL z?j!rg^{N;bZ6(4N_&+;B!_R*C6AJxdHhbjwx^BfQTFG~eK||ig1LwLs;eljmKqcp6 z=m-78Cy%4Vee@~q=t%&zI`l+#2$ABHo+Ly4%{EGOKKhJ|(v!*pg`I&YQ0;S21|A)j zZWalL-)7^W({pwE%-Oj^y~vQw!W`$OPupk6UL6eCd<@wU=d7x{dD?YOxfrhA&e*#X zUfdP(=suXu-V*~7%sxA>;ATL69T09&Vr`ieN4o8<*6SflB~ZaCE#{it?QA8}93sOIT&!q;nj+3(NMdqz zv5llHRXYS(h$+-eSOSme3RaNxt=~e^Iv*z$3=%T1l4mTwbrx`u6^&%7v8(-GeW9#T%lGwT-%t6vZgN{@J9*XRW)b9~(~G07Jo4I7zZcI0=L9bCybVrGpK zc3qrT9%p3RtmfYdK9)41Da@KZ#D{TDqK(;19fcIj+{A~~x(sJ2N9a$8r{Fbb_Hax; zzIS#$(6<8&MHdn@z}MBM^RpOo9cL~T-NoX1CrWHzzy|x_+7c!@f@00cM6m=JyuqTu z!o5LYD>Bw{@I6fgf6mTdkmrllQ?tO_1BAKQ0>=;6=irTIX#-zJlNV#*k{mn*49C{| zWVIt$sUrA8F=Oj8?R)F)NJNvnDax4Hy2af{mpn=_!uQGS>B83`U4xk#z7(ncOg#`o z>L{kRLdp!K5{04Z?8>?yD@z@sqd7|MCK4Sd4YR9i*=->2ssAM>R`m$g7v=vS z$qUXU;lH1{5+ikxIRTj?`q%?F&iSiIS~(*;q)Ps2H7Fkf?Y=(6ef@Z_&I_3<7 z@H8(p4E#B-;?_XBHCT_V^lYP8ir9WA#;ZN-URHOHrWgH@{y{dyL`UEU+Wu(o?t1i0 znXGKnx*th-hIOaz_<18d>#*(=MlM^MeTqJ5=SPAJ#bM>&a%j$HO3c$*?imSn$h{_{ znDX{aHf_pJm-`SGiOrza_nm_5D9FQX1bb-!YxuLwbB1B`qy{$V5lo2RE8Stv1X+Zl zW(P*yC2zYfB~d(jIEuQS#II8wLm3jLjd=7retp&2Pxra0g`X}wz(?H{e%f0MqpP-j z=pg%g$iCl)F4=YD56bEJP?E1>J`~o0BCEts=yZGb%lQvJZGuj&doKtwOZO6z=+iFb{N!R$&O}=!yT=IN&H%yoEGd^5Nf#q3kczDk&lz^LS zyQs$N$q}S5IDHuKB=7)afidbumq+)xB}u|#|5i5nA#n;^7=T{>KjtaskntIu2ry7PALJz8EG^p$^+`CA!^M$M0(UVts|CJ_{Fbnn4yrpU6t_tBiRV1^H&{ zDA{7Dgy2>II;>PeE9Jb|)(T3D(`zEQ-gJ>ArZ=f%zS(hU-a+QXWUfM>LMeXrW?1fq zsXA~>F=v)Efzoz@W*iE5VlMmEFsSVibL|V8XhQI#U;IH<zE7hD@GHjv0hN_y%Ol ziBESRksZzAy*gLk#Z#X$9zvgelLyS^Lr*-|gP+tvXu&=0iGv7zz(_yo$?RrCc!8pnd;rc!kQhw%o{c;n zYd`h7eUxFhpyAlRFQ<^pnlmyS2j^Tac7$aV%b%dqRNS6cagpxrOD7zo*FPVpL>)6H zhWbAwSZmkN{$Q@GVc2nJ4T6n|n~-WiikA-1^f}%6W~zD9$Tx5N18*kR(m3lFM>g5) zwT>1V$NQlObWD$othtoq*JE-dD5Qk#@@Mt3Lk%@O$hU-^Rle96`mC6|VgEEHj~g^^ z179o^6B;&mQl(>+LTK!4&o9cbcL1A~#T8ju)JO8=;h+s~G%4duGKGcta~(%Z_a8e# zq~47ISg?cQhGyPC$eFqTrsK&AD707);;<)mfwm(gu^r=|yZ+6Rq_xWlEBzfdM5rw? zoydkbyYQ2Z)|)UM!0AbGcKn4kKI)2`dnEQW0p?v~zJu)TkbMR}k}zkgwsMOd8z*U9 zH(TuEt`u#<^`S>6BHqck3aL3v?S>RTTDX-D&s~Yve`o5DYxsJVx||RFuEy(&CqrX! z>dlmQtDq@3q>Ce0yB#gW&--_-!05H_B>j2+hiI3%NhrnJ*6c zDy*m)HIaKzY#<{GfN%$pFnP@zbXs1B}n;I+loEyoPSsUu^9 zSpr&7gNl|#J{_2;ZyVZV;g<6zVi6!^P+2S3a6Wgsz=M@FdO{7P#!k@3-XXi|Vq`+~ z?2PKbXXArk#)d7y*mWjfIJ*H2o@72k+ml|)RNuKUV~D~u=`{*g)`{&%+erIV=lSbw znB4{09TUB6BF!d3HZjB|y4b`Bo5-{Y!NWOg9*t-6@@qTk`)9O-)Tf_?^9d|E`g(`$ zp~Am-bf!H9E>J1B9jYuRvKGe1(+^LQw0MamF?MJGQM}z~2_?%DB#|sKpr<8e&?bHa zON{1ug7iGT?Nh5Nbo7@dj6Y}BzyIoqTlz?QDB$#cX!kEzK4#CQ zp^@Wof+E`Vr&fw!ZfN9Vf`zyVy+hL#@t@UbLT555Nd$K~C93IR!ww=uHzv6JAlsk7 zbEU8kzvaqs{zx>m7l!G{nn#7PW+hIRjML_(S*vB&JP@yCYUcf-i{Zah z5MB2H!ho-|5(Yc#y0Pyh0SN?SKM8sgP*xAx#E-C5tuf!1i$6wN#O`u$9wQSIqqK}n zoUQOVz4i9iM*qqT^ubuB!Z=1er8;>I*hemjqd|z3fJHwigD!Z$qqkKuk6|Xd!m5rZ zue2I*(NzNR+Uy$k@cqV6U@;PCxEN6BMAT^X{AidxSqWWvi*@Hsp(bJ%EZ;rfzM9`8 zFg&rWNM!H+3T|-B+CYsJ8~(EzZNq*KiGzWJ$AIq!Mb_Xp`s~O$j_nuXpbXxKOxUw= zu`ZPaMwU2m2OFF>@m&aAqLJSYHA% z-0jM0od!jomJ{+0*qdB0;2wT+vFmyNA`$l4^)~&z9cu}6oMPm~Ry+wlz?g8TF1P~g z7W#cNue!wf4|tE8fnP2_JEnWP+W*>~gcyTxwCGZOTa?chs`pJkhj+b!IJQ^0Me*h+ zDSqr8T0R906uC~rrOz;C9c3WRj=V!*ujVoJv)B;`EQLuc)(+Gy(Fa_lus6B&b1Gh_ zi;4_^FWZe^;`VA3wv}W8=wrN4Z)iSv1YT;2$$t=-)v$eG9WFb;q0NHoG78u`YG0o9 z137J!8X037F#JJMP-boZ*zriDvun4;9aT6gp20EDbFbBU)EA4Bz|yVU(%Fckd+7wq|)U911l4&6cg!`cHoXGpmBj0nl#xW(X zTMqH4=ecN?zDHXXE%mX}(H~W9tQ_sfa0B28uTdBJXZS(mLQ}6V+pxn>m{OXbMl@_q zqtA@OxtF;ASz{bdo!Z9gNV5Ac#pz4!6 zPt3C;ja1NZIe;U%fmT z-2&grm`Mdpf#Y^S2mU5-g;)&3tR%IvPyS zO&}3UP=f?|6J+f5wOY*szd?G+DKF<@*7Ihx$l_|%9qe7}3rtQN0R0T%{q0^)T>;V9 zpt?^gYmiN4zUPlZTCJ`?0S#na!;g+?4R8uB{n0!+Iswe9lXbTObKf|aWhl${{HbXp zTQ_~X+$UsU{vK!N&_J>VY$@ULVAhMs>J5~UTe#xsF${F({Bw93dh!{WE&pNWSZGkJ zaj}Ml8b`yT3OODylp{-os$x{e3rKAkl@9>C$|gXlOJl1ouTM~%c|1E&+YX69J`%G* zGotseTRUJ42ge(<5T5POczedd1X6%fd@b6p=KbyLK9)WlT5|?yyvdyemdj0(ToVfH1w5#m)rmmI~G?dQ=b`On5>1 zSh@)re^JJhmr0WuPW3Do>Gyv7NaMO~B3*mCi}XirAieRJ?8-f5$1mcW*q`l{ww1`$vDDzoy2)BrZs;%3Za8RA7G4G_Rj~tti0?usg+xKJs?r)#x#){u}OFZU2Y1dri4tj)|mh)g? zY@|rP4gTXDxXXT@_lEHJJ)Tc6%s0*>DT^ZQe)0c_UVB5Oeh$A)()!-Mb#u|Pa*xnM zTLtmdJa*H?)tZ6zyKjp*Nw8@#b`~`J08ITHYlWaqP8+k1aM*si|y|w;g2&F@AFS?IS70l#!PLjr_z|sjNR*z~X=0{}&31(i9sAp1z@``gNjyxe zJBP-m6)*3>{%DOMr4>@!l9#6lMjQbGLD$RJkl9MD9R_gSpi!)v%wztHLpolAoCB|GHEIcap+J^-}KR?dVP{huKdxWay^y5K)4eYQ$zLI#)hzWvW8TNrWfIk zZmct44M@z1n#3J$#_Y0DS;){$KkXvy6NM3UK)=egWkBnHqK1J$-_~o-C|z<+#b@3gk4QJ0e$KTU$3! zS8Nod^iokzEIxQQ*s9O}s+YcoWBp3j>C_NYX_l|jM6A`w>KlRkSUqkk&5$jOS8E)& zmVqh)m%71}ct{qe?3$S=fu)lqHOZ7%f+w!335>Q4S(Q7*l;HX?OR#k=;mFb(s7P|~ z=cg#sY>7X0&hdI}?n69KQgj2~sTZa~EX{`otibpDIVonv7P>fD_P0@EWlQG{@0H~uipR;6D|iI7(7UTU?K=u zCJFpv>(^YbwG!y z>ZD6JuOlj_od--UFye0^S z{z+Ji{f097Mmg-J!-Z9E)v=TgbhyDUh#YR@&h$q=IoL83Vihz)fEW!kl**2{yVm)N z!IA{dHbCZhr3UcjVB^#xjw&IXLk_r@EkF&OGgi0J~P&!2xl453F}kfSh%R6wusMGdI-X~KeM(t=6=Fhh;>llC*!#6gebEPRY7a@pjD;iCOvt%S@C)#|Zj*-eDCJ6-ygCv-M2H}LAp4m)s zM;XLYq*@G*8+WAF{zRcE;H{3EBZA5l1ORv&n2f*JVuEof=a8t>7;6S)ger`Q=ZS1` zqeffxHbdA=K0qB|G0`X)o8iTRH(w;uMzI}yB%}`;QDG8IY|Aflezlk_tZ8o6n!8)N zXXZtz{{fYXQ{h@D9-Kto!_*ctr(t}=qN5}$vi-M~<>a$Z4RzE%PV{VjcKjD*@$S5E zz1WQTbz$@71?$D;SH_ziiA~!o*czK(ta0go{(3PP-Ac+_O!WMo8wz0-j|MY76rvH^ z*mGg+nalFYzcq{+&ZF+wp^A+?F$ht|Ok&WD&pO~jIKhjk){cnOeTSP3`X@6zIk!M8ZNX$izX7=pjG@A(7#1NZkPsw%w5}FglL^vUFmG-pbeMs0vILqDy(c#ZIO6p&25urJ6 zDNt6HT2(Ubj;k0Ke!(Q8Fn0Gl z&4FDCZn?^ocH`+*wUM#LNPLcFxsJ>odfZG^;S4Ar`IY7%@&!_+$*PrzB!{{gyZg z2*na_3*(2 zAmV;7^#xer#S{?ZOOOMe9gah=h!du*><2xX=`37B3Yrhwox`4VNwBufV^#jjRuQB& zNKGt8$BpVulh!Bsxatu@sKcBeN0oNXz*A=%F{E10r$*PM55_CV^mG7?$xD6qfI9IY zSGe~Eq0YtWHx%xp(q!}-{u5?J)!p5E0)FBXpz~g&GiO)d7YfABD~plYoyA^3E9l;t zHE4bQQ1;#ULabkyKb%%u_3hIq07-8h%vstw>M3r*c)#NGm9ZkOr zpYCzY!96PKQ!U$5I+-$E1kUt*{bkr6+j&d!u9mao(atagFK#=xnKF-KdL30<5HC z|9EM#$&ejNzVVXEHyKUr4H@Eiu*gC4v$plpmb)0)JD!;MX79LoqTWvhn&VRbX{KgqeVm=+Op*nb~0hMg!LN7>&HNBxRxbaDKizG>4*&w7~==O32nCr{-%+0=kw7B zCx-{WYyLbiBDp(m^H8(?B#W)iRhdTcH%OX<>XzD^oY@T0BqVow0s|rP2AMt?VOSY= zVHD!}g=M2iMJp>^SJ>Q)>keo3){Z6BX8 zFn;WItWM~Uv~7A06Zo<4T%DA3qc44(FU|OdKbD`alag-mrE7fYCSRHXCV#BERwpGr z%a=|9vG^nBtDOT2gu18X$iLI^kKUX_W~hN9o8i14khVDG(tvu}h3hzEramI(SQPhD z+jw&@$7Kc+8$~{TUMAFl0Q`}_equMtTT9fJM$wKfR2O%_wAT3Jg|w^EVKYA46M-#a z*Hg`bNR6bkzPGI|7X_$WDVFds)KCWY5+i{vUo1a-mfG8_ZAdN*_^r{t!O|GCukodu zY+3~RLE?VLN(SZy4hX@&;hdzeVP;yXsu&Iwt~q&#WwDO8})w1ML$#ifq#MXRflwdsz>|%_b z-?|fVMBf?=DbrT0T4N~&>NWV%O*So* zyz5kiL~lbnv0GXVJXMcm7eN4RNHn^82dPOh3gJ&9b>;|Ca<)X1TCL&z$&EpgPi$Z> znyy~xtocyGemSFXA*Oz5Az0cSc%e2ehc*`+h0&JGVih}j0LWw|e||HuxvV`&wnTR? zW^ydke7VSdiD?vHdaf_s=u2PcOaC+;%sUx8uuee?aQv;vyfnKjrT@2KO&ffKtMR3q zd}-xNQ=a%ElV<6pq}}l<-lrLfVj0G+{E=MMIuUO3hFz|z@aJ5M2^y>{_U^?c*c{n0 z#R;+A>XPlb=72Zm9#C)9!R47>jytop4oMi6PY$qrW2ddNHnlV{3yFMOyI|KlcaRo>Vq?V zC|qN|6a;R`eMMj|#T*wXr)BkxKfp$F#La>_pO3Pjk60jb1F40ReR>$%F{!Z;&80LJlnuO}@01KnluQ-}_lMEs@)a zB9w^;s4uSGd~+g%bBTow{2I%&NV)rR(YdnTOB16gcJ9b zf9K&JBQg@)L~3UKRi<2{m8n5z#-Ay_VnZRc5=vNcjvWIDw)O2)p_p2k^@4r%d$AM( z{(Ba4=Rj-*va}PDf|?xvHi0lM8D*JanIWx@8zd=YPN?W7>+iv6xN znG))oT$5_`apbl4KQUhRA^UB2$0_Dhpxb zT4(e#Plytzd*W2~n*0KK)Hml`gcs_#EdVh0e7%F%l@wtowvy~=JOTHSsD>@2n0IrI zAZQWA%{i=wn{zaRhx2%+=l~*^i|zog+y->jpNZpb$W%R(&>k;XS-VA`E)T~(apS`( z_4np=GJEK5Gk6G-0_KpZRq8yG+ei>e$oIf$%YAkJ05l0Id0f_ewite93Q{_dMNNGW zR*tE_&iLalS>3d`^XFg|07;Uhw%HQF68x$9bsAR9d_Xf!wiDvo9~Q@|T=Dno%ym7p zxAejJYJc~-9t&z9JPR#a_mC4=L6{rx$~p3T*gLyZ0Sc%$Ymw~~s2I*PLswb=xJ<1T z!1Way-QzkV2h#qgPlEc8Uz0kKjfT!HNAN=E7|nRLw^cvcF{!Ud5{9%?she=xB3UUF z(SW8Z&yDzJKb<2o0sguSxLh(kU9)@?(U8j!BMZ2J@0D*`2-RCcY;82uKxt)$-UFyF zsa;Jn0>{@ISHNyN#^`|UnbEw_1W^kCK+F7Q8fZE@3rsqE$)XOzjoAT+D-r+f z8_#d`fIg!ldTBcsc(I-Ef z9R3NAxN_%mTRnj@Nx1E}42MrqyBmyltKC1RF@c#B|4NqAW;7m8rP}9?Jvf*dkrD_m z8O#^>)=d@qD=>1C3P(!59t)|V(wUCyFryvK$igy>zgInElwP5)O%zOj7|07A;qB$q zo_h12v8Fc;^?wmWcdP=4`YF=Hi*+?%jBlUmt#cT4`e9JBCBqc;*;i7diy-yMAXKT% z7_>s%NDuN6d52zDpC6-4gi+Ym}50-2!R?AilR9vWC%x(g(5iqsju|O9H1-+l6hBU zvm0|7-2w8;m5KnJ|Ht&xzB z1Uai2Z4peJ4`T9?UYsR2IWz= zQSOKorr1F@VGyaZ*8o@kA1PXy+A~z2p)ll*6RP7lbzy6mi#xx;_#5Z{sK-`e!1@fq zsp1S_i1x13(a&~@SZP>L6~VA)L&#)oe($re`eJ+Vnd4pg$O zxkj9c&O*0@?k~+|f%OQPj*!{ulPGqRfh-W%7UU%O{begb#x7|Z`9{MH^(e&z$cyqB zLYZH^aNC0 z5dJF}hGWQZ#T~+#Mo|dgCoxuksFa zM9KQA%~iH(a#LMt@ZosjX< z$r5F%l$xBHhO@wN5*_K_5_#yS>-2@HSAbjGQ;Rvyl@2zwO1o(xT!_tsHW)8PG>#&@ znDe|qK$R5_Bfdji=@j90MbtV_zUWDLWFlx2y=|g}%m%Ufnz#Xmw?mN?0$T`1r7iW* zC)7ZLARa@s_FPC1(N_w)@I$d4&1nmqD~=#IS^z95)Dh+=i?lc$B>>L0>*&1u^Jq=M z;dM|^NXO_06UhAZ(dzvWO-v6`GyNf8`r&R(Jxw%ho4~0yFd#(K`RH(hn!GSTsZLM1 zm$-|?`5Gs5L~o^pxoneYyBSpAevmksk;e zSJUIj5If)Sa6K#w5o-_qz44LiBqc-fZ$-tl` zTCBVQ+--}DTdlvxb2omESHgR>!MHYvN+4V`nP^jZ)1%xuI z=J%k%I|DupM$8REkhWn2lI{V#IFmO2LrS&@nc$1WzMAuPJr^`VTGfIrAxNlxhPYfF z{gDYgXkKSg$j}6P4IzM$_zK;z8UBc(mPGMS62Aau-^CR8x_dyc#!%#Szoln$YBCPR zqL+CbA&eZfJ5!nsgfsBMwe|~m&PX(4Ot7_IYcxQpEqJmEpro6~Hh#NKLX>#FjU#xC z>*mtxdR_O0ChBG%Y?=t`C7g5t$hb9-!i8dQ$uLEcVSOSEVo+T7YKS&N#p$G+byOen z)QD)t)_-b^plw0zcc?~Cpdrl;ke1Rf+L!a}Yt&93GXmCI#Vi3F(-|AFL5K}p;B?FI znw*ZO0wF~^61_||k+(DuM}oryWc?97WzvQ7k0wLlL`D-j;ZG$YgGVDb5EviP2GunO z!p9@b$c`Sz<)G<*AVSTu9MpeXE(hHLG}L|{5kIpW6hRyG%+2aD3OXQj$x@5h$Semn zt74N&AcXu&0EzWxs^nmh6p#=i{^$yDak*XMF>62}mg1nO1>EdaxQM)^tZli;^KWtx zZs?T)R^v}KDGe!mvkW?!fm~!2k4%1-2TK^ipwABtihLLc9J-Tk0HiR8eLL40Z;Rs- z@ERc%ysi0iUwb|nVD0qa&vMWNwK;Z)pyI{{TGuT;em~HekjL3DutL8HGT~GlLh}f1 zN7tq!^h-8pqL09|0`>%Hjl9QdgwZ#*6bFxQ0?rZ^h9!l)v|~+OkkVM#X*w7n8EOb< z3_vinO0&U$pCoKr`p`5nJzMw_ckK2TWPcV3XVLDQKS6WiY1I`6P~7n-u#hdbD;=Ye z2ez(Dav$ccNiW^86&C=F@T!BFn`JQK^^Pd0XlsdrKE9E3=hR{YbYb%VbQS=j#p`X^ zu&ISo=s?N{iJZ5)>HgESKDZ7Q>v4L&CnZ2DuV`BA(s+Gf6beo(1$VW`;$aq9ZNz$g zV&UVU2Ik&f`6X79Unw!21fNa{!7nl%cH3vrSsM?GQBEKy65)ypUi1V^q8!qbD;dt) zX_Q3mxitxZFqPD5K<(NdSO{Df2HWoi#5=Vj9PT}n)bRUjfq;@ zm4+_;ihb<5Z{!`?IUCfd>bGepW$d8eis3#NU~mwhrn;bhWIk-llxGg*JF`f8Gn@6b zQ@1Qo7wN$Q8H4xIe$_R_uO5i9x{HP6G)=Pf^7O&1U~IkdkZ_pDPGY%)Bm5DJJOBk- zXCaA`WTd>KVCs21xBstFS;Z9fJ0O9H&b_YyLUf;%vKde9rg5-j^1B9=?BQczXB&4N zq>&NqNV*oT-;Ytq059{Euv*|@SmFXTiE#(elUCx;J5(U^LI9TFPwn0UJ&U&{IYHaC z8$lzF376UEa>^EFqs!Dz^!i7mj5=6^!Fc_1A#YNl*VpRGx3rYVnMv$G__w_+ArXL@ zUHsk&b^n7CDaX;hIUE{?@gAcX|DO#|tx2hqx!N4Fpb0J!H06R;M zvMq-5s;Ef=F2x!x;HOJ5bEeU2JW0d!W2F!9N?{|E1etyC=kXCHfe#Y8J!xXp26hq0 zw~5lccRCVF@Tbl){fY#m19r%t#)^D9w=1#9n6OK{Fsd*)4&0@C>5-)yrF8l-Eo7D29-rSFvr$4j1#I$iT-pi}~B=K>3IZb-8QNGo~%Y_P@xz}-S zoDQp*{vb7uIHw5&p3Yhts4SBqfcu!Q(j>!EW)|R2(}O6^`ZG~XUeiU(K>Qf4mhgnW=5eA1;tCZmB}`=ml0Dj74h!ShihJMvZt);E0EzC^npWBPgDB_&BfU-3Pn z!n!s4Gxot54rme{(A~YJ4Cy$<_u{@3DcQY-FG%B6v>1YBM26>Z)y?^n5O{YquJ^5# z5i zMft45pE_-+W|TBS=6@F?Y*;TCNpJwQZo`BEtJOkCt*fPH6R&g~KNX_Abfh;>LQ2k!hV$lSOWMdtW zO&3W4hcTeApfoocNV`eH1~8Yr&!BhEvqY;cD;X3e>Hmbmh6S9h`_-(dVyLhicQO2_{2|0v2}7pCIrx6S(e%J{@L2 z-S?tiSGt++jh5#`R;q2;2r9D?{+!Fc=Xl_=oXGnW$rK6$6jcY}Ig#gL+?D8_V+;O| z#j{WIolg7M`eGe*0M8b(c_q_xEvs6KfhJ2O0&+rEsNU*)d@HK4DZF9?@fI@)~ z79RrPEu~Q%ptvAp8)&6TgfT20ztLgp7E&+c9SEwro|C_AKoHqkfLi}0RChk{3xpdP?4x;Mm6TdHUB7rMl1;~MbLt6MN^G4(K{=^)PEl{TR*{;~AtA!ODJy>KcJL1qjdlwIFzlCtIiYyj{-p02r+0iUUTwkv? zQP$m`G0GYSAmcW48G3}c_Y1%%OCR)0=@}$vi)SNP_-bcbySMygV1!KV5OpC8fw}?c z8Rj(;o#}9F8-8Z<2xvv~IOSzrPD%E&8;h!pYLT(L2TFuf#6Ra z2wNt`9^8Fwz+C*Wbo$_7UEph^j zMAO%5(Fa*x16d&ERhQzB0uGkB$>S?DidCT)eX4kMO#qIPi7R!C6q8O#9f9wwof)6B zYU8#a(#E}L!{A6E)sO(a3yVR^Nf}nVv_B#CfX+hWRO-q?6}P4)%p>1@k9Iv%e$fup zHwH2-8#$DXM3d*P1fT2E`_#b+LmVNGT($TZGM2d7+MVe9jIFmM1mGBAVE(|uts((W z&t2zkl^pKyvn@U6&Pt>*%?#76ocrEpNT69^`}U?$c2Ilzx}&p^@q>*qg6(A&n zHaMElsElR>pHX4Uev9E6l?*AE{%Nyd!y+uopLz*X17cLZ2PQDjb=Tv6n}#L7rD0ha zn}wFN=Ik4AHL`&(BmT@Sm5eWlR_T_~Vm!n}T6k2WAu5-w@-}`jM$}P;! zN)dAuQF$OMSt&-qbO6Mm!h!Jf!pY=wJ|;AmOW(E=j$aOV#&h3ng0L}2y8zW}Id#zj z@D(0eP(u6K*y#W2gY7iSn8}~B^a~8lj-hyfX&jMTP7fAPbTjI1RzsUIs0TM}b@k35p=VK6l7l!+By{vSWVF0d^1Coh6q42BQdaW><(&UV7BWIZJlc#+VzMBc||lV=V>UjmYxRwpaU(A6UC9A%i+GLP9u&UJ~hja|)E=K4=UmU43f8A;1S;qey8Z zl&aI0q>CY^B~5h35?pF5j(kytxk5LC(gtnOqj`0jC!i`a)B zL1XGI5Zk5Ax)lbU7&`@`GBF$$C+nQIwaL`)V>h9Qv*0sIjdTvO+Zr>)r&%~;CepPZ zN1z$C9NYs~fho{Lz|_h=1g4$=P*ejTbvgsTL6LW5HOJ0x#;!%@7|Gf$ga>}cyntD= zA6)70s6<;k*RX)c*WT^(7=~x(VSNQ(nXPDIh@#h5oZA7UK!~8l0MlwpJ@z*O4M`wz zokQ5!52D<+nCk;h6kgWz&kj@MWopwTBx3p{ ziK+?w&#(HPpVm5|Pp!<{%GMsl>v)Zj*T=A(I7OL>(%_hr>gypck~-XH1|Tbx_eH1! z0tdnuOQfAV_8m%(R&3^H5`r25NTaO&0~SM|8|QMpu&eAF&Eo>meNY z%Sznjs%QI1=`t3@A^04w%Mf2Z%~%&@8zsjZpC_wxn>9MpydS1KlOZtqqB|9m4ATZO zSb#qP?sg_3PZ`+nevm|~K8;;dJk|EBi#mMZ&-rr`H4m5D{IEs+t5M(Crjx&}mgd%M zXVF${e;!`s$96mYC)7RZ$JM=x=yqXU>Xyb-FJW0J_hjtE6?$mffI7Cgly&~w?B%qf z``j$yHFaD272=IWl~nLRw9OgoqRo!i{y-G>TQ!EArx|f4_)td?s&6o!$cd(3`vx&G zfrIlV->uh>P4gqeUVDnjE&0A(GPF_iZ&K4m&u2MX7|s7UE$#e20SnH-`LR9kjhe*3 zGt*$8PpWAg>kYbsB?||ME)%jp+UX6T+G)cQbUkgQ2@NmDWM*Jmf7ll?DDtl9x-Iae zJ-}$&Cm3yj$t>MR8YfKg)K;Uu_SOTh#{JE8-{ZWChiFO7M%qS#OjG+jk+Alun5TDG zJL$7i>i#*b>$A|)O*OWot}(eHv~2Vo4)uH`*++F`@4z zufb7lB;1?+*Df7nQ9{!;Q=D}lfqsj>UYGd15K5l!a|N{1nOLl+L1t+#|Ao*+lb?GH z^U$a_Ng+ZJTpQ0+ zb?6#Bw7AXCrY*t#Q2>1V4I|E#D{9t<+7D5gJJ2A;dBP3#3cjY_m^{cq;|PRySS6tg z6pfn0pI(uOCU5x>7084#a(0BbnSo&qe>UV%A9d2vji?`#l+lIW2QdR8OFe87`S!X| zwUUt_h;etE>o5zuu{8y5@oj!m15mqda`0m;e~?3tY`A##THB6zPC@fNMzZyGZRwk9 z*59bFy_g2S%|@N+KcFqbx*hCnWyX2w zYxYPdfNg_8{WS<7$h6;rpg#mzcK_K%|3GXg`cL@zi?KxhkCNrn7yqXxt))oZn~P=+ zgnt&X%z5|c#j^M-_={=xXrJkhTr#(XG(a1vr>fHzS~cy!!&(3pxlNGiEQ>rDvH|)K&h{E^PO)8iG_7;~Z9(R@>1rESDDh>+@c?UqkKj zou-V`Lg$(gJR;*iJk`Z#&fM-FkPK54SP%mKDCzp#n?^nX6d||a!Fr$(tOc3~&+IhR zBPkXU7uhI!tbs=0^ONoXlhMGU$Y25f)D8wrX#*K7z@G;u=qi*Nq2681eue%Zj&t`T z8y*pHGVupl1r{DcK%DzO-*7|u6S~Q1znj=kXnxRJ8;->}KblyiS{I?(`JCtkKkeM0 zO1%@zqjuImyneRDs?fd9`}{bzxsWh1LO}W$6L;#nyVf<^19s2Av@y54Ju(aLYA0~F z;`~6}vWGYkuu4_jhH0o5D1v`T)!;^<#ucnX^tsc4oYEAEp&+$E@hce*g(+hkm?WFV_Yr2>WCJN?Ck|9r9g-#(Thv3IP1nzz}` z5ZDua^Mtw~q3G5Om&}2ITL;4UI9LRsl^^n^*%Nu$4Q33v7_!nf#3sb?GfRv9d2LND zdO#%dhd&O87JWA>kj6s7Fnfra`45=?hyK-u1>6)c#S_LEZ(wT06r&L!hI@K#PfOFF zO~mW;1~K5Ymq`RK1Y=OM<1O?T@EmbNm_lk%G&vieSK8Th7V7LQ{Rd;AmbY8zYl&rf zMl2Yi@vxE`FZfD0Y3({+Yz(Z}PNpSF857RV%Mk!Pb2AWQJa(-No)QEBYi1w_xPVfy zT_Y6-S5@dBK*m2bY?;_rijNwd&d8^ksrDH-yFnz@54z=>fM|^<2#0TRHx;&GmWeig z>}9M@Kz`*FU;aEdC^^v&NmYzqK_I)9)G(V(Y>m?0cC9DdfMMBRw@%dRj@+k#Bbpsa+!N%&E;p2B{5o`(3Ho@P%y=N51#4p5*sti^qnnsUS+rL@|~nNOl6ue{%Er{OK?nbR0!>+nT{4Y4GNC9-8i(9Ad@_-A|8&ibX& zwH?1fIyaGk_-Pu>dW{2|#cIUmm_?YgHO3@3z!ua^Q3LUY_tB>~gZbLhP-1i3YGcD- zOqUGk^?_fFB{q)*B`)+Pp;#d({UY(hr+!#H(^fCj39R!nfgf|n>cr1KZ9qXvD71K= z1gcm#(|qa9ZW@AZ3-okfhEsd{(j=`d)5WBncV5Ai(H4|`$Ti+X3pj$haV8^;X8 z>8x3^xUEPDSI>HEBELG5?yWpihgqY8av33!)GaSS-iL?bD!zG0VyeI2>5Hj8se>Yz z(P-)yDb!@$69l8`m{&mYrGlG17AOSc>8ih;@cx5)Ni@KpduY1fkzci?)~z`p&H zlD-K1>D#PRMh8|kh9YaSPR<&h9eFkDr0mF-!-s_~`XmJ26ojH%bxk8AU=~B+UX6jB z^r9gDHzJ4smze)e_^&R3>d5lx0z44|H=G8e>ywv1tXo^n+~he}CuQ5jCX`$z;-HOW zsQVC;m%_K*m$C#(GGY%US10Qy26&;u*+8&deqwGCQ`lDe5<_eWV9&ZN(Z~{e*b;9OXSBha zV`xym8LGcrH-_t#R!d{sr?SD`QuTaOvXMDxf>7aL3U!O_6-QlbR9Pc4)Ta!0 zUd|xBu*i>H{pqV&PTux6Z{bDG$A#YUS~~@;@|P3sdI4)7d%Auj>Fa#y248xXP1g>< zPBRcsjuKW^)wq{m&gCw5H^#z6mO*^GxC|4B+0hG9)Ht_bIAd4x1zRdtsRC%dmi|Dv zN~9wV=Pz!90zl{{nS5NOjzuA|p#~u61=T?KrY_*vKr$4or_9Kj`y}|y`*h^=rSHg` z+C$M;?T*AX{m#b-okxnC$m;Cc=1zgB-$2B$we_$E$*->i;r$@S_>7lVDJ=lZ!kO35 z8zCT}0NB*^~gXpAbm} z$JxGWa4kY0Jd+)h;@p9Xy9*bp{UN0s5bgvX!+zvMz6B}OWa;pk?Aji{cqs`9vJfmD zR8oWgnii?d0THYWWDAA%H{}I8d9yX%aMPUB^a+Su(KJ|K!baji_&n1d%khW%i=5e@ zr)xfDo@Xey%?^Y??@?4J>6(v=ZHXDeFyAhlcLka}A_B`ARghgfSy(sV#964_q{G%tZ+?BkB>GJ<^vw z2ib!ntI4SP9hwe9y@(U4Hx|Vjf!a@SsyppRc3*T^{ zp`<|g75wER(Z{1q&fu|0KYopW%q4-5yO593ty1Auw!C03rw01z+W*>Tpv2#86$p=` z*47@alB!mO2nf!Z{0H9D@32QN{!8DvPg2rw0KYXJv*wcG@D{rh6FHV|7=)Fw)`*N# z&kEIRtF@ww@mHZfD+1x;c(_XRe01{+Y-Drg#}vk@57e58>zY@*CH@}W_JIOgsuEiV z_vIiMF#al<{K$=7J`r^Gi_gE;m*2#Ed}`pYe}ylf|UehLPoJWTkGUJ@*Q*~aIOx6cH3%24p3Qqim4UZz2c0(GZD(sTMO4}^yz zp%m!hTI0yml-?IbV`+tNXEVNz2h@EKsO!d;@Luq2nLJnZEHo|Wjb3vDdaJuf94xXK z;JD>%TOV5;$(;i2K>=vhIC&z zOc%XXicBsD)gBC*m(!9N8KT}9*|~$J7_X<0iayA-WPSPv{g%1>2|jlS1wN?UF1z-S zRJ8oQG?Bjb{Y3L)3h#D6gw%~l5J+y|Sa&N8gCd{emIJCAnBAp$iTO08?smN5b(6S) z7@Skf&ml3~4O5-3C!9Uil1gbh+zXyk^#drh3HSifZk&o2tpeF?+JR5RmxkP|NT=wMT2wPo^miS|V zwfryQiw7AB;31PaG7`W;8XsgJfCt$3unh27N3JX4KtSyzB4A<*!9Eb)3owPEC#43V zoPU6TsWS|Dw~!27_z8*8q3*&+MvqBd+IhqP>~hMe~q5>KDJJ8f#Xa+;&qA3 zP!h=2y$uhZUHdemxpmuAy}>?vi#b}Qx}i(#6V>ME9TM3!M>A7vh`NLdkC~jr(sUJa z{egrhRiOMJ3OpJW&iOg!$!eu(y@CCq2#fOP6X#0CAZW$%y+I1Z6Y-?hrK~nmh@<0BXJk5qqpOqx=b@*N`_<^jzS*L=L0TBJFBZIJdWPVZsH8Y zZdx_!-cMn}ClRbQG-VS+4i;$MRUruSc=teR4FMD0ooI3hcWX<~&E!gX(4M9E6RnvF zx`p&TowNQ15{4gTPT%BZ*YE)}OHxw+9Q&=ogAIls7C%Xca4SfmT1xmJS6Ng z6kLjX68HcunAa#U}l_!>B%Ao-d zqo;?kX|=Oja5f=E3S)>Z19d~_oe_64>tHuaA38-4<-q{9t86wF(K8 zUPp1p$6i4kB%YT7Eivl+`G9U~*)HoawA3rD0 zso$zj0zm8wxtfnVe0BhMQBTd6^&P-hMk_#rs3{Y>`ynHN9M28HJ8;y8{*O4aa@ zu(BQ70tvK;8TdIp)?cnsN}=4=D5nmB%sKzTd95nQ)IX7;jP1e)4B491ADF86j83q1 zpUSrNs2tO%l_qQMQ!ofTZ*SRvUN2#;SKBSdJ8m@OnaU7YgA=70rE~+^#~baXp!%$3 z#Jm=<@hZ>6YYgsKnaa6P9$(_)L3n%_gQP*Y`v{MWD^}q##GXRZ(?9UoA2V3>87~}8 z+lu+sS11KJbYCKjX|6DG(=&5mO`omY&147N?19pe8)?01N<*2v(ojw#+5bki59mem zxRODNsL3<(>;)|(Ca)UWE~N80>J~Lh-1^;;eMstx&f*_1bm3TL+9zZn#xfw59mgCx zp5qv6>O|K(reTK8@Xci!7??Gey*n8Qfbg}KlLD@5{lLD9rg0FI|01V<;GrhxaS@2* zTfwkL!4<08zR*RjZm+_E#r3P(jqlypFJfh!%`4*5nG)yB@WoAqU7@;YA$t&&IHZksd#(GvFj6fzk&eqMym z_h4)Y)Ia1yH;D?H)lxnjEDx*HbA0F_56e}I551($Ds?X(j*y4NY9=2Bnulxoa5peS z)BS-*nySN_{*23<osRB~~H`ekU!qPMcrzrr^cUTlR7UrPT zF{Tv8EqHYB!9EH|7nv-82nrXp=R|Pbx!T1P&16wr>COx|>NIC0DzV8{HQpVoVQf`n zKwCBLX{*LT0$Ad1&TXsGR;C+-y*W4?1gHR?lZX#d?`NvPXfqf104e4QB__)O$nU6q zP|CRoM|X2(r86Jy67kZ8&v|uKbMt894_ezoiU)trzfg=KvUm&{_yk9PLUf(9H6n30 z)Qc)qb`@1n^U#M?1U(bz(*fs69Oc_~e-v2$oMTLXx$JLGI!43-X>fkW?}u=v4<=<0 zN7UW=F*9Hnc98iD8ik39ui}luo@fL{p?q0oJy>K!Bwt8tafEgrSniH?ljEiozrv*jc=E!6FQGl2JK*K44i&eSo@&u&Fy`g;QEW~KdswQ;7pR*;8*95Cl zS$dMtv03fGha>U8xrJ@{aFOKL0bkZ!T5!uYAH!Ue>FkZEBefXz3E@;VPG2_BWS@nB z`WFB@W8Eu2Xnxc_2msTTt_6L)L8YzIs&(DD^B7i~V8CWX2}p01ipFr-WwKg^ggO~g zl^tnBe(^u)Y8nIK=fvP?Fj}>V?pPYm~-I?)Fni zE)%Oe!-#SX0QE6YymssOO8ZlS+?j|um&1%{)isB`uTs6xtqbDyFrFmh<>TYqGlQooe-4&%9rCo z-gLTpPKx3DX%w^BU(u3XB6ggL7cmD`_f%6sf4>Eyh)fb7lb^d3%`e2!L*yP1aQcE_ zykow<@ylC=n4Kx>6{M#La zubjvNS*XD$AB$|Xxv;A1pog7vw{ z?lM^SJ+4Lf*NM+l6LRV%A;(OhFIN*GFX&YdVKSv{nXNny zg#^MmSy!!jICis)O$bHRwsv0mzQp(&BP~OokiN zxXvewqtT2TK@)wVUmHI9&HLQuo(zODP*pAi8i9nl3}|SbnG26#1q8R`3h13W4Z`u} zutsAwKl`VSWI~Vlnm9cK*(JO=5uBJnV+#qzo`W-Z%2qK0xwkk~J{Aog&Cs9T3(WANIC;88JqKB}?0y2}%u>l3bsvS;oQHb> z$sxTg=clN*phS0oOq~Pa%SCEGr9=I% zh@93yQLI`nMxVt3(eC4J(d9(uwK=`aZ;Lsztp0b8xHL@2s>YexV$a3@K~&xSBobP#ZpRDqx(hCt zb9RNA>+#O$E!LNMtH;lzb;S6=H0sqyyA~u9<8)*E@#g7pbq7dTog8clVkug) ze$H>4;qbh+B(``9ZUj!WdUDd?6op?#hU&PX5cY)KWHM_u#wK+7)8v+vRq9}A z1`kT|&GW_IFKIE)L z1d%nDu4}~<=#*+W74R*WSaG?!iVySfkR5p=8+Rn=lVo5Dx#Tn=hc`%^Bnl&70@#iFAvCgkf(!%)mp|+?QAg2PEP2%a2VP znkES9B@bsNG^_F83W@vjE#kw3eG33~f?lQ4J*@N)D(U;ltNsh|UUPwnHx(`i?C`2W zprz8esDyHhQI3TUB+q6tqUHVGNlqCiIg+$5(ENh(gtM}&L1uPMCMEOrhd2=}86 zo|yzvE>yb<@Z#I+_J@pm)m)4z9M0!xU44U^8u-d#EeqN%o|UN=+^L1R6V>|+Ee*Xv z9C%^eTLVWpl96OK{6jRk?0ldFnHuYS0>a_Na=hTuF;G1k{O~#fJp_(%Xx@8EgtO+p zN&rtp1{#>k7O?_3c!We7;VU?}c#^E;>Lrpl;7KY$5pRHF_HCJ8k#Mt&Fxbr?FK zw1zKsVI~07GpkyHKW#HK&ZlOtkoeXKnsW`Q|1;F0T$mEt06W1Zxw;?Jib5rLFYOrV+eV>O~AXWeZwgo#9B z`DcsV>jVDRK<{YsrS)oI04CJ9gAI%hgteVya2VJLH`0SBr+61Wc|6cs29>3HF0Q6&;h4| zw+6*!3YT?yf>%?XLV2Ld!4Uo$Pll?XNcs)Bzci934G&z7kn7_( zVC2v}Q81GATwo`<+S#!C}5@!aVb%w+hhrL{eGR{$PZ&6d;rpCWT|5N%-nF%fy58j6ltueZ}*OeLHl9@Jw)?CU>pLQ8+4+gr_)*RW8|Y^*fWuwIEb zP5~a`D9LEw__{&_0D~f*#SutI!wQ;1)-4{@R?!Pj0ala#tXuTelT{@`u|1(U9gvE< z(Mt){*ANBhnD`p0r5g}I`M`M|Q$irn9aY9?X-`$39>iEr>!vI@zX-yvcv$U;f7TkF z0CC`;@^9Co2e{Tx2Qg32CHztHma8}L&W5+9vKm z6LE|~Kr6h{VH_GH(b81?-{7fd3BBb90k& za|=rI@qctxNq%KfX-U%2`IBrq*mZa?uQJ%R>=+a)F3l?_EJsP5z?;!U(n4u@(U_tV zJeTF=kIx&!M$7Wb3ri~HRpnSbS5%EM9aa?PmFJH|WBGaPtpa66NUg%wHtN(%~4LMo|9ekaJU2!G>VoWM6HBvlkv78H)ot17NcD#@Er z7(B4+@B@QI6~U6y${;d|N(u|mBeT$1*B*!U>^Us0Td-?IFsGKbU-Alq`FX{~!O_KeV=989OUr}(CJnT#ciS3YIB8;Od4XFGr0{5!1d%8% z%&VwmVnQkY=Yn5FmBQz;!t%6kM+FNflvPd&=H})X=T%hX<_7Z%i;I&A3ePD5VFVYA z5-IVpQk_Gtt9^I`CSQ*ut|mz6V&4<~00%^G@~O&&ks)Kg9v+Fu^C zhMhXNU+!@O1`imTbU9gk z*64gxiE9ID!d4*sMU@pnVVSkIC532p0z^+FDM(vzPF`_UVb>nLDuR`z!K#un*tVpk zt`)ErW1!0uU}1{NiVJlytw=4EMvRZ{A=ilU;kmtX4=*m70InC5l)@b46~pdfuDhhN zu)M6i5PyUDkmWIjz_k*>46wQbCRi|SKcT!O(CFexL|hDUL9nO<5K_yLa9e>M2FuDz z^9w5~dL$(c8w-P609B_l1!-4M!7IlalaT<)yhWa^*$T zH!T7nTUB`p#GtYY9FdkQOG|?j@=7LI_P8tzrn#Mf$=wVX5a~D>c0~(1NcKbWAX-67 zounjCA+NGhEFJ$BjjF0F)P~&(rN+~1-&5Jvjq9?$<^hjZ@Ak_2^|EzYp@+X&I2g*7 zh_vVBd&IReu?Px6ca>jNPA3jg%Nso!d=a*bR#XJ5D!`?plAs$X(5HZ@@us0QL@mhJ zCOm!v6Z}aKX%1L853CSQBEran%udLgB$z^#3JQy%Zv~cL;5)3>7#F|zdLj`tIYdXQ zWEtNJm8YMqI50>(DJp>-gkpen)Ht8$4L?MzNY~-rx>g{a&wQ$|j*Fl9_OI9U80{TUu3IfPT&?gz`}Xz!IntykKQsNq!-1-Cy8m22|%4mNC>i zL~!jolmM6K6;%|%OWV9w7S(S=<604PJ^j`ZL)RX?E5vA)l~xq7ue@S6bPyBjkFMCl z=<-6E&hnzXQLv#0b_K!83r811-tvLPs6qq)uV$K?HS;&$S*G{t7Hd$`u6bAw}+p;`D05cp8^Iu?X>NSzhJX z9yVN)em#D**;$TE;$J^7SIAF+z1_JTUv+p?sD`UOA9l?+o7r)-yW}Ih{f<=I zJ5sa?@y-=X<9fL?;OABK;^KF6sr4}!=3$#wt;n~~8p_v=8fUEv3 z-ITTkCqAM)`+mQTPXl^af-_`7SwdColJq!VjgI0}OxubKkq*!uh?mh`xM^ml>74DP z6WDl8;mEi#v6OPZGQJxU^Ni#xZr^&Gcx(O-lF=(`k77eUhtp40-R~oO1>rD)gRtpG zRrd`Dzrr^te~9o^ghL4X5c&`Y|Sg!dt|Bh2ibb3cG^ z6~bE)1j0LhH0Pd!aO&^o-0vdf5$;B~1>p*WW`q}iIOjfx@O6Y7!bg5E=k7-6KzJ*{ z-1q0)pCUYk@NtB%CSP)M2xAES2$v%?A^dCNCHFYOXArUozu5AUdjjEe2=^fjB192Z zKjXT;dD?YH*kf?i0!2^>Lim1OKp8ZIiF zfk(&TkfFp@od0OsB^@T2fl-{gl?P-H$0jU$1CahYb%Y_mz_8w}*BU}4)OlyIeQ&L7 zy}N71ao&d0Z&JH7&`dOJc#!siX&hPUNDWtl@(kMXuK2o5?VGT%Orb&K;y|z1O{Ba+ z>{XNpwM?7cx5u6tpOV{AI{!Q5Zt@X8JYx`xIuc)6a~+UDhF~tmA(M`M@Kq6YH~esE zwXW*EAZcU}9wo}&65l2Luq2aFEXuoaz#-qSjg^JBwQcq;C2;L^Ql3=ly+Kdmi9wg8 z2L`956i!er>9)OXt@gTgs$c?p4?EJjzI8+E#>=QBi;Ta2eza|to4m|b3wM!A(9>r2 zY{aNPGbOhh8R^F!#t9EA5M1BRI9yl|pz;ZSXzVU*nx^f6L1-UI)7!B`T5aeY4!7ip zP-;j{Rgxz5Iuqk0clf9VcEg7;u$vCf#cNPq+4REu>$i6z>2brt zVS*!09j_N#kKqC)uMqqV=}`M?X}6XLFSx%qnT0Yt1lmZ7JBZIfpF7_ zI~?f+@nzS4G~+A4@|9ypn^QP~*+ZiWasKG=7$&qSd=z2*fRhxaV1aZ5Jx_#hCL_Jr zJOsYlg<$^DgI0&V7?&O}4Ey-xzz7~bW_)8ujOuRUeKpj1isQ?LUk+V{3z(foB1W+@80yp{A^lJF!}dl{G9|nlQ6Oc`IY$jsd{v36reVxRP*T>jM=<8&-Lar z7))m-rCKF2cqlwzHqUP#Je%m}uw!a+$DZvlk`pss6BFYL=#}1L5ya?VqGG<>I)yEU zT0av)U}ACgYH+-`Qhd|Cu)JuKNoP>u%@4^Py&oAF1;2rmTHk1XOiGi>kN%DI59_%x zGv~Dr9rf%ptpJECdDm6|vI^8Y`FH{OLwk4$rJW1nd6< z{jZ6M_0MY-y|zI@!W${7PV>Cah6U=O@@3q&O38=tQurz#Zw079<(UlL=@Cz=dfce} zs=OMvT6;*}{q3o3qg8q8ubvB(f_)|%Kb2pzR~+As7a-)>M8$8Qhs(a1-hus%->za? zo7g>oxvYvq5BH_VVAr%^z)GQ-#QA>TIweuh67cb9J3KlpT|TfJ>LUA~=LDr-eDxD} zIhXu%hVp5}#JcYB!_y*NZ!9E8)eVZ$#sd%l5DmN%{HsrP65!9ysr0qV%UmPjP>L2KPYQCfC1P z^QRpIY?T(jAiG!7UdS(%!e-YKzqA|vd~GK*Kel_&Gi3|ZRK9 zC18uo`zkRuu9nq3;}f!3vbh^K2;FR)m*h!-{nSEl-`PIP_L2CfQT?7hY?j6CDmLYy zcv`5lwU|tgWw1aZ(DT|%`qq*37~3&`d}+#odl`9n0|i=%wI6&OIVDKX^?Pr;kB~E8 ztF7>kTR6t0adaUe<7EYUXJBHIcbS4`9>3r-kdaS;=~3EPN??qq(gLSjOjWsf;4jWI z%HW}WFt2#{if8chEvRXJ7m8-fh-~Pnkfw@X3mVRh$@iL4>1jbLr`OH|y#pa%#-oR# zX(7)~V)EA1s2Zt6Vv=T*UKY`BdNm4h`N|j^Iw?UaPwT$qsoyuKjq~ULOa#5_hpn>r zq7wM2{LuJb!ltW`k9YhpzPjqJIZueci&owIg+e?J?7vuutMTvWr{02p z-cD8BFzUB~Yk?79J1`3D2Rgtcune383VeBBmhyplU=~OpG|mH`1(tzpSAgFh@Bt?A zxs9X1EU*A90-ps|fMsB0sOnaMabN`fOaf!TEO0aWRREquzry$gR2cn=;xjh40OP=C zfJNYGU<98JTZ2#f*&|i=dY}V52rL4R1H+@J2N(x7hma541k3^BzyffD&_REI5qyHM z3Uq+0S0NqP3akKQz~}_p0W1L@1x7Me_XIEpe1Y;O(SOLFzPtJiFp7^!3JdZ9wgV%# zfgYFzW`QN(qrmv>Rrh&d890P-%T8C_2Y}(3s#^q>fvduh(;Zbe3M>OBDgL3Vdju%% zM1O&KU588m7Yy z7p*$CDTn{BSc_DUi0FloIdo81;clyW>v^Huo3i5SH(zo2#4Xz9XGhzQc2 zMcS2=aI+QO9~eMsA=(l8k(YlPbeQrwt?>PUPOJG)u*#(I29E1^%h?8GOV7>Lf4f<7%XxKw6YiQ18LO zbt|o=m@FSb-V4ZEM|rrYd<s`~>(t|*ot{tIYtY_e zjxE(oA4mHoXm`;#c6)kJt2bsvf>$<}q_W?6&~6@p{h@ks5gtR@uP?!~R7#NjMmM?N zck2!#YZF2lWzTGGk*F48^FvF>fN@(sZ+HLSEAYv`~F2v0V2T1A8t4Lw$=A!eOKC?oweLIvq( z5VC>J&~{Mtf})ctwu2(Z6dg>_>7%$76-|R?8`ly;Ej>OOswKv?bZ{+Oty#>PT)+Ws zV4u|vQ5*9Ba~fT2!utFDcU0XgurAjb(xnASGmy&4ze5I9~^9`#~*x1*KF)V4uZe5tmTCj%w`eo8d`v!jz9erKKZ$B_Og(zk(D+^P7)a7*7l zdmibtl#T`+4Aj&<(0{xQ~U}88`1#Pm#tDX>b@hC^ri2UMMxuQjc~y0S@4=5Ub0>(XVigH|){mng~}1YH zL}@kG`a;9r{kD0ww;yCB@aPARH?uw+3P2y3N8nbEFc!MoyYCLsf(hmrqu$UiU5~OK zHughn;1?La^OgwW8O_1I*1U zt(Gmm4R}vzW`2SVUQdGWDebq;#8KM)*kkOK^pif)jukq;pZU&f8NS2kU)4Xn3n zF^G}HR85y-5QcwYGiZ`5KmU4yGZYzv+BUE^6o3V@aWGcQU=!r{gU3zqkvu{5*5(V@ zgzG(_#pc-rL1T9ibY;-RsZXF=aNd))fX{omhTy&EaQng&A7;2ZzXQblbG?;o*njSU za}KUJ)OdgJFjW&qXhpbuUDbt|lo#2hW@J1HKh7;@wM()sJve)Qw{A<=ex$(bN%(;N z8E1+m`{e}t>vyX=6bE?$yl#R2C*bL~oFiDa4m7^sQ|bOt;DcVIGX#}r?vvZk?KJ;+ zha%-f)W}UeJ>v6jsX76$;#kh#F_^*JjX_KzDf^85w7!m=(Qtv87(x z<5}=J2LIJ3yf$cE-Q~4m8`*c{A4Ay|__lV@*$gJc?`Oaget`S}`{5Tz1|GJC`YdOg zHPnd+&JpGc59h%3)=-D#%+TNDjVrC8nC0x0k@pdmgRaUopap)g)%V~X-xW$m3v{z~ z9t4Nrjf;)p5XdLN>n8ZtHj)CO{r)|LJd-YH`o~}rutY#4$_93ZQfF^Ac2KZ82xZjO zdZ6k)LSrQBs_6^%Kl^Dv1_{sO9t#Ej3rWx^ufjg!M{SS5|99}CRrhmV+x2VLc#kQWyQIM8HrgmW){cwT*FR|CsyJHXY~dW_B5(Linc^QQFX7GZx8FA zW#Y7NY2Ut}W(+3IDs{}{)=(Vj3;klfA<~;>IGR?&Px}_un>uqWn!?C#^(|DeI<$5{ zQK4f>QEB?f`7;E5r@-%1b!=cwuAV*pr@BoBH#z^otNjtIFF?|Z!vU;73w$$jjC!nc zAb1r`#)V_j9SZ)-O6zpsMynFo3xvxh5WF;iyFR20a0;rJMQDMa`o3@AoqTer$~w_U z_C?C}Ly+FNGDmuG1bUH&UL28Qy0~7f4WdW~iq1&PK`)L#FY?d}O3yVwFNjCJfrLqI zIfk}8`F-f&T-w6g<8XMLJM)Mq&|dQ7);S97kJ- zM@i}^or|N8$1|s^?roRpw$%J&(nnjfi#>391-FIZkm`p+s!uwmvVmKHn3foUS&Tq7 zKnqu$#`T3Jje^P^6{-I7=iC-s*N^=k>q5;(%(_qv1S5;;!uu|;P6T#XrNB6_!zu;> z1L!Q=ex<<8Awhj@g<~SSe$M?zdUhanK=xJoqmA{h2X4FI6OC}f^`lRH(ivB2xCeM6 z;wd1x0t4wq`Z|RIT`1^qK^*x}CHdl_gfdKzh;}@cnk*q%sLs|WtD?NK#W;1 z0GCxA(QXR`f3Q*(DF?67MIXgYA85OQP>5g_Rkp%O9{t3e`;94GKlZx*IpD9q91K2j zj&&Ly(+U`!Mgz*hz){@XfCNl8a76+RWp!HA5emAPhrwPH#XOWnQRM>&1+-xne%alZ z=r+{mVNKs^<2~MhF&xAg_F5+q%7JYd+2Fk^c&wq<*n4BF6S%ns#h~zV@PnaqxyT1~ z%B2_9Goxq*+3H?6u})bpxnIn{U`pMPdMI^6woA%}?OS<8Fj`*^(eppL*gCypyH#1y z4TO+Z1n$MnHAooO2`|EIVWF+8xG|JP-N#XP^S!QHUXO@Tcg?o4OtQgXOZ~e0Lc!b$ zYc{yu%3;`NF=E+Z0J9Dwmcxk6Vi-XDqu1dM%U1AqPEmX4Uf80YvDz~WwB)txH3_^X zf&XU-*k4u^mcOL1v5z)f^qA9@nA@V_A(52v@Rcgw%<DT#it>FAZOvP7n zJo_FMznjaK40_S6@?Xd0Pj;xd#pR3K-(y-{SF8NS{ zSsu@~b3D)b^-h+5oX6v0&M$EMO*+o)xq#y(qkhNGkAj)ssoJ}W=}#K+EP3)uN_iEH zc+QY-+^8>U@Xs3M3kJWBvi$Y@xz~t02K}!%KgjLR8S=;({mrud*udqJJl{S0!Ts0r ze9X`%$Kdz3#{7vJ`7uL3?lSsQ;<%mw{tM zBF6lCx7U7Ezq47*hc_^Nh3A`Ie@dLcn)BmE`6Bbv_C9Io>#!k@B*(SA`(3X+Nhyzr zp-)4eyi|VDh~H$y;|Bl#@Z_cX8)12AeGhYedOZ;=|8?BoZ!^8t*N8D*wlV$@Lq6S1 zujhA!<@*-qpJ#p+#~){T==CXV@cXin|CG_b4x_$yj%#_(vi|G(;>LJAVep$a$`2Xy zC+g8F`F`GL-$tW7j~MyFSiiq!)Zb>%w=lh4pWbG~n>nug8#c!0CZqj%-tX+-`6-O@ zQNzCU8uTlT`fWo$f=2#jMt>rn{FHt>hCOw;xeVOB0KXZmY*#`eG$MyJs z!Jsc2_M~LQg~6}F^3wDFliXjezZI6B-rt_#`Ksf?++RJOpEcSOVR~)fs+_Oo5$3p- z$FG>bo^NHNeOW_3VeX%v{}oj`d>8i3uc`8>-EAm+8;FP3mf^uiz|Eb zDWiYw+#bDubBy)wAC305a9r!h#|-)9x&4~oej~ra`mXo?jOUz&E%WsYN-x%~c zLqE!fJkPK_)$)%U{r|4Pufp}|@qEyb#~&NsTyP50L<5w{9_Zj_98}Wi+-`kD#t;LA{ zJ=5#)KW6mz%SJq5)R#Bfce9cIIfFiI^sm`y&%;LfCL{lUSbloFIAWAPWyn_;R|Ch7 z@yiH!B|=`$ahx7Y$}6b=ue-=Ady(2#>-X4mA5w98L_}AU0wLeE#QhP@U&HA-|2HqG zS6wUg%n=to?xxGZGk3c1`l!5`g9`OK6M`P};bM^Q#Z#P%;h#5Ddzh`5<$`#`DKEUX zDKC1wMpv>&p-w+}v5M<<&eHQHT(48$)rX%_8>ghlxIBK8M#i;V@neB9?r_KPTU#>z zur9#-UW4UHAbv=Vt64gdR4Gx$IAfC0Vazh-81sw;#v)^hvCLRu6azwDVMd!V!Wd=a<8FP$z#sXuJvBX$rtT2k* zTtB1D7-5Vu#u<~04r7)v$Czg)U&sbnAGL{(2j1@+)hwEpw86%8Q#yDe= z(P7Lo<{0yg1;!#{iLuOBVHA70eny)y!WdZaM z{dy!_z8ra*T(9zv=!~v_yo)m3CQdJWPu+BKJX=sV8#rE>Q#U;vj~`Pv8#$i*8+8-q zc$xY4a@>AUwWE*YS&qM(xVG={Z>e~P0^zWI;^?9f*`Vq% z>n|Gh3#0zBQGdy(Kex>Kr@4NcTj=%wlk@v8j;jK?|0UKp-Tw-=PxrsD%=(Lq_1~yJ zZuCED^gn0xKfcWQa5LM@{J$&BHud}CZ&Px}{Y>@W%)g20D^p7I%>4V9KL3;Y^k3lo z?7!8|53_vY@*j|V=T%i)Ja~y}Uj%PZ&}G(NVE&O`C;~J88O{&?vVQ&!<{!OH&9uen zBkSwu>-vlI&-b_%qj%`%I_u=`=lY#Lt)H*kR|wY6@7c2D3cF?3AbwSA%3dF7`@Ocv zy7f~sy8iABk+#UD%Q#txw#*Q|%z_`%5N-5Hp|-taQ*HRhDt!@W#+O82&`qryP6=56 zpH&|cZK;{D%*?1leB4Fm@F(BrMH0xHNRJE<0pB_J1-CZ(*oXhn==g28@zdK$ zm7}pp%H_Mle6R7ti2LcwBoJI0Z5gZm(R3RB5LteD`Ohg-d`Qvwzqvvwnoi@NAk9x- zkX}H%U(8tgB57Cc(sVlIA>8=swf|M)>vcg!Q+^LKy&iwT`%I1YW%C)>^P}SP2EEPC zwKdW=r~H>$KP~UneqDc&KJEX|!FSDXOc{{^s01#m6kK;~MF>f1ZzC z`=gI6lm5q`rS|Lk^X$*gGkxKFWcn`KcTcyc5J>kmz4l|q?^5kA5p!HLf9cZg)_U*} zr1|T&4=XzR9))Eh@?Dx<<9{H}Pp|#sBCF^tx?#*i(`kIsq-TGJxKGgw&oQqU*vv;m z6*t7otA@`B;(kScf{e58vVHe-+knPX`y)oWrhk`_PG#xxqgS`el7HXD^{> Date: Thu, 16 Oct 2025 08:15:20 +0000 Subject: [PATCH 84/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/test_cpu/test_auto_scheme.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/test_cpu/test_auto_scheme.py b/test/test_cpu/test_auto_scheme.py index edfefe818..fdde697a4 100644 --- a/test/test_cpu/test_auto_scheme.py +++ b/test/test_cpu/test_auto_scheme.py @@ -23,10 +23,9 @@ def tearDownClass(self): shutil.rmtree("./saved", ignore_errors=True) shutil.rmtree("runs", ignore_errors=True) - def test_auto_scheme_export(self): model_name = "/models/opt-125m" - scheme = AutoScheme(avg_bits=2, options=("W2A16"),nsamples=1,ignore_scale_zp_bits=True) + scheme = AutoScheme(avg_bits=2, options=("W2A16"), nsamples=1, ignore_scale_zp_bits=True) ar = AutoRound(model=model_name, scheme=scheme) ar.quantize_and_save(self.save_dir) - shutil.rmtree(self.save_dir, ignore_errors=True) \ No newline at end of file + shutil.rmtree(self.save_dir, ignore_errors=True) From 2959aa90b0f237e1d57e5c9ca8ec01ba1fa72ed5 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 16 Oct 2025 16:44:07 +0800 Subject: [PATCH 85/88] correct model path --- auto_round/auto_scheme/__init__.py | 2 +- test/test_cpu/test_auto_scheme.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/auto_round/auto_scheme/__init__.py b/auto_round/auto_scheme/__init__.py index cb1bc4056..f4a3d2b23 100644 --- a/auto_round/auto_scheme/__init__.py +++ b/auto_round/auto_scheme/__init__.py @@ -39,4 +39,4 @@ def register(alg): return register -import auto_round.auto_scheme.default_alg # pylint: disable=E0611,E0401 +import auto_round.auto_scheme.default_alg diff --git a/test/test_cpu/test_auto_scheme.py b/test/test_cpu/test_auto_scheme.py index fdde697a4..f85162032 100644 --- a/test/test_cpu/test_auto_scheme.py +++ b/test/test_cpu/test_auto_scheme.py @@ -1,16 +1,10 @@ -import copy -import re + import shutil import sys import unittest sys.path.insert(0, "../..") from auto_round import AutoRound, AutoRoundConfig, AutoScheme -from auto_round.auto_scheme.utils import compute_avg_bits_for_model -from auto_round.eval.evaluation import simple_evaluate -from auto_round.testing_utils import multi_card -from auto_round.utils import get_module - class TestAutoScheme(unittest.TestCase): @classmethod @@ -24,7 +18,7 @@ def tearDownClass(self): shutil.rmtree("runs", ignore_errors=True) def test_auto_scheme_export(self): - model_name = "/models/opt-125m" + model_name = "/tf_dataset/auto_round/models/facebook/opt-125m" scheme = AutoScheme(avg_bits=2, options=("W2A16"), nsamples=1, ignore_scale_zp_bits=True) ar = AutoRound(model=model_name, scheme=scheme) ar.quantize_and_save(self.save_dir) From 019991fbd4972a89c6788010098b9d2eaa23b86c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 08:44:56 +0000 Subject: [PATCH 86/88] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/test_cpu/test_auto_scheme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_cpu/test_auto_scheme.py b/test/test_cpu/test_auto_scheme.py index f85162032..a0eb21001 100644 --- a/test/test_cpu/test_auto_scheme.py +++ b/test/test_cpu/test_auto_scheme.py @@ -1,4 +1,3 @@ - import shutil import sys import unittest @@ -6,6 +5,7 @@ sys.path.insert(0, "../..") from auto_round import AutoRound, AutoRoundConfig, AutoScheme + class TestAutoScheme(unittest.TestCase): @classmethod def setUpClass(self): From 76f4f8bc92805f2a0bc0a11caac36323911661c8 Mon Sep 17 00:00:00 2001 From: Wenhua Cheng Date: Thu, 16 Oct 2025 17:28:07 +0800 Subject: [PATCH 87/88] update so --- auto_round/auto_scheme/default_alg.abi3.so | Bin 300664 -> 296568 bytes docs/step_by_step.md | 2 ++ 2 files changed, 2 insertions(+) diff --git a/auto_round/auto_scheme/default_alg.abi3.so b/auto_round/auto_scheme/default_alg.abi3.so index 052e98273dba44bfe573d8f404e196d0c4c327f1..9dac1fe7b1054253ed73a5c0257d4a206bf251bd 100644 GIT binary patch delta 100252 zcmZ_13tUvi`#-)1L{Y@W3yLTR3W|47SMjn5D!CLEnVM!MX_l2GC6rxkNkG|F3Om`c zJ6-H!7YnruX)4||Gs)CUQ?WE=MFsDg7yj>OW)2+Y`}_NP#h&N=%slhl=9xKj&Tdz~ zYISu{tD^3jdvT9}O^e6c_`l-CkM(tTU*^vV^V1e0m>NDB(Paq#`+q*OMvHdlr?p>W zuu3M2ekD&d{7mmG`q}#MPraSivQ^TPqD|5~^HU$9{n~17q9E%(^>DujqyK*;w-J)l z^|$?ox0&=+VPtUYjZHf5kDB++>L(gMaH&y=e%kLRt-1bmU^~6cKTMmXpKBbp`kem{ zep;3Q8^>$C0Y0a*pfEkBNsX2Pe5&@hp5Wh3ciF>MHJzzxf%-%K?N;YBOVqSc`te5X z^l~&_?jQC>bC(uIJUyd%n4Z+Kk)9RUMlWg}uU*qOHh)e_=3}G>XJ|j_D}zUCyGg3% z&|9|Xt4-IFTWrww>6(4C_C9ZyVb9RZAm^Zcxy}N^^ray#EkZvYGFlr${OAi!ZMK%S za9h-C!b#$<%~sURW@}|@VH>^rp_aQeeR6B3-m7)O>UUZnXx#YjX^+pCp0#>r_&ja( zg$^e*w;7y38IzC|;-XXr&BV*U5_p+{KOiGiuHZHOggO-pUerV2H3E0r%mI?ROZ{hw zPZPIw6?BM#m&FP^O2Km^9>;27UltS~|bhz=zRo-PHIDR_oMv@cii!~}s?D0rFlXbo^0zb*v#%UIgv zWHdui(pv}$QSf-FP?Unlk!AQ7uizmv_U2@Vd_KOdLMBVP(jiV-kfGoc^4qT%!O3Xs^bi)u7%K<_Z{Ge1Q zO2Mzji~jKnULh26+maMUxfGPH;Jc)v84A9{EJO-kBo)n3@NH5~5y#D#RTVN{ek-4MXs@83F|_l6aJYGqVsiz!CqF{z7n41BT5O zEC)_k@PkrNhJx>qc&36EOZhnpUQ5YK`B$VcywZYV1-~Y+lG7bUYjsh+{7UxpNvuHeN9 zLXivwUv7p#!KX<@a}+$MuU!9&6vkxPp;*CRk*+II@FE%GG6gRmAQUZE@Chfzw zR}N64-~qCO?b(JQsFCwNM8R{ULQx8yDe-uT*X{o`IY5%qAxRF9uHf+!&rt9TIY6d@ zr%ODiA?~(a7$__%YRIsq%h(sIxNKjd;5((O%M|>k^k}()7x5fI{#7W9ancht3O-W` zvOU)@1a}S;bcli{nITZ{3hB9c!YTjF=~pBdi6o^%ro__~e3TqGL&3|WCo&cMjBKA{ z!TJ0vksXRG44QvQW(X8~g!E8}f-iRnPn0Qmu{jGAyl8-2|0@(mrPQ=W!LJPx1K8$@ zPt$Y7a^MgJ&y=o?QgONJ#?Mvu{~2VS%?(eX@)?-qh$L`1rL#Uj){xS&Ebn z>C)n21+Ou8L<$}vLs_QaWm0~*6{q+|Nk)a00sNc{O^t%b%fe!NpLt3bljG_8JAx)E7s#bB9Q7$V&|avDtLEg0GW!yn=^Fza}a8Qn{l{SMUOX zyKNZ?qmT4crh+$@1Lr7sfaz}q|3Tu#3T~4ImvG!tzE;W(WlD!zax#}I_*`jOg@Ru( z3%`QTlDz-!FPui(REds|+^kePIpf@egFc8Lm}1e#A@m%=FC zB_{7g1zzY;SeW$>e<|*Z?LN;8kbco6q7L+P@j>LB;c+pBR zfGN*hp^91yIjpX|EMMuXMF(>rv%|ZxgE^3ied@bY@DY$K|z?Um{kUKn^n`-fY!bNps=~M_6|yZ5ikX&~f&vxX6cnZ4>!d<) z3LYgNkH!nU{`ewuv(U(;bof~cN>}i7>FS9JUb0ol$yD%tQqJ54IO1;#%4xvBv4#}1 zRKbr(LB$FlFD)xm@Kdt=4h84!DgT(lxGX!=D0q?_AY@^~(D=6zbeMvh?GqK;F54#o z=lO5OI9(2qp>$|3JIqvYvweYrcb4so6g=rC5t^m-xR`<Lhz|p22~5DX3JzO+jS} zZpt~R;7%#0LcuH6it%ksVT8!o+X@=GI!y`+RPYd)j3EkMv{4uqr{E7rIq`0VQFc;v zNLO(46_SYxUL*(3Q1BWV%3=jCmTN(&#OsbP$|SHu=};kQ^V7@~Qz53b&yb}z$_=g7K|_YkEjtt|c+zV^@NxyOkTg^9cFzd;R~7t(hj7F{>eYst9yFIr z1y7Q+OTlZ_ivcDoc>H$)pQ+$EYXm-*aLPZ^r5UnT7b_j&I|~I%6}(K+I}|)=gBakT zf|pB$m<8wgUnVuWYGIIw9H~&0x1opTN`(>?JW0|n1+S5%eWHSwtrYdgHdA3#NDJmF zc(Lgr1uru_q~em^q2NWPhrG)1MULqrr9+bGAr+U|9<`{Uhq7b|k5llHG;_{2IK_NQ4G8FdVs?Jf&G(}Le^ z!OhnnNYQ&NxOpr}c!mYfvB-IpaPEJ67FZaMTRIe3@L3josRfq{9Oo8W@GMLFkqr55}p3tncyU$NjjEO?#;FSpo&WN+%{&oHbE8fBg5Hk4RL<>&Jn*Wk4I1O(8b6Ie+wh%Afg5$+g z@sHv^(Zax+r*;1_EO<~IYO~F>;K3F=(}K%4kU4j*1#f9-p97rdA8Fsp!YHtGXl=oZ zEVz78m~)p}@K8(pVhi5Zf-kSf-L`fXMoB%x7G}XqEqHqiUS`3=E%*)#-ob*GTkwt? zm-FwSg%M%tP+`F%EjY8_oh*2b1&^}eS1ow71-H3hsn6cd7Dk{2?_$A2EO@L153}H1 zEqIg#?`FZ{B<`lz$5|NhmJZ!5c%lXGVZoCucux!Nvf#Zfc)A7eQ(ylu3oMMjmJS&f zJl=xOwBQLAJkx^rv*2?rc>g*a@z1d^9CZwoWD7pff)`ovL<_#uf)BRf#TI;s1z%1$ z-4Yc5+EqI6pcUkZ-3!ZAhqb&Fx7CdggTvR9ocUl3_@u3*&xEhXM;e*@72Y@CPjTQVTxCf)`ux2kUXSZMlUpwVq)svEUC`@KOst z&4QO%@aYzOhXsGwf|qj~_EYR z1wfBi=rIbNqR>u-b|`djg^sn*Zi;z?1t2<9p@S7VK%sB8ZI~mM6#9%ppRA|dBI)V@ zPVZOf-3q-`q1P$&cM82ip_kRsZjtrx*8zfFqR@p3ov+aI6gpd>pH%1>CheBfZi)#= zdb~o9QRozfb}F<(p?fQItfcW8qp&zaX%MQ=!3rIq&^JSs98l;p3Vl-0sKaLAQ5x)5 z=-mpvRiW1@^mhuqLZO#&+Ff7y-d7qdQRqU2&R6Jp3Z1RcPb%~bqTQC#J4I_@XWXLzQxtlG-tw?{4?2& z63VI`;-fp>c~7sN6WMDjZvI`l*K4y{rVc)mwLQ)AH>~>aBY#^}b9V35+FIIdovdmJ zZ-fTvgR{;Cwueu=5u@AdeY1yY@9ML%6YV>2;NZ%=S)0|tRp>nTj=nm(kM@rKclIFs z4SzNPf770Q8h_V6`&eWOPA$gdU(RX)JyShrm;-d0=ZwqyQcM_&(^H=7qBYVVd+tH^ zFF1|#I)9IDWD9cshQFzv6R95dN@cA-2J;3#g&kj-=U-Q8vMtH}19^92WpkG|^n228 zZEbC>yB1~G)pR>FZ`Bc)r%?<`xB-Ja0Uu+jhh6r(DSV9Ce2gdg7z{G`7;YM*8SOSp zU(w3KXg3S#lSn!Z#O1E-04n+ziN+||k)S-IVghmfr@Riq^^_=qhv7O}UPs~jJ8U|s zeH?rDI#;WWsn+)9*vosJf1+`a^AG%m;ig`sRu)Mw-DpdSqMK!?wV6cKsTJEmt#8N% zsn$q2WC-gnhWr8x9{0p}4C!?~%14|*Bf1+7`6|hTugiwfDKv(>r@@d<&>+GSY!Buo zoIKGKeLaXo2XoOu|JRs!DLH94R3OP$$PF9arjvc)^`f(d=*Fh#kGSa77(B?i5`Psx z-96}q?`T^1JbJ3>c(Gm5MI|#|&qWE)vjCOO;bh{Dcysu7lkaIY$5rRgfRjJD_ zC*WgBmu8HoL6Ab|fSuPHyq%j+nY+Z(V^5^-SkmYw5qy;9pS!f`vH!_d4H3ssuR={8fCjg}Xh^&({1M;hS7?YF!SEchj7y z%k9_65~=$I@)52nw~xtd(%manj3QiKlj_rDUsC;Uc}=RXmTi(qn2Wtf#l{mrwV(8N z=j~`p1{^|5+RSXD~2IPc(Ck)s^y3`r)7P0H}U(PFln44MBWu$+Sye9qsA_qVf z(!V5pnjpx4Hd0YCV5t;D23({ZKnH2Skt^JQ*YGf%2cRj}zXR95eVzV6P7T+{pl$8+ z;+I0*wPbId*1vI)yqtV3Rikq9j#Q0QeoJ1H%HbB37szYU_`J*z()fEyNoYwL52@4G z8?c^R<%a7?kYuD&%ZbcinDP^z>2g#0V$yq^$*>(00DqNGyhN&&Hqvsspq|3E(4Gy9 zx=HN1P}rq6$*#lg`U&O8u>VO#!&v@h?#bKD;aB2F$LrjTjzP|i9rTm)I=h!p<2sq6 zrTQeZO4?2$U$>;^m!y*nZ>_}1R~O{4i9(3?$ zU-yASgPgs>^%vZo-E&EoI$!so{SuTWU%$mgiBd2``ZNsqXO_I|Vadw_(qQuSCV5T1 z-X^b;aD9htPrjaD^>t_2hdQ0MCx-MnXBjP&(s*|}J_k2cHXTJlg?F|+9D@ubbt-$BS7WSQ&&Mgqd?XL%O!n7 z!TO4;CW4hNeM!NZ#kCN1@1P}E^p+0=+D*}_kpxBShKxEzD@29eZ#l_3Ds*B54f^?gcfbK=f)p8Onke-)u*&x%pSoqc^Wy!d- zY#5ilePcpG?28!Gf zM`vB5qZknfhjOS{a=Q!Ophqika_vVF;*7#r!y<$PT zq~@sZR6UWD{|j=BZ5eoMvP6 z#D%F%r*qH#&L-=xF6`~TIe@o>>1c&K4YhBj#zI^AN@c zRq&yM>Dr5KDG4X4+M-E2$WHT|BDf82GWUqfi;O`48kqqY`Zqk9yCo{kQ!#mpKCqyz z=F-O$bkV-mpDt+Qo{cg2&3{uDZvOiuoT+*qa+CSc)hro76fU;s(@RsV;vW&my}q41 z5Ohw#CIf_Hl6-5(N4-tRgER)jKbcEx{l#A>2OyG2GN6e=lC* zksMzrPJv>N*VfVqH@k7|vXBqetOf*m@phNz7}Jl^s7SDOWm#2-XWHQF_PnkTmgc!d zzI|BV=n2)%>VJ4{)9iZNSG&5y#1OwN5kpLZHOPP8(P#`&+r;H@4o76zGu-@uphkS& z*zqbLcNGS>7UDCZjD17VK`JHGuECUrxU=l1BYbdA7QE$K z9*LT8879oghhDFM1;JnVN%LMcxuc1@nbOh|@TSR~1um_mc^4>n!OH^4z=*Gpf${`z zr&hTe!_o?4XDuZpUtnK=0~~({7sZMKqrBo;ID}C1f1bbk_&Ko zit~rM@@unZy*jel@~Eu7%L9_Kny?r2cNcYZpMhVhLRn+zS=GY+!<@j{UtQBLv$y_& z-n0aK7D1z=;J@u`G2-v41-O{N+|41b+^vCCi|qH6vCnvJ1vKHSiKtn4D91 z9c=?Dm5sF%oTVnGoH$AJq?p|l0bF3Rz9*IqIJu}-o#(5otfy{rHCT1WTG=pvV@ z3Y6EWHU1)&m{VOSbucd>ZEFIwaXfPkV-$${+s3rw@+B^7dBR>W_g@6_3BeqUb582^ z8j^f>wZAPl(U#qsZRTo-M0*#9r|kk1bH2IH>MBg)uvEz7ZRDu#u#d}6S7wo z@ldf5WFO>W4RZFuG-Mxjp-c~0gI>Vxc^#-n6sl1PA%%|TdBVD@|FqSI21BwCm!iiB)}hF82b+{ z$3PdTbPz8KJRJF7KN|se`y=HN%COku{F;`NuaA=71CHJD*Gr1R+=~Rc^%nOjsy~+- zI-X>C9Z$iK4X7%Rz@~|opO`Ig+$3{Cdy1CBZ1pV@Max&rmV0^2h~%>v!`u6QeB6hf zhc`G!!yViP+*RA|P4)bVn>d%}5Y^Ue2t@M4OiKhZqUY@Uqf_dlyT#^dN@$ z^QQDpGd_&tPb_TclIl4q{Av!9EQVQSHa|><>R!heXkOnIH?Jf-l(%&Uyk<83290GY zuoryHdpZY0A)fy(njHTe=0=aZA+?z&8nrPSeR36z){91i%tjMMqiQ4ymvrk28oezV zh2L;XQT;`$pUqa2s8yC|b@RGx)Iv1ciuuO^9h7rox?R4`&0InA((8PXntL{}#^_D+ ztg02YtY#DVLg$!`7?@EUbi_>-2`Tg1p`F*cl%#mJv8AZe=0@3xx1&i!8zG`MM9}tR zd`i6zZ{WpexVjZZ*y}v}pU@!}MS*m%O98)QVW6J7@QHYjx=HqX9l?k}LK#nVw(Kvy z8RZk{1{dnwM*A43)EFw6+V%vCB+ciN9J|Qf&~Wppdab`C!@Z6YVtJjZQl)c$3YCh% zP4l=bS+bC@MoO4(k?^>X@S%_pC?%9g2~)X*N@gPg82c(MMqa0n>cp)6^{U<>T=N^> z4>>m>lPElob180nL3bJi-PiE$ju$QQd6y@E2v1w#3!Z%N#SOkg7WXm#wG&%l9dXWSOxK9drNeFPl zLQ-L!ML@U^&>1WU7%2r*9gy*FD6cjO!h0kBd2fg`{#!WaOS;2D zdfY2@+1Eno@ChW4PVeHg*0CE#(Hb5)0P(1G45Z|xRTBw*9-%>cPbbfKgJ0qwjuZ%I z;F7jy`J&=@UC#f$76Bbe0Oo&|m=@nj2~(tmvv^=b5>^Na-}BjB&CcFd zufkz8feP;u%j?XN0y;_oE2Mx~Lcn+_V1-4%JHc))VTh2>UrIRqyD(ywl+aU1c#mgE zHJfFTFiEKJ3bDM-^Zb@?6MIJrxJwE+B~}1kdZA~%0C!%PkT6$xVWpHXNlFNo5MfPQwcuh~;%=N&#U~z%SsE8)uRLgfK%0x`r&HRCXhaSTa)0%;h+$ z1YYMVZtFJoF}Aj5F26woK}4hw@dHGVWd53l+(Y~&lHxteVXyP#d3+!#3ik?sJ}d(mn@iI`&_6@67w~6pe zcRm4}_e)*;q%Jq`5FI*9W@AYLthz%;+J)I;>e3Usm;(+J6(dGi7A&QG_nWY61Ei6V z04~J&0TPK#(igoK9{M%LGwm^CtDp6???ptM<{YuO&mfLB^mzmiQ+Mx0fBn*XG43a@ z>fn~V&8~V8@n~4pQM?M1^enk|J8J|!EdyQ0nlj5rn@682)u&&5!DB6uB}$xVgMi?~T-cUwk$l}7Xi`~(Fd|0ZT1&Koh9IZA6j z3id|RhnA*c{rE7BuldZ!>EAK7cBfE3Ov(*H z>x7+a=pGFxbx`RLspOf3C+*uu>2G}y>9$4I%R7X}=F(9w$~4*zh#dDhUjiKSxkXMd zG=W8|{~j^i2fWo4mP@Um#*6r*lfgNhu4Yg0-p*0z%{I_h!`zZLe}Io@CS;rP`b>C$ zD^tzlxvWZd1c%jTKW?~Z8**nXdbm8B`EJ@tJ-ngS@Eu&3c-~)s{ll0CenY88!xwgt zXpxQddVrZ^pUokMY-f*yPf2zQ2R=N>1`9FsaW~16Y|0mmG0Sy$a&+}2%- z^n_=NxKD5kd<793iBwGGkJ*q@N4E>npFlMEvJWjj-rzmxz;mhvzi$n`Pe=}-&=)%2 zK8*psMN4yX%|UNSex4VLO_ZApxZDVP!6Yg80IzLXF@Xs?5e4Yy zC=x@Hp_<(V-;v3|yn8i!iFbF*fLmrn>PD0NA8v)-Vh>2Qn`Z>zYnA) zLZju>#OsI?m2eW*V-wS)4xOcddq@C!rs9)jQ3}ZJi$oS4r!aaQdts29@@tbUVtsfR zL%Wcn*c(vjVbQ0O8$zM*2CpFBdjpQbSv>a~r>J#sB#(^a4}7rs7}w0_LtLlj1(@X! z=BiDxvDIRs`igpB53~f5@H~S# zZkS=m1UH8V3s0P zF^`OMHiS@qx06G>#$8g$9_6i^VYG^@)q8vz?cNFTcvOXo8H2bF zhcW_NvN&1J*Pt2zb;oBhqk57axz3>ox6Phch`*o$AjM|S zpGK6|8AX|lhZ!NR?ExXw4*%E%)Ggegk5wx_j~=}WDd5Y-K~@Ug zgDkbLWIx6zbnK%o4_>fdw7UHl@qStd z{f@6X-qqrWzs=|QskZhJd#5};I;{`KsYAdU^fCBqRw!hAuou@jUv64kVegbf4L8eA zw114ZcT793fAv*I?RS0WR}np8f!VTqj)xWJ(HD+7iD%GWXNy0e>$Npd^Woas+HCm$D&uJzD!zm9AC5W%zbm0$O3JOGz$ zfbRP`vRm?WDCYT(-k^p{VP>O8@L^Bc0i_elap_G=DD%1XKHm({F6xhcGg2$jS9~*k zNclRd`&px0UaV<&M9KR8$brM?H*OF0OT$Cx!qA`$F17m%DT*!ByOs>n_UI3mv~@G| zG5tng4RLv&i-$7wXxh1SD%5@M2Wr6ml{^lQF4@0pd2F(qj`CC_KL8(MpwMf-LbwO7 zH1}iMgci-16Ah~lfuD@v;Brs$IM#l>Is3Dk;>mFGEGZyhB?&-iQ{h(_b%NQb_mM{) z>0@d~0}XXPhIaNh%WyTEUu_E9g!kizI#WSqH?euESExc-$MV5nyea~*d4AOzt-U^A^_a#N4k5hH>I+v#-2aO(>?25rVF7GnDKh7G{6R8i zmm^k9&;qLu0v|nvx5a|}JKRvUx+r-ZvhQ>to+f;8(x1QMA4{ruZqf{`)o-ot5Isjq zs)VHf?Gj_zF;>+9{!I(^`|2Uzw)boN{rymAi$3Z5SnWaG{e36xHvTtK-|&5^)<$po z!yQ4XQqnK1um0c}*qiB((c zwfp`7?h42`t#AA>w%cbId_7fOlsl?`2Ox^7K{5&@oJs!2W++_`UegKxE5F`rdTLws zNo%^bZ~jefEfq3+qm%nn%#naIaOIkA+AsRXHSIKUy-3&A4h`Lc?k5Sq$`8}; zS=&Q9r@yo|OgpGAS=&u3*VnCGsBP9$ewwOn(wF_zU3*#I^HYL$TG!SkwCVxJry@JC z5M=MAiJhvCTGvN=RiCpiDdIO&1g!25`ypAK`=pJx+fQYDonF4KSLYI_Os3t5Rr6u0 zAD*rFf(AE|UV~ss-{>vYbkd#chh>z3o10?OXj56}@QyWo*1;5wf|t7`Z1NOgJY8d) z$y0oXzQA0mo0=}qbv*40ad`&BK+I4Q<3SwRDE-g%@mj9lY(tFpwcd9_c*~LKSGB*0 zPfVJV{W3)QM+K(m*Y+6Jfx>>42$7xUhpC@INaTVEg|FHr8?z7MP`kc>jfLTw_fLO z%j)d%T%bY{?()7I$hzxWHaf?7|A*G6>&g#iS1OufBtJ0{dtrHPEv6(|?SRbWajBjg z<6WLh!g7fD0_J^=j1I{fDUDdJkKfcQE(n6j8K}=wP@o}r=TBVls&W3Z7E`6YsDHkx zLtEm2cm-p@h5(ibW7ul_(556TW$$k86F%}YC~&3-Qkn`Wogsy~4AG}=9;A)cJD}0# zWo9GKT0Ydz_libq^nW&Yr3~0I)NP;<rof`6M_T6*?gD(PMaPf*rq2nH@!NyN9s&`(dOW(33!u=(vs_Htq zJRDbGJNdCu_NS0Bi9{DVein0qy$VG{5Z0Ot3i8&}W8D8+k8HrE9?4RVXwpN^*cyFX z(Eme+`=kz0T!$ZjIvw=dO&zpW`r)m8M&y2CdOcNi_*-^(fyP1x@J=)% zxRYUAv}kuqwi~P8xh;A?(#L!*w81GbMJD-ykU#-KVlf+BAa?9oNhp$3XnO0# z+hVjG`qpjHLr`HvZnEYu0s*$?{X=!uI~UQ}$|{jRJWK6)-*VWo3`;f1J)%c#Zxdc> z^0S_X+gU2-(f6;8>341K;@$)RMf}ZT(t++kvwHEGm;E#8aq}apl)YlKY~1qWxEW-c zhj9}`#bT3T2yX$A+3#2VDN8dONx0_>nxaE8)wdjIho}F*)4aWh?ZgQMDx+Sry+hPT zZ~%>*hJret7A#&+6Lw-qs;B+*o;#woBz??|r14F_W~a~|&7Xya_(IyS9e@U5!Aiy4 zr@ehoTGI-HUvY6se0}$v;_px&M~Di)_qW{(PvobA%@@L5`rkWJBHM`ORj-qw6X6be zac*r@SrPfcwK`?zo0@y|2eepCt~2Hn(7weOP9y-+iGHu1&L#SP_4FK~SJl(giC$Mv z-%Iq4dU_<$zt_3sSMZZg+R>Gre+m{-~07iD}QHpIoHfK33Xl_9a{Zu%IIcAE0fM zKeu%?lz*fMcP+(O-f;&Tk9_{1OfUJhtM;t^$FJRKm$4@t|C8a4dj|V8d0W44PuEcU zJMit`Gui(WlWR2%a9w|6&)8P!8<4(TZqQhBpM#`{8}!S29AmY&3CTJ_+p+v7$x?4X z#5a_(<6^>Svsmc3OkU*~JdV8MLL=;*Z>P364aiT#g)a*?3+yCh`5xEjm5+_T2`{C2 z{*$l1Rxbtj3m6Z(vgeWL5O_AL6MZKWX>fqOrq`Aa(~j%Iew!EdyMb!Br-<#M&&s+I z&(;#X9d9N6`PvB?_NC z?<+srI&Okl|K3KaP|dzr!u_BBG3Q|Wc$SWg20&$`&Hk_Sm3zBtC-sASUvkF*t6DEo ze2pK4tI#pvc1S679&louK#XIaz`Y4+snc(;81yj@vWxf#)hr1?lCQ<)Q9D-4JBo9; zo&e|=z0mO*WK!Ne{iaaxj-wb7P6aEMv#QyBlC_1D!C7db<7rYP3MSDeiE`~zVtF03 zl+o@GhdO~QNsrjqvsFjhW|Q--qY0eXXsdqzzIeX_Tl7WyMvY&D9WYf+&G#DS>mx&0`*aM3WP5&p44Ry~F)4NW zxlwfU<=XgfBeq_jy+3@=;muGCh6t#^2E&&q2eBwP3jm2b=dp>V zjxV5d)sN<(0o#Sv5KpD-($z?802?K&GmG(her68u72V5)1KdtsvN)%^|OCQXp8mchhp86Nd;(* zRUWU8uRuV=t`l$62eey8Dx4+z#*v(LDBzy6=7XQ(_4iD*5S_D!}^bhduyNQm4_qj%NOyWJWMxP z)$D$~^Lw*vv~w`G_yJT6)F!k=Lk7^h-J8U@hZyJvurt1-dt8@(F)D?-7Hz3bu_k9 zG5X~CJFvJgG_Mz*w4wN@o3t}PVA8g7aG%hzpPq8Er(geG`ZFiTXz%M~Cnp4a zjjc&5y~DrlqY7XgRj)v@!0UYIOMiTurYvDEl^q{zwoo7UZ(0vP7JZ0Y zfY?yEiq>aUru07yRa~S?=Z2LOyT^b|n40yKl)Oo?d|p3M8K;Hl!K|0xOWpOMY=B>E zfBk9JNn4~ZW*zOdZeBz06g70E{u3MR_F!PMhE_q2tf8x30c;-qy+cZ6&&@SuUsv@;N^xN^bRG(25-2s7hd56WYr*h5yS=qP|!EWO# z{iCX|!TsR?ET#C$VWw@*h#%y?OZGw(f0tz0@1MZp!JNY(e9j`J&v(Qlo;yU!cspx~4k?x*vV%HyyQ zodlg+iaJlWr4f_g`1I3XJkz`12^0~0J15XJxGMVu@@oBSyqztc}uBt2?&+6e3`26kqj!#_*J@f9o$+$7#>&pH}zsyBe+U zt#Pf~zuwU^=3vRXDrKXooyTc-CqKTi8tf8~6- z)>GemzJK#$*mq$ENh&>{x4h6T4l5E7>mbCu{R$ByZbDP}{j=@3X~?(`(PI&ouQ$um zhQ|Kur7Jv(vgcP~SRkhf$vZ>@wERr}--TGe#AW)g7urVJAc(c*1BhFPs(o({N{ZxeW@+{>z$m?m?9OJh0d2TeTZ^AwTT6+7@6=+2^1Vbx zpw+3T!2ce6C=)4!M+o8Nok)0oog;9g`(L}6%c=dt&{42_`CQ$A-}1O#-@I{;hR3ZpBHa_8kdhg9$f$O<9Tl*!q(U;zgj2b}V z5swebxQIR6t@A?QnZd158b|8~ZjQjR-2PU7I#s_F*7{KdLR90dM`3cpWwuP8duv$h zgOFYIkKAk0dd;@!+i!L0ltcE9kB7je5!@9p;cFyE)!UR}d3w{@5pFlt=Lcx99~$sJ zI@t5I+F&hgNwyZHJ-6g7Ez*4u2dWh7E~$gJ z&6|)~csq6@I5EDAcZ%?BTXHDZFnbF&6QyuB4%6^QZvNgCfu`?KhW_v}d73|^%zKR1 zlB{xe9sSV{N3nb1c>efcCXDud0SY+AZ8P^v*U;W+=xfh&k`G|k?Z#+L>m4}{@(LXz zr~>0|C>%1y_z$5od%}1_(|T&1jP;tf&^`K52z>)MP0%4As{X*dyY`a2YlWD6$^%}z zha#DTAi9S^vqNZx7SqGUKtWJY@W;xz?0ZC7P164&VlrI7KE^n%T!u&D#uh&;t3P58ib{YjO&_LBDaal!o%gvzh1$s5?7`!Q|7sV80f`3}_?N+24|@gKU~i z!Stq3%@)I@*s}8IiOeD&eHW5-XgJ0IIYuLl(H?_rWiMh7dPy(%Ym89!KDn3~%lx$& zUFVAlV=vf`xk=B97IRr*tFV#gYaz<85#LCAamdQ)FeSh2W{I=Q(Z2R$!fIaVhyvns6@9v|8Ga<}4*+_HwwTm?R256z$K_ewVi`Kq3 z9;J^r4G(=hV|+;;>BesQ=wnplBdYlfJ|dp+(fjp$pDccWh>Fn_cE*Trtab1UY5v2Q z##)%>*X<4C(I#4~R%R47(IT~E&$(S2}gL!e$xgnrH(9 z7l{StSEFyB)}h}tG&c`SI`H=o@r(hh(h%&iXw$R`d`K#&q?grbsy=PJ6sXPC9x_e` zYQ405M*F5(4ASzprrPc9oYyIaj%%;c5Vh>jX`=5g-Zylb;BLk;N!+!PyIF8Y39hq1 zaNn2QeS+Ina37U@-IBXga03N*=y5slLz4Z6V4ngEwQ^w+4LnkEXYg+^G>XA(Yx?WP zZOyb!TD9?DGgJ-O*i4IRn=TsWJL%m*xRC#F#FxgZX4>GU+>2LOvxkj~&9u&Uf}~Tc z5ZYPZ@sKck2u7DX%g$U88p@GII(lbLOwDbq;Ri45d88yi)E8gUJZ(G_1Qnh#+(D>W zV~pY;t&O(a_%29G?9UCTWP!*`a|8Wg{UC4Qy+D{bmR4V(%Xv1&h-$8NabtHtmUqq& znw9dFp=-I;j^^0<5x0@#F5}#B^b>K8BFSS@MNj&20^sK*OpZ1+xes2#?)m65kT#el z+uz)R>R$rN{$|IrQGpfq{C}IG#Zbq0gysE71hR6rakRM>Jq7}~((EFm8dtMdyt)n?M(fn=$tG1CAtcAD3Fydj!Y}_gEH51k5+|$OqV67cO zyd)R_f7?Tp5DOwOxDtX*59%{D6RTGu|aVQgXWZGQCO7~Bl|Bf8J+1vo$Osi=L-j#Au9lE$^Qezr9Gm@@eq$+=n5tbM>? z=5I|*gSr_DT4|99YyyNod5JHKlzs<^zSpGpupLAXHt9-MYV2>Nebpj};M=kE!QQ@Z z2u%n0f3B$jQHdMRI7~Tf8Infy3MtdUe zmvNA}1kR*{+(5(|oTz52hw7ltOOJ0Yc{h%ulOUjKugAz? zY51MM!q9)NaUW{rN}d!`Ct@ANqKrFKet%NQdK(#SwRZhONy7c`8ZF9!M2zG_ExSGz zL{AXO>$*W}Ur(q=iSUo{MOze{t;UYFTKCYI+_)#NlW}p{f+zEX%6kCQPfUr(}T7(cv*w)c<3j#mr7|T13>NR2@!`{GkCiIq?Swyz7^SK!&mNMo^g6IR*^!dl<&Y@{~6LcOGmP2+Nb5 z8^%3hTI*(L$)EPIYU8OeZCK3T>HGzQFyGN?^cgjtYvaRwhBwfi#@Jx|8m4vauoNxR z=(m@Ae<9zJ*YzeNH(y1sy9|4KEuupP8u$Wv?6$LeiJ$+l69(&n2BJSr5(a}=0G6^0t`>vrHMUS@WSP+lIx3k0GBDDEl za1xq|=O)!`pwX?PHb~1hrgqfsXg`f>k2PftzoHkGgF$VXW^C%H#V2Ipj;Arw-ds=a z7fVHJzGDpNh(7SG@L4&1ndT0oON7=xb`IfqPvQz|ez#cmUWT>w)~G%29vFfRL(nwi z#Rx5KEWLH&+lO5#e^cc!QOj_PE^nL6=V}kZX*d_UfC7ngA$-)I3xPl}nRdgR*rmoL z$o2p7f}atEJ>7u*FLJ*(rxYT;Z-ohTY?o{CT`RFnE~k|jexGS9iPXAjMaH^FEuw8J z#Gl+4fR(%M>C=-&O(eYdtP$5q8|7a9JY^2%=ZAcC4=9;xX3}W@Yv^d0obmP@_2GJt zJUKMrU$m5&v=*0UCQYE@#i6lf1m&d?0kvKjgYI?W1=FEmY>-jaNelCP z<#{6@N=xwXdehGs9Hn*FdKlBAv~EFusi-i;G-0k9OQN(+_aAh@$p=ts_z7$_UP`KF zTYx368%^^Q>msVBa}6b-*P%Zr{Z_)YxXhYaro(OG@bP~<-j%2iEuyuAj^CjRD`vn`I zH!&cf@dV{9Bmy1ncN)VxYYBIu4zhwY8dkg{@-uu73xc5z67Jt@I=V%SatZca!DB>r zKW{DWdP}jLLN#?iMLBSV4L8;mpAyct0jUuK0yc^55Rn`1=BX!mQG#!E5UL@mzP zAES*5t!o_i0i?p$kYmong+kfYXjpgZyWW@`t3~v>-*lYwz@&P|m2t;8 z4-=HPnmewREip=CwHUv+ImXdgEi4erKJC6|8@DiO(=t98uCQ{WcULW{Bbe+7uJ|KQ4*en>u{L zcO=y;iioO5B-I)e-S+!_;XozxBlc4uq>P<(^M7UpbfkQYB6s4wI8qiHfqZgWb6_d< z?@##f{RKYUhT%Y?$=43!V0To!c}7SNt*!eJ!k;`x68H(}TQz8Mk16^lc^F^F7%xQo zNDq3$i45pPJ@1s}cOqf{CwSa9Kmx_RGT7fnKRJrp_|Dc*_7vtD3o}?Ibfjvv!&ufs z>lC@;HuO&9y>T}Tf@0Qz(|7~X_`8QTq(dgPiNX$?#-2_@j2MnhFb3f`Xu?3kg1EVR zSa%{W@CKEvBM3SO2=o0RyEG$qQ?zw?_RxWR7`y5;+f3&o_&I$TJ8pah+3t+jat}KWfbpERO+YoQrq z`Gdte>)91jj}G>w-hc3D;TvP5RRI*>8F`;@J8MtqBGT1-48TtTy^gE6t)L)d_3o`j zb($`yBGP0MKwD~QzC1WM4LNBn_E5^xB6y5kX|3RwEwt?ZAGt8(Q5~K zSxr1>?2p%CqwnHM@2%iU=c5gtmsp#CfwvcdQu z0e37DjO__pFTefAj7tfc!*70-(W{>}PJ7RIrJt5OZ0{rT*adr+&t}Ny%|BrO%-=h% zWZ{?q<_o#|0AraQgl4p&)f%_@X?=z$Z)*8twt&EN_6;LHo$~X@E9z&=L~tbv}iO&eiywtAQiNJ?w8tF>T*0)TreF zIfm`YK!?_^@u%p8J>F9etz&0wCQXMdOr+u4@ZrnrhX2U;%As}D!j0b@xGk7wTz6oT z_mGh?PJ3^v38&q9&QXEe~9g97H)jxs~}g{%SJW$ z+H|AYAZ=J6LPE*@jd9l?t!>k}bhm?jPLc8CAe=>gYP>y2OSm^2C^hOah-}0E>wj-*sqD_?Y%HVW=kZ&;_Za%14#q1aox+cb3|biNcR&JMnt04Mf=G}PQ*#hW#hg? zty70*p)uXE2SR*)C)^jM;kP=lVXbDr7>g6NVc5X^nuvYeWFum*HW}xB&kxq_Y#t>n zK&r6!jq<^oqx*eA+ZyfwG5qgv021&LLSDCNwZ$8KhG?C8r-)svYiL9(T$k1Lz;DuVWBf2}?f92^q40FWro((Q`UwKA zVu5XVYZ@~GS`>zQ_VHWOOTVLZ@Qn+OCsTmo*7W>e_~1J~modYs#rX~0XT0vzoOtZ8 z+lfs=aZd=of~6Xp3Z@gWf)jtR1w?EBfeJnb0ehC{v9jf3M9}l!G|wOW8+{W2(3h=! zwOC3KFb!D!z9)Vk_e)Tb? z4%fP+Hs;Nn$M~G^KTS=fJ3H5%25b}K`yYY$wpbE5-f`#<9FMJ7NWx)+Vbfls+8`fj zr+w7eGaL#2j&T9saql}D0(>?a318VC&ZGA6Z#+WdAp_69MW>m4&|-U|7^8Z(7H0f9 zRBK{9k%Xds&3H3O>#wC4JCd}C;kWU@ANiB|hmMCB49~}$V+={wqFVHe!(h+R?EuV* zHD)H`q!^pRWZd9(GR`JrDRxpGHjlw}jWl;Wc94sjSZ*fET;v;7!jy!)>mR1obK*7f z*dJ_oH;CRVM3b$o4*>3&)d2Z1AHe^Qt?v$q@_7Ds7ZEIY7)3xuP{B^IoQjAgHc(LP zy+n<%$3|j56;Sblv5#F7V`7gjL5*TfFqUAACU#A9&xSEZj0(J;+2=VR-`~q0+%sEe zXJ=<;X6M<**O|CfCuS*kuJ(@x9;&xhLjm2=5biCj3g9OVd;d72l{j6gSs-JPj%&osu(BM)P%?CeMgI0 zin_&(CO|XTo4>Rap@B`$F#%j#{emCkdV$svn#e_Xk~D2Uqe`vBNLeU|7PJz9w%-HM z?lkUmf*W>Z}joyyLVuvt)H$a+;ainD(|2+kG18zDA$ceFZ=U z8GSXNPuq%G?wkucFQKh%MPs-a{%kAiSDFCi(NsJx)zx}|>z56AQlIjfF$x#1LXD$D zx8Mel)$G2)7&`OJ=rOqSgnkO|H?zoJa00X1=dN%B5+!Piujyfw=;IERit)xkqeay! zZ5eu~l3<8Ccca5vb*m7lqegGT&%Y#EgqA$*kGB6}JrTm;FrAOaVTzq(J2(NXdXyW} z^(edNEfgpc?L;f-=1+6miTV|FDeIXGUed3iv}1i1<|-QV%k*12xayZvRy(m2$L*BT zUi7XSUy+C4daE9HU?q$*uj=_9omDrGM~tZ8qsw@0q^V`$Z?-LPVvy@53fU<0B!vkP_v(i=EoeZ?(AUuZjRV%6Y-i z=yl#H4PE2^$UD!!u9IwhAO}z<2S=IE9nMTRI*D@>p7~P)qj`eAE9Yd^8U2id{uR8> zr+96?A`2Y!4_%nH8wNd&_iJP3R4zhiHLE2CzD8?#-)68R9kua=ma|TB6m?AU^FI20 z*%tj<3^);;(kk#BB3clq+ra2z1iFYCTzIw-OOOV)7BJ6L_bV85N*u+CK*6&`#3$+! z$DAcRamkvq#0F%hx#G?oU1Ytp$OByDQAScU*ZA~6%{_#-IBzYce1h;hH?+NoKD^4X{7pa zn3qlH_s8@=?5+u12xC(Bu(Y1aFgpx@4qO_GKz{*aPhR=NFVLxW zGCx$?d8wOL(Q8a(R6=!}N!-w*uwHG+QVQ2D&n-~umA(frT`j>haOe~~W0u51d=VaA z(z7HAZC?02g&ly`VxkkWg=N7xmd^IzWL6E6u3Hkz>1*?D^Ql8{ftJVUoAs=NbgvK9 zd&uGrH2p^*Pwm!S8rIZKo(Q|OEZXFk%3xElQk07ltR%2d_zWcsBhD9G!Ks)bY$FJh zLjO8|j>BN?v1vhO3Disa4#ba1J`MAYb(Nj05s3hwwbLUS_dI)Fd`6|6_t!K2|A;1pyy*x zjyco8@i#4r)v)75+4Z+E9`pS?@qenLWpTE0D^KvW@G%7sA3bOWPR3`bcnzgFC6B+e z$QW}q^E%(aB_L>d-K%VGvyoOjdQ*D+NAO&CmiV0!HB0qP+}&Rz-(zm;+-W%<&}WU(b7lta^^KEkOv{rHE?Wf@^kD9xcr{4S@E>x2@bw* zCdDHOK{_Kq=4UE>P;Tyk7Co7D^>wHThxBd-OYcWU03Z(2axrQQ_bRw}0vOGU;X8Rfd_sGz;yn zcUu?|izk*R#Ze`fn{XY@{hBkuxI5r$fOVOOM;?Le2L%h}8cI5nuFx*|Q}@fyPnx!hxoDBhwh=arG~%vxKRRe0|Kv0;Vf307JIu z$iKO8vly#?tbWpRB=HGWhq+8EYP;WhnaRrTMYTt_VB8y;?7`RcM!1I7aAB_joCU7> zNx$vTknxJ}E6yp2H5owvKLV*1$tQu=JU)~*x=uqMeCbEl=8q ze$-T;62*ut1{4#1$Fg80+Be^CyannSw4Yd5cL1K&-i@0mTtM2@=M@e=SVP`z?7>H2 zSRvPvc#7NPc#LN%Q8#6Hp^CFtOHXbZ*w#7aMtB_rKCkZE&iQMi8Oo9rggQD93_Fi0 z9d0XsGt-?v`OEPkx7g1-g@?quBySA2ppY3NIUP^h6S`Ovkhi9v`QAGS0{*C>Q}DKB zVPdxi!L-+#971z}Bva^=o+^X5wF6_5GcAeRa?lm#!WIXr=AJkCH0ckWj0sTwdVo5V z7Oz{mHmb%13RCXy=_V7Cz|d-HX;4{j3{Uhj@q592+k?79pfK;^GQtdKI3bNNIYdGMsbPrCJ@HJH50)7Rf&OjOrSXf z3V`~wz)Xc}G`r$XGQ0I7%ym?W| zonJOHxvU??k77w3Q8U_*nr`v%_ks=ZA5wf`V3^vB=Gg`gLlefv{6cR$Jjd5dKwmYL z1KwNp5j$_}9GSder|J4$#(VeHy=(wO#%(mlI82DSrWJMjtnaXQVP=%vPr3dj6vhaLMG4wNuvfc*p*wk7cwBUf!G zk!v&j6Uf2vEs0w|d$V4Enr+q#(77P7C2@n1uRcPSX{g`^I++>dyK2ho+5wMTj5$K{JHy22_glFE+S!bzAJwC;MTGFU9dW52FrWgWoS_KWlS zF}MAmy`4IY<6V{!sKje~EF;wuSIZT&v+M=UD*zj~n`CxSo8P)co#u{>FY?^T*hI#n z46eD(>tqEw;mu_dr}LJVXM8!P>;7B)CzRj4!LysK=XYgvH~F0#_C_8O#@b98s=H+8co=tCUAdx5^oDzg06E$sJMYcF45f-;@Kcb6 zch>GW?f?Vu%-#UVlYB)E`@f#li+p5hkKBpe=ma+iSVLotNp2G8SilY~yX)ctHCct@ ziarkWRP-<$a7MFi9z*soc_Z@(dgp>Tn3^vfc4jsp)d{Qss;?a;t9j* zt-*jBau}$~VBM|O4f1R5xEsyjNF(~tNR_=2w>naxW_3btCT z|6FwvDUR9@^Sn)_GT;Ru&j<`~P6{kT%?3_R$(tO<^hgKfrMNzLUsLgv%qS?UR zXUIs4LEF_4W7ibELGvi=nUiOB0UO3_zs3t2TX??Kq-MaU{efzqP4$;9&G_ zNz6gV&<)eh=vJOOR&p^MnbZLGS~}&h@0%5Bm!Z+2{BX3sR+lo;+n`fxEf)CQW7* zEl`;~G7e%*xj-V*-Z5LRX&hw$otLgAI(JcB4b|=9CD@L6n|;y>aFWd_cQHAJ*#95N z>j*8-kyZ&A|8(xn3pu=C&zKxpUClzDj+)qeqVd+UiJYdSu7HY;bWD)t^Bt(p99JjG zhF4Cp(p-1yZT~Oag@*H^(|jEozPVmb^V{KF&V8LFrN&pInkS*yP37z@UnuoQPe4P9 zEX|n(_c)xC;1yq#u&E62jZ=+$$9NSWDqmve2*Y^Pmi$sl6PO})*IyYuSnf?R@6Jlv zTjt1lhZJo@b@=}@h_8j-L;>|H^NS609nt~C4f8$e0|-IK6zw~JqQEjOtT0kW@P1wZ zF`g<-2|YZ=M3&xp3-0A(KoC-m)DX-%)Lp&2e3%9CrE*{FaOo$}gJw@#oFhPoxM4Xu z&hSoQ_L4W%{P}A~2#?6_IvSOoCrbb3P0P3F#qkjnCE0=XV+fhEJaX`%v5+nso3HUM zI$CO^iZOd853vOX{D@*=JaB*mWrZYs}e>GMe7IcBTStsautO$_P1ijZmRFf`26wyJn2|j~v zLGe1T*jZtO#5Bh%uQAEP_-Q@kp4#9t!C(}AQRtak1e!W?`|zjtuoLkMjmO|^aicI)pL3NNzvbSK6eF8W)OkU7nf|0NIg zAF~AR7X-|C*fzS3DUEL;I8%;$}b$qM=1zm;Yd2qs+w5J z9c%%1o>!cYd4R+)?Ob32_^1k+)eQ8{`Z0~sB6CxEbb^45a9XWG3Di!C1$HcQaX;w` zVCkY>6rvlQMC-`=jF{be2z1xO@aJYYpzf6lL(fjnfepz(DppSqav5_^LH#?6fVw$4 zk?~oo@iC5+u7dfTV|fCP1e2N74RsqlwX3rz5jJH(#rMwq5q_wX5hTU*j#xt|CZQilrT0#eDBzi|OedQql#cX(J%6O1;gY4&8)r<9NI? zkhd5x(mV%8Ut}=RhB>F99Oq)6M88?8>6;w-qMKM#HWhlWj-ul&cA#D)(|~c8?xKN9 zJrw$^yYO$j8*MSx*;p*N9^p8%7Z_rgA!XpRQ=gk9t{WwM5G8Mv;12Z_Qlrt>i|*nZ zm%p=eXiX2{?{WbTY5edl9`5vj$XJ4hcRfUyrO-O;5Xrxfz`tk8r6oo56ybGx7DuBg z9w}?}Cerw8^LQi}W%G_lmU5;JVVHvcHuV(sDhK`pLKr7y#x+vi8KsoSL)eV4BdD(Z z=uJ;Cy3U8ixOuO%HSC7*Xo-djTpX2%RP0=JHaVN0`)>}d=q1A3_rY>j>j9nbCA?~S zqY>@xk9=B`pR}wS#6IST$iF!yu^Rxj3j?+E78Q$L1@mAD?W#--dJAh6@h{J&!Jz)0 zOl)gK_5&F~{hsRX>l|9pTT}|Zf>bpOv3btYsc@?2abyLI#8@((YA?FF(Hq1~|0joD z_Z9&ztzP9&`97kY3n8mf9}(Ns#(m(9;!386ejP zVnL|wXyZg55$L`Ql~ov;&P3B*3FS2k9CoMA|{j??(z&3MmaZe=1|i_h3p+xR$#`VJEDwQ3>Vu^EL7b%^ynA5LPk9)1IGW?O;r9 zm;u#udE>|-*CBWjGXoD{L$D6^;30O1C|`alUIhZKKKa*Evb)JgO+$f%A)=>?Z?2?U zL&PALLn?<_ekej+=Hns$L*Xdj0TMzvV1e>c^RTFtiO@+>pfjlW4m5eFs8pdc=;@Ty zU7*71H-wzc*!IiS5p(t6i-tm(b9!*gcFBWEVioVI5#dj(O4g(+EIr3f&ov{c38@QgY{@V)M27sgj84&5Clf?b|H z#F`u~+PG}OL+|0DahYK_19KeIg>M@>diOtQ^KkLLeD4o>GFWVe)=k5iYDZ0~Gg?%) zZM=g@S+FnkX2Ftf5mJoa5_E6r%Eo180#Nb%^Z=#RNPhZ~pFZHH+q$R@7=^ya&9p|)sZ=;FqKMc%IFz^6?I&hC#j}dJG$03FFP<)w* zo0^I|Fz@Prcq7)faG|@JU%e=Iv?xNS$A~hr;%&M!MuZj{&9uX`{#c!SKNi)?|A6=L zxvl1!w2opLEmq>byi0vQ79DGKLIKB6ovg|4u9_i5YhaFx_kq4xzuugnuRArIi7o)J{>12$;)?W)i_Z{ zF8ZC$jT64npZv~k^NyNyRIeUOqAz<`A>yM{um8zxZb@7Sr=yYC1etk9YR&3V=y==+ zvat%aA1{U%JB7h89Rh+VZM+!oyXHDm7NRR{Ahw;>aOwO3v8#dtD0+gZ8s6=>%=44n zo$Hh|gsFXe6#5iA6={7?&+!yiu^Nbu9ewWMsr&D=8P(-Fe>y)wL{`cC4LNR{^Cv%* z!xMPH7mb^|FylIfO%x4nS2E2x09x8eNyFz|3prjePJEZKo`0vw)RWnH4I(!@0(OBUR0>CDF9A{sdt>-0!GwREkvT)=1+s;A*XNn(c*Yf z!?O+&8Fj;KZRt;E;zdQ7;75<+AuKblQs^gQWYt078^(OzIbaS~i5kaKXShs^6WtWo z?yhw76EWIn{S{qy#&|>mx>@G7=J3)Kue%GW^{1k$+c7(KuNzHi{HLO2wa*u7XTw$_ zcvT;;>L@W5KR+;Sm{{|?sPAOqQ_Km^^rjgF;b&!O17LBL!Y4tMs&|$8@NcQBG<_17 z@1I|37xzD(n0;N7j!x*gT@GA*1TdX>F&4&57~*wuo&XJCZDu}HZyfLfQ!+}=?=&7a~> z9$1ZqYGU5g5}@i`rU;t|C{-gz8b&UwY5}VA1sY|;D|(kM7-VaOFcv18j@}HpL>p`( z+~x5*NjGfRW`C))e5Qzq{FxhuTLq(zWwzRfO1h(0sw_T{<3yvgq4L*5u{<=!ye3pjm@ zOBGaKqk34c01TMYGWf>%NznRE!) zSiJgaNjd<0l;1gOks!SM=A6~biMh(hl6V`oWS(bW?FkIJ#wm#@+7tLtEj~-Nlf(k= zi%vEL4}K30))=lmM;DSr*-$T3!F^njagj;i5F@pD9A)l1H9PcunD&*pkzykMi= zShJDtxY_5(97vFR<><*A(YMY%)*GBO(>T9pR?c%7>JUHp8=?HDf6mb4xgyBg^ageg zZHYNWKaOHcp??lp;x8Cg=cp}x4vSOx{X8FUed*F%(V zAN}iU{e5ZdJQ3^C@Vca<^TZ^1@+37*hI4V#No>a6F!->aP5vD<&2WMZsO_i=FMR%i z^?&kLaK6ib^OHeTXZ?koe;L|Xf%?7$+Gh{GW7Bib|1Cfdl7)ZR-cju1Vanio{uGa| z0Rq3>#!GHn7)r4&naGL!cAeuCS?7y}%@>}4PT~xI#p0g{*m3I#Jd+98eCJt1UD2Z& z|NJ)|ryD##B}dWr`NGH2h-(eiBg1#jPSBJ|#A~T+!7U<+m^&ta@AIMeLF&kX4 zUJ2S}$KvmIjLo=UxgdS{XJTN7v!AfG^h*6v1A0Xd*hvKfL}ngCBm6=?M=4;R8q4q_ zJpqN(Uf1H-j232us{Pi%sEN}QbbUrNl4VD^?sO}+Nd7tOgZRG>FP`w=S+G3C>=@>A*J3! z4j-d5cQiqd%H=4%O%e6|&mDolF>ZM-f&(e;{2-DX|FY`2>o|2-ASP96z$v4Vt!ner z8JyTd!*rZPL+~|uUXk>0f#~Hu?Xu>gl#iICD8X++snjFX<8u)!d#2IXpNqQTfoW)= zC|c)9@n)b102Fo%(#Q7cN^;_2Zkr{qAy(DXqf}}k?kn@*KG5vUBNVbw1QZM5VFBqm zZq#$3s1Y~yFe*Q01{%jtCrt(#!oVs3O!poPY&7ZK4gkJi=9r^9L{KVc*46+nWI0S1 z7K&Q7LP#_a!ABo-z!WQTj}D=zw;oEmy2-#&vyt-*ILt;)FmM3?`qo2IyOF~7ppU~3 z{GXS+3duml$a$q}r>i6+)^jXDIiv0;d38tSr47nUYp&A}LUf3BE)wO-)bgYKEA|f$8Mwq2rbSDjvvxX2JCt7(6HlIhw*LA3HPK(e!mULB{0sa<3+GkvV zk1&-Fr}2TZQE4*LT-3zF^kylpscn>+uOk_T)F{KW5Zt8jkm ziYHe7YBTU4+ZX9C@TmM44%G*|=tPy=PtTX(wy8gIk(Z0p7|-R{AuTv*8n|2xl?@Kj zspZ12XXu|GC-&2sKsm}a3hp_Ec>jtDUZ|At& zjAMB5VUKYXNw|8W%6=M2II1nYpXTGY;-8?E);IccIc&!Iz?Z2uO#PfnKayx%#Cly8 zN(-gkhh4m=+zQwpg}$c_D};aP&v7#f2o;k&5bvNMs#!fdNHbT6vXv`BH8Q@1b_g@M z1-N_S$_D7z+Q`Az1`g8x6(Y307hkOcT_g-8Ai1C&P5@tX?})fI4o9~dTJT1wE^vpd zE(fXnN)hDK^$*=O-~E^0f@<>}^;{`JYJY+kg0&iUfM@LtOBj~?W5oETX7ja$D(On0!5kDn#{%t`{aG(wOZ8iyxfP!X`W0ua9-Gb zwP+I-hHz`>CElEguksA$vm>Ywu7F$Y$4GdFdvm7NPdca4+0~+U<-&W=U%`_pCNp%> zQF;%0rRtZXa%)8RfG-eH=R{|c`SJ}^LVa&*``)_eSpKneIAvLXv|vne3X)?|n!7+= z)kh;h$yb-RmSriSks0+p?O7v^*zCJ_F%3a*YZ|uhg*vP>f|t;EoiPd*t=>b1Q+-o? zknDJk0;UQ9hgK9PT|yGN>-&@LK)u|}^MzM`wqdZU>RyAwNz3*51IIDyX!SlG^n9%- z?Q#KHNRf4-iJb33t=5T#!4A9xg>|E?96vl;lAiI)5vS3raq=>kEh)r$c7{jrF)t zCV+tnOaa}H89I=tDPg^j`yuLFRQvW&xea1yizeGq%pN%%>l6_K-Fb2G;Yv489z~8i z2Vy$S&`L-HRxhAim}Y2owPOz*+aQ7({Q7PF5ZWU&gzX5X)(c*$y)a$8ls0qB2-Z*l z@25Fx&u*&lg=pe3?V6-MU%*oDScq(2h|VR81AqP=CX0dGopk#PQK3pvUL(p4vcowb;1TTQs8XRS~t-B?0ruH7>>M$^s9<=h@YN@|6#EYZ@*WfO(yJmLZQjxRpS)&&;8& z--yOOB{j!>6{tsKZlS6Mj_tqx26ycLvYDQLBPz9Ozs2CvR_IoTAZZ20`M#H`ZDt1{ zoBde`SYpq$A?sO!A6G@UP`Ay(E-P%Jdz-}~TTzHWwwf*=G?>5Uk))tnIG*7jA0^`R zXdt-}^E@qZlB2Ue{KSZH)5TYiZ*WWMa?n4TGYo~rFtIXJ7{l2irEUE;o^-nYAYt^o0X!Sc*^5G$7((xTg{m( zS(#q+C5_%D8kOq7g?JZrc(BzO^(B43O*E4Yzo4vb@C1GM4b|8#KCU<93)pac&gP2q zFRWA51qocQk-qE%#>GIc9*I=PdTzho7j$O3m|SZA2Ba)O3KtGgTQ*R)9U`dIY$WrX zRd;OQ(i1k2eTQgKCX7pe&QYJqSJ`X`P!%`OogE@nKK+tP?G*Jq8v$Qd{BS3Ky0YVN zw(Cx)PER({?42UiZ`)ck7kfFcq(+v+Sp8lOF0NWjzwCqt8vj2<;3A9AY@M!pBP-9- z-|v;#6l&!G7`mL~( zotsypWnPI1>&X7CXy6yYjq$4rst!N>1j`8Iy~)!Tu#RqhE9yl&2BK&`z_FDHnVCX_soM(_F_fWu0YrH$)+F-A?URtCz)S{}9TDFdScf+@| zU{AiuRIZ)oU3hEGGrQd7`YB!mTO(CRO?yPO!ed^zbQe?w$Crt6)>_52wxO09y9Q5u znLNkS4`!g4*7ws@cK|%@<_-%8tjf0K%wUvMk^I!0pX#llGrL8+*S=%g%?>j51lC%> zO$M9p5j|wwYTCI+G^$v3waE;+U#8eT%uOA9g*U|Y_u*7{FC2j9)>56lqIBukyG$Ce zYQbx?#~SLfSHx88wTf}-idpH2IR&-m>5}?to2RSk*j^FX%sa0P`&X1ZKt}|`2oLb> zN&ue&P=E2$G%i4xuNRh{uIVsC)uYuExKC8>{yh)g*s7A!fduqL1mk*iXu*VZ?fO1% zSNGUZbiC2z%2kPIFke&{_Z+6HF-U#9o5|>@2Cbwu`$S+s6Yko|q2qP=sU$yD=clIp z^lT;F*(aKN7ygzfD@0h^_ZGGj(c0L)i2fH4k=u{CA zd}=x77)KbHx#yk7HSs8+W@1*H8sdZHqP4gDMi!b_JSqb6 z8+Vsi23_FR+O&iY975q=7N>e17E;I&pHrDM(K4|0LW7w6$$2Ne-v%%-UFUCNeux~r ziKeEBDxTc|S`rUw74V_*TtYk3L<6~E0X;~=-nn8CRXu{BsgjE+?ue*d=FuWUuee{+ z8L8=$LUWD)|D;8<Uk7&yR(pH9L2HWk%hGWD0b(| zi|F7{VUt}JP{c8?bo~X?`Sef1lyIwpF&SNt<9j}Ngu8oR%GYbTfEqE*q> zC^VvwdUh=}?zq?>ieCngGb?v}k(2uNgm@>bxz0t+`4GBHrq^?@8V{N#SBjTs7$^W) ze+(fv71QyFfvt0>YdT_|=b@aV1za1fdQ4@Y6M%eEZ5Sub!ot_y;SAN2;e}54D!!@< z_)908p&Bu~+X-J))fqnOgtL?n!x>IE)27`2laQfaC!xSCXMw9qG5pjCBbF@+CgVa$0nkYxHk5NWjbI`m#&iE(N%xAjmv z3#Kh$S#ToBcYzFdTXrC9rg(uE6%N2)fR9$pZ$9bq|EeSBZmS|w>GefX zt<=qhOpG;wGxY>|>|97;mqd-WtC5U5owSqiw(2%R&)d9a*gbf!I?OTQcI%j}Tz8*Q zHzhKCJzFEVu>dE3dUl%Lq79dXZ_vgKn*0@zkAv2f$S!NSo=4v=`N!&kSy4xi)2&Ow zyKy!2ls^v!-Fyx6wYHv9zP3q?Gx5oV1T|vUKc7aSmvOaRsp$rX4bjH&Jsq;1g%8=% z;LBo#tTdgnFN?77Z@9}xXv8U|3wyC_Pgg^yAx$6CBsN3atS>ET3rg5;V&rP`6dHO( zG%a3e0k#Fcur_r8eS1a3dJRJYCMr676mADddXC1_)2USIS8OG5(<$UvQ8%zv3eWO( zA7uSCxxg&*3l%)eS*rJBO8gb9RDUvU{Z$NYz4SA#v7T$RC|F}u-Zr*%l0mF6&9sJ- zQzF+qKhIUklc?KOFn1$ZlgCofEz!|)W)jn` zA}lLjl8q-&;w>@BcPwyphA|Dmyz@~#tlH?&M^Ni#) z3AkhR7IJ*SwrO2XZ+{EvcGqXKonDy~>y&cLqo3(}QeqHKauhWTziPmE%DE%D+P02D z%~)+C=Bl*}RMkL+TF8Kl2B7e9=;92d=nyx@udv+>L&z)0i+RVRRGIvq^6jEBzy*wT z#hWt=cHaeKbcE#QmSRfixLlU6?yABlx_4LfsBWKyL1-GLSiOCX+Qwnc(gK4xJeEGV zC)(P&jznwk*kA!!+cIzs0DBr5Gq6tsIB{g)F&8u5eEqGoM*YY4^4rfx{nSj2dU9FL zEZC{rShUg_>Iziq>!3(fk`JU~W$hQkXFaqI&&Mq)cO*5tFRD~pHG;{uRG+~(65yLq z^N`{=rUNQ-)zlG`a9^ZYo(xB##|_5&p}IYs{QrQ`)_*u{_(Lp_KMy18pJIRt=t({jg6m`!?!~K1OVXT&op52-Q6MN=McMO=n%?q zh|a~fA(N?aX)x7#Br25tZYYx(ShL&qg~ii#Xbj@6Mz-*B_`4`17tWR=P)V?CPEX8V zHr-2L?)C9Q31IU0r0Rxdd50<}bW==;df;nDHj}RcU?IbQ%B!DI-q0+JkR$#mBeUec zeT_WB{7SE_RhVU3P-d!|$jp;rcfK=ZM|CqZ?rU0?DJt0Z4AO(t72iS%uys>WW0A#J zaKjplDP~UvvDP0*0{#2zJY)MHoF4F;v@6(otl7DXnR$2DS=C1(su-SAx4g;$G#sKW}y8iYRaUTk;$WL#?V z7h;r?S8OW12VNVVG@c{=Q^!_O>>IIPezYRh?Jcg{l~F`h z--*!ybw0``>){-M87nB2Ms}yY??k5x0Vu^NKJR3rh_|xU;4YZECXZ>>OA(eDmn~+A zs=Kr_w=mK0id1l;{z6vqt>uP(@-Lso1nG70 zGL}{e86Ypl(jorc6-&4HcR}oODFcd+iUktPHZG;;xvTV+?P94FGGz5w8p^*#Vrd@# z{t-i4`S*AXo#o#RG4xCVy;(7-9xk$|Xgnv1hi2%6jxYQD7GJkdAXe0PoET)Rb)aetu8Ff%kSf9e_+znG{UagFIBT#Hgs!u8eoxd3e(Q7LOQ2lL@B&fxgP z$XhC8DS5iduR?wLf_^C;xzA8E#ro!zD8%t=&j7aA56z2Nqgy8Ow7`vMPxswqgG%>X z1GDg(@#c5H(-!N3k!5gA1p6P#?4-u-vW{&WpIV!EzeW+qc6LPCT??i&r5xvt^j-zZ zvz&C`e1^-5`22RMijV%f``74@IippAzpY#GpB*kDyQ!CqGu-g8shwar+MO~5Rj8@%& z<3Uruf2f|crWZwJBYCkU)h;G0HT||FD%&G>LF&Q?F3KW`PdHa=x&4cwxxK|iI?bLv zVEJe}f4%OZ`nwg`i^;uhlOFTjc*Nt!oD=@MDsbV+xq#G|{LiZn>M8XEw$XL0E&p~- z-O9G9NyTLu;kGSHKu6e;x}k*JF66Fe)S;yO$<}BLFB=?FwaUk(2%me}U3YW^(st{d zQCtEF)N%IJyQ}3b^g5h(9xrT|2HICF&h0^6A?1t!kr(gh1&FM~OU-70$P>`SIo`u; zfhSbA;j*hj7!msMMs3C|jF8fVmWS)mg%S&bz0~?@p<8#ci z4B1M{^Ci6C?{~!UdCkVA)Ta!nIy546QW?3au)N(c)!$coiSkhm46Q_~G^?lQ@n=%l zbDC98hLNqDoKqsJE?V!%$CUjVQmOK?a`_r{jCyfVNk<_MEQuEOc7a-+b*Mvm={-0r ztbp$GCfO{_x=+VW%!RmfI%x*%HLf#~n;)kEd_s?bzYVS_hocOfD}&cktY6=T6ga$s z0WFfdH>Au8awvsVkV`_ca2p{c#rfY5m3mZ^o#dL{l-Ut z13`=TR`eG%L+jCcf7#%}P_&Mq+d@6e);(M3*7;VCY4!43FC6XkKz$F|J0s18MS)-m zE;<4ZPRMi>Zgw>iGK*tIbXTxk8VAU#b=S1iUB%-kHvA=mK4Qb4nC}OaW*HX|G#*)& z#3BrU?P}Md#R0OiUtB+3uUB^y#p*5dP+VJ=E(FLj{#Qfuk?Q>Z{~*;nj0#thmAxAP zbEZQR6v#tvQ5}k`By0IL(Z_Fwr}8u12<=cehcd!EHU*6Uf974M* z$!fMT6M0KAZr7iPC_}^@BgL4I%{95Q;p9tb$_%PMy{oUSO;_cBJEN5q$=v7F1-q!T zb$uH323>RhhdLi;r#L0#sG#?a;mo_<1T9jJ%JM^7a}+fcdhNC9#V3nUjBC`^EhNUZ z$B5PhbD2z47#0<+-?PxLthhv1v)dbGK*XAM_uj3#hH_${t)M7TADjc-2f{YqCQSN4l==_G~lmwl(H=Y2XHy7UAdhDu3CO z_5{jF@?veO9fYL^uILmbYlMs+%e07WkJY7q(qlEq{4*TOK>zPf+}xqSnP^J=6il1p zShv-kY9QnThZy7X0xmnI(Ifoo_xa=4D-Gm+;|#mCF$S2|NNdcllhx+qvKmBD9gO5f zwMpMx<98T9boc-$Xx%s$R&+}+hjaFOBF%;-=V_WDbczcGAW=Od&HwsR>AO%mQ0N5r z3%p#a8T{GTNZzjN!hM~oeygQ(=9SS}N~Wq|AUp^&z}cs(8Wkh;d4bp4COi)7hpo)) zHD>L<*ma3dfwkvak617qM-ESw@llF_th^xg>==ViC;1mEu{w4SVZWMbf0;}XWaqlCyk+$@A3 zzbM8g?SH%?Pqjz;Hoplfb&L2W`C@M@SZ1OK*RLb6O_Zd<)n$-8F`8=O*MB=NM`ram z?5_&HW?4{yANX!L+(B52KCUi9D-}USOHv#}g4zrE8=7Ro|1VTdC4G#sTx(SAu9&%55z7y{()B9 z&gvqK)DS(Hh$=0QF4gjCG^~bPBVSjcA~j{jGAl0d2I=}mI3K0z$ckSBsZmWi+h;bC z_a9$8*+Y zu$ngWYMNOYDSdgT2jUYdlf9NK-?BR=7GsSQiO~!gx`6ecXl5>)d24zZlKKM76!6_a ztWm=)3z`01WpdP#VV>6;@|r5iCD78cAgWPY2HO0g6LBlvx)ozv{*<>!)JzzTRUKhX z)D@3mhl5!72<6r3kvv30EI6=-qsvsy>gg&2G<7cKg}R+g6@hk8q_&F%l_$hZa6 zL+JU)*19xzP^T(I2Fvoc3S10u35}L9yvnT%rkaG7FuchPm@YZkpfS9z=vf_s^VKQ& zr|NWGfYDqW=caMIQ1Q>Y!f2cK1+^VmIIG;{&D?H1fcdd885e1WFsL;EdAu^SKjKli zHbvi2;jJODGO!Nx=k25Ug9Y$uaRIE(M&|0wb3FYG9&*xt1Xr?KKt?buv^dT$<9>Q%( z%>y{Zd2}U21~jUMT?B(>f%0T1W1p%1z%S4H4g6{W-BfD|cpo8eJs%rN!J)E!#jkij zOU`6LH(Zy-H_q?tFuN?(b7<=FQ0XG%-~d`3Cd)N&`H4MlkhriOp#MI^m4wiBkDV%@ z2A=}mRbA(zM*L5`%Jeu)K9n^p(d9ZaFz^pQW1^uWq&o&_uXvWqERU5pU3dddRu*r zQQ}B)4EH_(xsGlx=x$>flDf08TqtFlCpC|h7fL*~AnS8Jk-S@m{F=hWxVaQHXbM^O z<$K%>`-ZhD`9lzzhSart8d6uw*ql-Z;|=Y>-LKIXB;y+z15kEiwB8c&&evGO#ZyN$J|e7UitqE9D!GwvmNq~2>Pn+WfnRk*f?8_#lz*Wi7c z7iS$HYjYVUQ{SVHn#;CcX+X$1_gM2R75=VFCz{I^-lr<-`q8|5q>%>efO=d#&@}`ae3Z>-$o2n2@yw)7}=cW$m)K2DioGAM}JK2F~PPyX&z^m{9CE z^EAlI1bf$$5>)wpxm5;q!RPy=ms|u}5_Vm!vzSw?T$VS`T|Cu|&OA`)11(?>vK;h` zvg@FUsR2~0rK~GI?@R+)%BFHaNm}0$hT(Aw9cn2n1)2II*dd^(oO8fg${QDj6U;~fNf?VS_I$_l;%Dl$o}&%a{=9oLlLSAI8m(2UlyiTC@UDY$6TX`GHvFAbBBsp?UT zF0__a{9bp2vQPzsa4IQ3>rEF%j=I^=AYYsMntauh!Nr<&wb3a znMg~MOn&r{(AUrjPJl9KBqsU3dLCn<=UAs#hBdQ}fk2O9+?dTcd_RZH0F(V0qL{dy zCOXfwT=hPYt2T2(c55{FL949|9|MkQ9X+9FCaF2Cvt;(%tsbr{t0%dV8A2qkb4o zZZR@Ve$$5VUUyl#86(~N?vypReRg$1`>=krlPw4qEPh2&QLZW?hcFbU_9nIKKUeK; z%W1g5Ehjr~uJl*Ek>;w?jLGT8UU;Pam7SfxYF=VWSd%VkWX2}{qdsU$DIMerTY4L& zP~@bV;Dhqf`ThE8P2O6_oZNr{TzD2IAl$`gsLJ0m$TST{R~IzG;bgm*`3szb^0^AD zsA@h0FUqr)))AEL{zqvTY3cXsfB^<9o+-rTp#XVr!}pCo9R1 zN>j5qn3wZf)6h7$SOyDP9f#fICrSI`u$vryk9U(hU?(*KUxUu$B~MJK!AqAk-B{tVZ#j z(wdVr>9+H-%?fb*l6Mwl<#Yy-d*9g8E6n>(vMNR&zXi+Ei}TKZ&4f=@Y{^| z?l|@~{wl;+M_E2eU!^-YF7sULZmh;4I0NDljmhH5hOILUd7Rc;ry0wxkeSv7X=xxC zmq=I9aRw9baWiTh!1;`$2=~t`N!4shGrGyLL9f5p>e1km|Fg~>ZE_N5Lv}XVy2&0c zRbOY*Z{1|0_aEcK;2hPX{p39N>G-q+;#ok2*B(5Dyi`HD6R)`(0G2WLwd(FS)!ND-OICGh`YM~g=p{SU^T22f zuCydx;LswwYh@i+5~f}NC1-FIQ+NbOsQRH|<=*n0+b=k*QI2p5>LWYIe;ZPKA34qK zZ8#FkhSTdla~QRPyXmK^982AzwBD0 zMkez=?zN9jzR5OwPe;-TjZd5_UJN<>1rrKt@1Ezh8%9iO@BsN-xZKXnrhpG*H(S?mcM4FRC{$^Sgq~#5{YBOU=pVsuq zz(KSrUPl&@Q`DZ zNq*^a`jMc>$q-zZ@i2I@tmC5I38Kk3L0<41-I*+XU25eBszH4ST1}C0 za^6*1K1Eh87FdY)i!8PM8XX55G!MKKV|V=vVH7%a{?tD_DIRs+@{e^&Hb%XyJQbaa zsw=VTxN7({+!-jVR9T}?1|;2prNe)g#PWU^mfhOLEco|TnmSd66q{i7`NLJ(%M~k{ z6_4gsG!99u!DhkutMu=T?8c5u_fz z7{E2`!Hz%C1I(YLN29}^ny01fbht+F17uke7wO}VOts+>Rh%Isdb(WBcKUGBYH>@! znqcsMGi9$vD=-3-;6a=>2cbp*0j3}) zXvCMb(3AfXtJlJj5=@%cynCo|4qSWsh^v5h2 zP--Fgj>qyVZ|hMi`7GD)rrNV*d9NLJx!RV-1*#3opuV%Qo&Rv!D0h0cTwis`ImS%u zmHZ09_B%YEVQShLO|ir@NOP5WmOf9A)}qbdFqwloWzmrYmvZzlL3&n>&oEc1#}L#s znFMSdtm%}cx@C}WqO2jOoTXNYICK2xEDcYT<$bT6HKt_g9VWyyd_n>Pd7$e2LTeKx z$zkc#AW2p&5p~OC=G~!#a0`9

o>P1O)eg(6WyLxdnKvrt1!Z78l;Os$p{trD#g zrR_;;h^PIvv^VbmyUy7+uJd~D=dWQ4B@ck^JQ7g{mZ`!lw~z8u+!8G(cQl_V$cOD z01w)a4f|g+#Mz*IR}slBan9oosJ;Ce%3lCmqtEEq1)`%k?Mn?ZMdNzQcni+0#f;dx z#RjRSS5)EIgO5WFTRzRm6g4aKF)}M}kkM`c{PZW8qLz5ypT5l$P2UPR&8=?wK=B5UYpt`&(8S0PyG&`6(_qVbozwL8LJ^+2 zl(*2qxzWs}F3N>_q}{Q3!YN$hnRbju0^BIaahlP#VH}sS(`YcmXRpP!4KnGHsUT;m zFRXRSl+N&l^T>CRs95C^c3M0$v4?HfoMIM=$GPE6$%`@AwCC#JEC=Y*j-+c);$6ZUd(f54(0dU*ojVuSd=F z7fx}*;9e4r9pIH02hA?g{iNAkGX;%VDq2MTei(d%+-!~aNtH{8=9wgR9}J)T#bG+L z6f?-Jhv`>_a}JYVHsG6wsUgFThv~lzYaXU?3`-8t;%xZ#8;2;5VXZgmLN?}LFM89z z*AOAB!#Vvr7GNIb7ki{*+t${zXxFnD4&Hgn z0MEON@Q5bvYk^I zgd?u^&rvan(3gwOJ*~lD@hskP4$&@o#4~F=;fpI;V#{Ye8)29U963aLR*2@It&F@X9_EXUWaLn-)f9fjSo=A)uL{|q#rr{NMtbT)E_zMz15$PBP~qzv!3u(?6@lWcdP!Ok5-CRC;qaFH+5= zN6|4n6)2{#qmImiuuadw^fq5AF+~`fEj;_v$=0 zPa2;q!W|#3743vi-UxWlSyde8){A*wskynF&c}+^@zs1@{bci12ETt~K3}cp0U?dA zHgLZ8a8)$Dgj?^b{x%~^4CHTp2qT*eKtUokSqKyK>! zjEqxdK4oMkkN`Yq9gjxjbNz|>+O0Hn3r5%Xk&I1rBLPoHB^Kr5WhT$$9B9q44>_=t zfE|iNy)7r=qUaMY#xV`tF?-!H!S0xY?ig=(%&}C17DT|rK`-6{H^@_n?(?n|okj|a z`2!-0t|{beM(!)*1S1m1usPdaMuHWxm665 zPkpi(SXBeNpg&}!k%3&*?=aF)<-5wrD=OQUjPM7S*({GTlBTldF=9NFm05W&=H9&5 zJq=?v_d#(9Z&5hOr?%Wr0OJb}RQ{+4bY$j=HTGIRe9P+suSW1~#(4WjEn}4GH4o3G zVm+(Z-bOvQV=URbiAHS~aovZR`QLQscfI&*{_U`1_i8+;6FUHNl69QDmwp|r8e`st zcp4gKuUMm-J_MTK8x$NpXK{U|hN`i8(f4enupOdr{e>GL-;K*G`2&3>5cW(tpjY%8 zq~i`fnse;AwEU>3;n=rBjPw?_a;W+qOs`h0b#&e%w%SF|D#yJ~#c+GT;58-2>xDAj zf}8kwl_PPV2=a-}Sj=3Q6UCy|*flHle&dl+)cZQe#I8MTYWK$_wBd+oQKOzwD%)6u z6pchFy)x&XeiBz@7CZhpB39Z(;Zj<7OjL_2x0ws$93d{@WGkP*iCJXr=565 zxoDB2&T+BK-mg<8h}AXfFdlvArbi>jB$C6?8D1SeVrL(xUtP$48#A?qs7v?Df}+uQ z;ZU^EHXrdx$_%f?N2uHhQMFFN0uWBU27bThtCtIKb(*jC9>dikzS534Vor;G_Oekp zU8oPvKomB|Mi+|ejyEpiU`oI#u1)j#9#?^U_dZ)!iyBM0`SbIiP^A7lI zwlWdEfmQ9i$i_}oW3ni z+9Q|G@NEBJv=yduBP+Pg02|2@$2hkCETUv|gAw5N-Syx$99P}kVm0QN@+u~uJJt8= z5aWEyA&SIgdzDjDR1It)b&<~)vnl4d2zG4!O=OFJj&E`#IxuS|d%L5@J#L*A!yI4U z7mq}nHh4=-QM;v}8O>Lfam9|eHY4YjAi5e~HCo2eIJ3mai`j@4L&^CNtB=F48Iz5o zpHYxIr42mto=m^j9WVS(46?U*YY>X8Fv#Ks8H_l%w(!B+nh*4D9Mje)>@_XfLztU` z9T)x({}W=z8anlt*k0$=(P~fL=uzlTHzZFt7DNu0Ow&#M z9j*!!Ic_Na6;)d4ax8l+`pSx4E1yl$E3I(+_}K>2&J zZ1le*A}?5C-SE$-Jt&cuN%?9}3MU%1FN7P??Y*f)qV_?3C{mNHgBGzPhGeW4EPx9p z(pXKl2y3vGnR>d5fUOj$ofo6gi|LpqYgb#{qr^4u6j~yEA0K={Ue2TcndSILli~K@ zZ{P}#^NPunGktVdYUYip_cuLgh_}SE$319n-b5UDDHR)fV0!wf(q|f_!9Eae5&&@R9YZ|NbAa4I47pReQkK!35s&nfMWn^+8Q< zETly~GDMu}LOU3*wUEvOrrP1dF(UFL8LDIg{L|Y!hoUc*na1)}-^pVt>NB*b>MJ%* z3@LT&z>vYGE;s{P@fp$qT|AhjQNFU07@J9HzOq(K4-_+|>&B)`DKp=KnOXVRu2=-t zJX-DD3G}V6j12p31=sexR{LQAw_2#r9k8w+T;PcHll_ClyK$5e0%K?tOM60O8I zKa+mnO+wNOu~%NC@Nn6@!nFZh3@1g5XHHY}qSZ7aTtPN_$!J8(+-uYhy~Z=xd~i zc1`JFgba@iRXO$>ek0rHEUYWuv@z~^>W`aHY%SR2)h2kZNp=YN%Pf)pGsJS|whjL2?Pk#8Mlj72;k2s}gslBGoofVwt|WTUNLC)XBHZi(<B@pE)-0URBsK)Y) zJ^ZV#++Gj#*vO|#)?UbLPDjT@P@krIuJJ^_M}H>*rB=5_X7ibh8F! zG=-7%s^QqvR8AA(>rjerCa;QR)yS_o^et;p!{)M@f47BG=A$jC2~)|-aV`ZyUJ z8jFFkw40!_my!s3jlPMKwZ-2<9Dm2j|H`Tb^-wl5cIX$-8xGQh$NYd*FBi(zrjv=YBqq zLB=^ZL-=+bzjlHM43c;k*)DthTS*jXKDa2$|++eU#|qN-07j_qI0F~5^sY_C7F z1!nk-`ZHy7R53=}B*UCy7a-Bxv}_PSojc2gqDMKp+*w96ywDuQmXtGUPOoD$Q+A|o zwnAu2E~UKU*m4x!MOIE-6zdV$LlwjZ3UX$f&rRTgFVBK?Ri1+?kI~i{u`YupMO>&n z%Zxnz@kAEhrPj=&qFbhJ(kIg1F0w{+4@`I5J%Ht9E$LnfW~5Ppr-|I{!;Z9y)W5|^ zVIF!8^`RDBVKdA6I0kl=(n|#5?HfIDA0s~4@m^0k)L!*F&0IHm;e`%*lr;KB(b9Oz z=^-0ALjEfYWvD1|Vu{)luR6L{^dJ^ZG2Z3G0D$*RbahmINxtLlGwm3DglT<_cFQA1Pwv#djip99E0{ic!Nm0S@MK91u=&cr`- z%J@KPGD@};!&=e3g)&$qHKe(tWRtSRkJPMW>th?`kCKgr3kL{C$yOEG^Bmupp3EEw z`k(*eq_}tBJi~SB_z~{o*Iy#b;ybJ|c=QF(fJ%(Xjl^}poGm<<<=t@_9l+Sc9t!u* zg9eUb8qprzXFq(UF=%t@zqh%N8n?M~{bcZO-Zj(j)hGLC*|g)4`p^OdbWinoMPquG zA9(ZoroNgpV05>!ny`1ZGSZ1Q9{j#8f7OlGueUC!LK3}|88FcX!dXyG&%ptBzRZ;Y%f`ro9Gu`;a2{(8*R z8iQ;2U(XiOSJiX$A1j||twU~_m9gtHI(L0f)~3}mo8L(FXkqym?RHYtc;=0lZ+l&)A8FR*}*Pe45NmViIpnO{!PB3($H14sKuU%@KUa|K}^@mm(u7o~+Jdn(Z6&1%8MCRh%O0iCP8Jb_ymTuB-IM6j@me zxJK_yk@2Es7#-mFepm3(N(_MyL-El{Oh)sfsm4^Qm_tIq?8iJ?#w%i8xC|RS1FK4(#4&KhoJ-FR zp+WD<+*&V_Og1W?Pl zvT3ysF%2;umNDE0m*JnD_!Z5VE31o_0vv1S%CGIxUZ#Xx>GCc5E?riscdBpowf|0I7FKKAiL8!gO$q>oP0mJC^=+F3vM;_*yVlby4uOAqp=A2VdN zipSV3!2U`Z=u(#a7Ra7qXL+UtWg61J)VY+oES=#;$qS&v5q>mpfo$O48a1G~n*C;@ zqYKcylgiMw1#*xms%VZ(>wT$DrmQOG`j(LMkgQ4RnX+=z&#$O)>IoiSz&jhTsjBZd z>7n+iQOxeN$SABRE1abBnHc%+ouG%AGNStTKBy90GgJIy*J3c5(e7`Zk~3u`dT}Av zEX98G`a&66_Fqhmm{z!6beiTYM8D|jL%9p(iz23i!DQ&K-ln+uET(5y|Js|v7s+}R zfBe-jZZA_yE4&OX;bW0mvYdS5o{iG7;I8mJyA4@z2QMC{!&$OxSSU{`pW8f_ zG{eNb4qz_iCca*yK8t1jGL3LN(XjQ~-_WeZ*c|An(XPcZCTK1a%(=k?e)DNSK0R0r zXP9)NbZSuR#SNHGJ+;_UJ$!$}G<(xPFu<3DgH}e0uU?4P5HgQO%Dhv0qNIgOU)nf zpOxw6QdvLvj@?|&K06+Nj6*!xP~)Nz@t>@&fI!5*I{tGq&vsx|!pw@K;JXZbm9k!XFSK^;q< zA04z|gmkus4W$iW#4{gsfvbLMer`jg- zHT1&i{DX)Se+2O8k@~0LbInQ_TsHn1QjA4m`Gb^?n6OF)Ddk!`%l!MVlhRhnU@E&x z`pYZ8s;z=AymFn&tcKrseUvO){cZ009)_R8tQ$^#oLRbt?2U^JtK^$9|CWt>R?9jK zY}c8G*^zi7kWF8zy1Lu5J*BYnA16L8CY!$Y7Cz9xd&u!jJR>$-3CB74XB;Oq#nuiy{(&@njf|+$%HaBQ&t9T`yPsCCk@cm1!$zO4kqyQ78vVIOhRcFSPO7?A)|H>& zB7UuGR&Odk73Ss)LNxyY!`L^T#CR{f%crz*t*jyUUANJpwX&(WE$R2Qpm*zUCzW3( zqsxpuqk3fZ!4*)wm-k&%0@K znn|bG$xp^|FaO$fWnG+1 zW=oCz145txOOdY{Lnm+Y$&r)FCYb_)+$(vMo+JN|4<9;d#s&<6Q72I>Gxi|XJs$Jk zQM-*~9LL*$ZQjR!JXP287FGS_^(uj@FUBC&Y zyU$Kg&PLfmmP5)<8)dzq!ALQ6_Zrsv`kjv{=T^@UnR*f6wKh<3!t^?-OIY9W;Hj3=QsI3=&Yfy9X2%`~i1!Y$xSyM%C4e zY5Fntsaxo`%~-R%vW0xN$gqgGEgox@7oaxm+)QBA84LgI51iCx3m%p$*}+``Z*V$F z*;`~%rQ-W$W$wkG?z-osz^$01f4GAtZ$-||xwL94*7A<+2vI-?^lf$(GMQ-%#ZPF*?!9l~$(q4MT&SJKyOu@(fcq^c?&e<|z7k_Ouh<;)V zRo^amiDtQUemkUlu!$aQm+4~8W^R-azguuE*l?PEz-*fyAJN_&Xq(NO=-LiBO4k2X znel5o8Bg%@F(D~;8T1M2^rRIq@j7h2d$^s zd2&&$qV=Vf@s^t~&fP@o->l`zFz$3;PqDjY$EG*1%lAwlKJyC~^w_B2>?De)v7B)3RdT_Vpe2{upe zc>W;FKRxCeRX;96D~260Q*zx2@*4F#E_;N1c*|@qPT(ds8Rq+E?XJ-7<1mlNEA-`Y zIZ!?>bW+JOM11m{G41kN{vC>{`STqoE$(gp zA(&+GT)#C0?9Th{c)SN6Mm27GKhXb}kKMkn4P^_mRz~=ssx#Fty z;}_C@`8jz*ES$@mIicbpQOP-ZNOYM?2hYP7Jo&&;cwWZJ8n1r9e9VqaL#vc_C2;T# zI{K`o)cm51sklNTagHZ9N9&*DeG&Ry66ihN&*y!; z{8Bw?96id>>1R3G9a0`Aj!d!&65V|3Zd6HV<>iwj|)d;~K zK`$wBeTJ|J;RA$3g!Tww2>&=sTm=YUJuY$WK$wT{7D5+<+6Z=pn|P}E?~Es|0)+ht z*$5L55)hgoltuV${u9?(gl0Iz7+3*+_s2eQ@r&uUBczXg;#kx|dnoqDX<0swybfBT zwfWa7w?*a}0JQU}2r32P$?R1?6 zTn~s>SlIq-<8&new!?RVrvVlK?gfnK;B;+*gMIP8Xid1{)4h?7w;b&1<8*}u*ivo7 zUUIr}up$`^c(J<8b^%@G4wm}46P>O=FmM1c67Vcw9AE)pFF@Z(Fe|_mz=?n>0Mj`g zFb6RBJqQLE2Y4PZ1uznG(*uBUfR83SU2DA1t%6gWt~^8x06Y)40+4U2o&|IQ>VVOZ z^bufZz~CuPS6fKd4X^-`0~SI;z?4N$9iR^A1T4S^*B>WKqLw;cDS+vKI|21<5CF_w=5$5Db{-Lo z4RF9p2vP|eSqB>d9DrG04&ym!D!>%XEhF)fm*9=iEMUs!RB(g~-yLW}KB2b@)(QCN zW7q;%Ikn< zo1)i;g8sk~S0P}^6s*m|z+feu);N!gxZ8+;^*_S56_V&+FRh-?Ytv7?w7RKj4V|uV z3~lZoyxQH>wRVXU$(u*9w5c*Jo!{}|>W%tl-!N8rbC zr}0)!TMi<}8Bu0r0j`~hgFcKOGjtchcRZS%V}_Iuf3v(VH-t^YfESJUb}Bv*@l2Qp z+yRK^Jp6+$ZPADuj<^>%Vqp2eSz;)1!(iip^Ra_&&8R|etxdIF$P~kw@JmGCDR~Ow z;^@{8EtsbF*6@{@2~HOp!uT=qO4wjYL|i<_bSfXXK=k!5@7uYIQ8ChHBdwVwjT!BQ zv|OY$<}~~`eeBMuFV@wa^7T+t7Vq7H;|m<4WBI_|O5$uJhnrW(eUtg`Mh#XJ9^hK>nBM(1bV%{R>gY- zuFo6SA>!f$n%Z9r5tk>>^8Q*6kv@rj>aTSQISr3*&>3xNo0}m8^4*<8?O)cagj9mZ zx2ESJz3qE67U^Q*do=fDt*bcl9)10?R!clGUl40M!qlN}~p94JdISc*--7X2Lm@)(r&1 z1Ky|mfc{YsJPIv#-oP7(_-Pa}2$khdqrQW{>z!$|e2~^#?43@(41x@Irc>R)+ACt( z44N?*baK;Z<6y0sxSU2;1|xs*ObQvIwe(KGfRZ4n$!l6|nleP|Q{_-QOvg|e|JXA4 z^XF}6(~Tio53d9aCbuYN7_2G&HLZpCa~{3-n$}n3&8IJ4(^`5ZWjJx)9}>gbUPpSb z1=Ra>NSeQZroOIC5IYyq{nsIJaW=JiLmMSdFQ+YUfa%B;^cUkvt7urFHo^NceEg!a zG~-RJDrJt){OR9BZG>0Bc1##)Wq&P_<_*>QiJd#?#!!@=wu_n!1J#SWY3eZ4681UW zAEwP02{&l!o7xBE(;!7W99sX!PS^ZUTKE>ImK&j!)$)Z5r`^M~DxEu)k-As;v!Z+X ziwI{$r}FvuunjI=MZ{Zx9T6c9`QWMxVkWB?xCGEX4cdv7O2}upHpRO$oJe9jnmb$@ z>79q`$k*t9!?mg2(QsI^-=_&9aGi(iJ7?&BBeWUbE#PRfo6~zEwI$x?ah=|V%D<&e zD90|OTd2);yFc8@r?>k4M@tRdg*f~!?lj>Lo-*MlY~cnhZf4?T49AJVqJfDAD?Glw ziMLg_jd%C(3s-o49TTsw@HB-tGVxSfnj#S4E>O=b&_dzis(?E^O{KR|>4iMk!!J_d z$qh}siW1;XD--up>47aw+~2@4olCG3NQ^Zz_^ASMF(zJK5zJQ-?p64pQjH}VjN~;o z^H)^mqf6B*M#d=Ck^#*0eBRTaH7)r7Tesfh+hIm(FYswTLojIz2a5UmJQbK^}+ z1}i8$!NNeXN)K}9x1?WG8Y`QsGQ?R_8eOU~lfiHcUfkSdz_emWXsIyYU4f!sXvrU_ z=%u=?{HCRwd~d-oS_t%1O(s)EH?6;MW84{0N9_qTi7fr&a)U%v9fBn#x7b2=2=vpV<}&t=(~H> zBNmwL-`%eZtrb`boU>H8%pzfyrNYlF`2&@P1C^1V1%0Fc`KpMVrlyJmRFmXKnRt}D ziJtabu#KU_Vgm`Pe4xr7Xz_e`mTtJPrDpbjAH_hiDo|MwNLB^hgH3{J<4P+1q9uQ@ zlGxoV@-0St!lID_tx`=1-EI1zMWuNbqsq55$pH(VXTft-g>L#?N;MX4XfQ9;(rvO- zg>J`@rkbF!lDM)W;5OPUOOvHpY#`5~p>-Awq~@u9SwmG|vpAwMs(>+Cf#)m>K`%>s zkcGjOmI|{Jt~8Qr(=8R|S=y|egDEE0sPHt(Z)MWSZ`uRj$)>=mHuqqYY;m=Tihx^X(WPycXtZ%Viw43i>46qJ+A?;e z83Mrm-2HN}rNU&3go&kixWT~xStQI-5|&j~UaSVMDhh9DVK|NRv;DgnxMfkHO-bM; z_=6=qK{4d^e8pqDAF4HCDK1(cLTzU$odnMGM2J zaTbol)zauCHTXoAT79Bn_2Cx0gT-%zTiQI?(u9G}rl;CYSu}9ef(I&z-L7_pWwcAT z;7yc3Zog61BC*#qIPBkHsnGT;hqm38^e0N9QX{o6@V6yB$&%jJbF9F9D_gOpK-;G( zUD^?!ZBCf3#Z?INGH0spE$X~?Xw`JBhB)vJ9h$CstS06kbmKGOh&q!s4Y$5Eft=% z8$wm_%c^K*OQ53J8eVciwdQDV1gglqk5y(`JVy%-3^cPo6I->yrz&&!elv6NNhVyO zMCE=qI_Ri7x7s;NrkJ@P?dC#Nm`5|`YE^?SKQB7iSH(uthPhh!GgwW(zf@v8Jw&3t z60Mr6)%6Rgide{C{=krb)Toji%N#MmWf|zmMxs8Ay+H`28%oMP`>V<0#zU1Ug4iT-pm_6--K=} zEk0(ZyR9&KpNYGTFzt|ucQk=5alb)6x|T`CN6`ns>84#_^)|&cb_+lm6;>XZ8Tr5z{+5*Q>g=JbRzce%1mP~yaRl|sFA}s+L osGNvxhN_#cEz{zB71oeCPRE^_=;c~Opo(&z*RxT~9IfjA11KrKw*UYD delta 106389 zcmZ_1c|cUv`#(MxL{Y@S1w|1U1r&EsM{yYi6CD(jOjA>nGD}dGqXMKm)Ma8EIL;-w3nXaCy*GkQ4P^F9kXR6XvebgUxNZ2{O1cTdiU01NB-dN0`i_#__5b$kE0bN-u__2WKc>s%wLXD|bka(ot>N zq=zz2O>VM9*`+Gh;i?`Ore;_(lnTh&qRdejgiKYe>Zy?7%3zYv?rcMgrKzQj#r8gT z5C02~H?mloS(;d;u6wlUK1H3`JVEW=e89S8&5zb!H@{7`5;Xd;r>0NKn)=|(m!_|) z3fC2v5t~5nIE!sHzgQ+>X5i&talBZ<>xlrBNO;kGTqm!DhYa9&8OQrui~+p;IHO$Z zVClp0Dhcr zn76Nza6b`finqR0t!)dtexi zZ^@84+(_aBWXiY^0tvq+@FEE>7kIIRX9|T%=G9~4*My)dg%9kNIuwb}lu3Ajz{@3E z7Y0;Gc&P}bWj-G#klSyu6pJJZsbP>`ES7SikWIpm3d`ap{Gh;-Bz%krS-OOWxCU`e zG9<=G(IHd9s~o(2j)Z3jmljEQy2zno3C|J9>nf2L#e7~`yb_)+lB-O@O9Wmn&e;FWQajAatOM+{sp;Z;TmWPC6wMEUn>-4GOsk{Kf5 z@y2p2;eCXM;w3zDAeWmY;Z>qyrV~!%8zCqY9Wta2=>pG`aIY9RN5V5iJt&g!nZgsr zCY;T`48bTdF=+nv5CeE6JYGz{G6|nyEXNX_A@C{*pD;|U|CZdkAxIYrg-G~Cf!icJ zp2ZS7DG3h|iY7_8MdVO=u2dQ~i2*XC4)G!xGbP-1A6Gm_!sA8AiX?oPkY8-zy#7IR zkrX8cgZz(w#l|d<@O2_IWfC4D1}>NI)uMfsgu8we8d>t{h9FtsArhV_aGQh|B=i39 z5}rPk<4F?!9hc*>q)UvsLQsZ;&k%T~geMIZA&~ITgvCV?-b)xz%y9VMn0_CM4kc0t zi`ch&C48+AR3_ow1zs-UaYhKtIK{u6U|90&h9FGfArc;PA8%)q@Cil;B)nMMf+b0~ z1#(#arAv&YA$))g33nPHknl!A(Hsf4iR3MkaDUOhIA7ZTcM`5EkvhZ=*ogc!=<5QXL%euM*qg^g0ZS<$@SEL&7tJ zCo&~`>2R)Sj)bR+_C*q2P9CECE0!4l2v3wqc)C!?E8$6EfHDb>7kIgZzd4*MS_Pct zzme@98X+j)4Gi3Agh0YWgsW{5UL_WxcnJ@5a=A&hIG=xIB3aUF8MOFW&Ke<*aIa{e zDd8mo&yjIsIhJr+idg@PC5FF~Yg!`VA;P6z3AY#_knj~kPPv393d5>g5+hS=GA-`9 zAvj@#K*Hk$Zj*4gh;h7x7Yh$13A|?ipDQ|~OC732!O4*DOfhRRC48QclOy4o##&Gp zcUj8B&ZoF8!;)$2h$Os7#L_F_8DfAk3HOS5UoPSAu+Y%_ml$QDuvlKN8-f>vWg!wC zCR}Qh@FOBL@e*zmcoN~1|Hd>c6O44JL!cNSL&EO~g)$}lrV#=O-z(%7nQ%7$io^iL zCI(HvauNFy3BMus0bU7DH$ouc+r+@-5?(c2tp8OKW2u-n7Ej#}EEaf(gwGZh+ax?k z7#1(#Wg>Z#JYxPEA+QJw(xnbLVhfca;kvOSlJImfK#qiOFhXG9y#B$}!h&LhLH@@- zn>eQ{k?=AR8n1+V1zslM=Y{9WCEVo|*Bs@gmRi{8B3)7f7M;im@9Qi|CZwg5}vV_q+#A{25;_)kJkdL10`&k(MjP=|r% z0z0{7nG$|b2%0P5hMXJ;H{>jn@V`h7_+UpX2O|-9)@N^+R zM3eUa{xLjOVPMcdBW6a2Lggxf?8S(erHXqu1{=#m&V zk&Gb{Uc8lC7AN78g`ju|FFVWIr%Sl;CdveX*X;j`#lRVI2Z0w$c(GUuN+sMY=sgl% zE@7BZ+drDaBl<=}2xy1z%?h@^bB|PbUl8fk{7T zIX=J*3BTj!xb5A#iXJsqO9@XBw39w5|BPu?yoC#zAayAEj^i^VJbgXK=Sp~{DAmOh z9^aEISZcy)e2Z5AdrS-_S}1f>!pnt1x`dYrg>FcAkx4!7J!J5}s~&NWzm04@r2D;UNQO{)bWN z!UEfSbzL-3)bKb7FZS^_&XOctYsdRLCA{cw?zs#JcTErj%#avW*?z1?j)dol0SY9% z_!u9cSi(OS?N>>7&Ux0}W${Xk5D|hs5}y7Yy-kNZB?({Jo?EO-c)W!hU|CVuqa{M2 zK!(Hr#`KH-hsn3tqz)TJhd2ojspRdGBz&|`$SL7T&vOGZ%s9p07+{8(0X#l|cgT@& z!-4_{_ongo#S(7FUnSw0trfoi_ezXhVZj~=&wN{iK*9}+bqO!==k0B8@(|D~jr@vx zF78b+|B2CfJ5wWq6sIX**}VZl8Hfs z8~>ap+$d7SOE=+ojkV_A1QU+eSZn@enDC$))MA+doaG-48f;=@nmUL#nHhJk32$m@ zpJT$CneYM=-rR&2)#5ITc;T3_m(?;Xp{5SSCcLEyUuD8uneY-59%jNzO?YdDi}~j@ zG1{0q>@nftCcMmqw>9BMO?ZR}FE`RVI9Z2`@3>15J3T33oY646lhX$b|1P;e$PA=89UFyV7e_(T()W5ORc;RPmqQZ4SX6qy*4YZ;bhCVYwsFE-(i znDA95e5wgAG2xGz@KT1uev18LCWhD4VVViwW5ORd;bkU#x(PpO!k;kV^FyCMh(Ma$uV`v zG~utB@VO>D*M#So@O%?qV8UM`+!(yb#8_zRu*`(JO?a^hf8B(yGT|N*USh%vO}MMn z#CXHR@S5;9P52%YUSz_{O!#6Ge$<4&Wx~rj-keRpB_@Whu6un<_u#iOf^0K;)U!DA zy;*+MWzO6kf!kTDYWr!xSE^(G?y;^*#>NJ&DIM!stWiTPmVd30w}4-i=-{}z`~Zo* zE49BW(Xza=b@{H?zv=?9ha~zJiQXyEniC!(yE9=m%>ev-^fa=(#5?v_K`4T-( zqO&FXS&5!*rd@FP6f*#NoJ5b3=oE=|NVHv|yGeAciFQ%UBTN9%p%NV|(E$>Dw?o|= zxhl~YCHibF?P5vyS1rKkLlXUqMDLX7O%nZ`M6Z_Ul{K`BXZ?yAfYVDQx=^C?C3>Dj zXG`?65{<)L@)MkCNyViFQb|U81{5bS!9>x%Nj$4MHV4SfT?Y`Ys+8ndM!T=!+74 zHU>KwbBDjA28Sg27m3~}(VHatJBeN`(JNzc2V+)Xh16iFL>EeQzC_QH=xm98R-&gr zAOTY(dYnX$lIRqPcEqTk&&rM&6os|F(EjAN{uZYvQJ;FNy1HWXDEGOF_ebTQ$!?J5 zi5(cF4$0iAyr|Zj?P`2;FmB-7m8sP`^-JoTvlIMwM5xsc^RR>RcldNIB~u=9X2 zo!6?1o@=MoB17xpU(LKjYhfus|ZYUU)OEwlsUceMfbPn+}GwC ze~Wc}dDNVTM&)13u9x7gc$AH9fA=f3a!zE|S-9JG=H9N(YMR>budLl^?$fa9)>?l{ zMPvQJ8`agc**aU%6yAspSNmmM3hV@*cp`>(QhQ_%#5eAr$xgH$#Sw!u_ilAoxUlV${B84m0{*5wKO28HKR+{a9F8eQ0LbTFCo%zBet~0?Fi^oxsP|p(N@ZoQ&`%J1^UvR$KAA@;<=VQm0=04{vO|~Ri ze<1H}J>S^r3Ef5dZK$rUc2%PcI~(qX=Iy%zbJvT}6Yjtucfdzj>S32PFPn|Akd2Yg z#?T>?jp3p}+R$#Zbb;axqx~zS&m`$I5R<#F3#fJ{Ni;^u?+(g6G$s(&K5-p_>+E=r zhv7OwT-$KH6Pr$IAEz(B&D3gPsCBY2?gBiH%V-?rsKj3wZsZ6OF-byl(qObB$qkrfm*M1_oohvBbJ6t;(LV-} z*84DckYg|YN`88TBthw%p>*B>(W}YBHGUf3tyXe4m;5b?Kl8^_7ZW|3iJny>I><2| z*9bui(yG+ymlN<2L>pNdO1s+H0itRLK)i2~p&}4xjX(fDC9Wwyri*Kv)8z^HO%QQl zI7H`oTyGV4BCfAeLPBVgKItZN`yJFyX2AzI@Ao*?B2|JMYw%a{%|Yh&JU>h0ce!s= za%zolrn_j))a3SOWQow-pBmzta=ViJg=^A%xlqwYxVR?OUlx5y^(VzOslHvbNg@GG z{XHr+?f|O&q`xb#ydfEI1wDcsRro6z5GxGG3H^>6aFBGVG2l~T*XUo$Dgd9GdBSC+ zf4aCP{Vxc+NPlnV$GBl+K#Wk73@8zT$bfov4fyLiGvEz8J!b)E$Mo;T^iQbKKgiJ< z*T|s59o6DjLS5E6TA!k{;^kzsP>sq7o2J6$p9(ce<<2IR-xk-Tv7hKm8tKB`PY~yhZ)1~#bJ=gaU30k9Dj6C&(3S_ zT1ky-WR4f=lg!&YX3yFU4P7#tep7}{BV$<^LfK-$*$x;p zsrQw@Y3c=v>kM4i6H3g$^<06|4E&k;dV&jS7*FUh<`}F=`i@J?D=S;K0Rwk|jd9(+s6=o`Q3%c!mb#$q>SyXbq*GN;ff0?rrg$B-~-?kkX{ zN0qa&gXuaK-BJ?HRJ24BD`cm+{~`F!Np_g|J$e_gaF~&m zkm{KisL#WYFsP07)tMv{`}hFlYoXn-0twIo=S0D$s;g<}s~wrLS%IXe4uX6<%;`R< zzpTFQj%<~Nc24&;N{}5$xHR`YYIs=P;vVD|RZsgZ4F?m=UvKIP;qn)(nnDY+kEz}?QD2Kz!3*oBTJ%fMU5 zd3y|A8Sy-!B}9819Zu0iEwuluq3O~S6bWm=*Q`mz{jAIJ>~#GZb?@sVI^AqWT>`Gq z2P|7~#bRmXe|?+9QeB-LhP1v6-JwPM=ITJt(1g5C{4GD?bj30T^XCH)>CVD{Pz;lv z;MFw>4U#OOdOcDJ;x6f*sB1l;u2D;&$IQQ&ptooss-Uk&qD)+aK|P53B_*gk;1z@W z47gD16@M}$ctsd}L_A+fr)R-ys8R0Lu(ZP1v8xCPSV~CY=QH?;1>?H|${4u7;zx%( z?tq_&#NM1Fx`USx{&x26^8C1UZMSj3mBY%`mR5QBt|7~kZ|1hG329BF>kb+wW-vTS%a*T z;JnH?>jkGH=j;UManAWtaDqAKOK^s9PO;$pi|Ll{b!9CED~_|?5v)HL%bGWb%0ppj zea3WlMtAd5e$>2VSu@)i{PWOpX z?o*&b9|bkaUA_eeFL0RtFgi_q1VXnO-5N2W_TO4iyHly|qui-ma9>ItI`L87HJTr% zgLjYpqZWuzZDZWwQscW+P~8E~8;#eXu_zDLf{$2FM?WaUl0Tki#=hgM$H2zIhs8YNSs$-L3oATeGO-e?mh zpKh^JzVWGc?KU%WB~3~kqEK`9HvJMNBTcf3W~k6qWbS9{pZ!Thh*f#`Ac{GROPSXS z5_bVSuG{g6fDa_+d+c}Nh6FE5X}$1IwpnG9=nfO=*o<(KO82lsHEiwz772QZ39@e> zM?%95cm{1~{Ur>C@M5rB&?!Oa^v56EFU4S!gnnGYH$uWoCJ9e*2^z6Hjw<8;Id+*4 zFjWY+K`Rr+uA)zm;}jK%$9vVPI-(_BpoAiO`A+H!A!dva(}@N__itEt`y5kva#;fK z@U-Np`%49k^3mqPF(Y9J7;Rkqqo+Ows9 z&m)${u@tj~-02W1O#zn#tl$Fbvox;MPYkIQ(2)zM-~z@5Vp@}gJ4bj#+X@MjNdi1N z7N2RJ;CZNlD8HV77A)I>15n?Pdyl{hRyd&25Dca_G_77F%*#;EJxsWF|ew2|bqy3E#D0tquzT*OF@mOyUAI z(D)w5GeSTMAz-`^;L73>?68nj_|7Du4VTacEJ)}gB>aAuyQ-y-(2EPWBUH#T3HXC2 z|0%A*KlQl{ZwUcstx%R0-Ycz{5#cBsq(U!~gk@ZUh1aFeg@gx$gilBU1gwPs>iIbr z)CjGO^7Il?!*Kf$W{;zSCGsx4B9@KM=SHHKiBED7dno52c~w$v)P~b&9y?o6J&s7B z%dSJ*9RtB79XgNz^o-J1_)gaHsLxfhXB$^#b>pBCv2kCIkYR{QEl%|$=d*TA(Y_% zAh<~HN#q(&a2ETpJ0M*j2&JLegoknOZ_Vq#)n12*N86AuV`^0j6jP@o*P4Y4ewZ=1 z*03q0m&bmV7nBp&4I=*A^w$n@7xn;~)IGpBq%G!f`Z#3M#QYj>+}J{I4e@tQHa;=MqK<2`fz!-sKW{atYx= zh3&s{8=esYqPc)2EK4f&$4vqzase+A%j0q!W4C_W^jC$1`-Ozld^z+8FGQOpgmDR( z+zamr0m(u@un_Pe7jS@;+DiSjqt**Y@WdKk_@Ob*=;4M5LV`s|_!(^SV+Kh;1SW7v zmyu>hE_Xtz8Oaz&mB8cpg4w!De;4~$BbDExfgph|J@z$_K!V3Z7v9@t&HIzbbO|M~ z$MK6Fk7@P+?$F1DghN<3sOv9$QJRkwHFS9&!7~Q@noUwi1EGr*x)^G{BGSvlB@AR` zz*VXLJg`>Z2HY|2*aUD45xV^I8`tFy9<7rOBS-?Q8p0)Q$K)||X%1bC0sHWR(Uu!_ zRpExM5W==VnA45%@A8G>ZMr#nfYL@RhYeX7*kwq58xAq7)L7q2eVbS@U#O@Z^#e){ z_b%L=A7njveVa}_usuekG0%TuS&O%McB^OJi*QYY7_`6hQ|$=xXkXEmzf=?a8o#x= z06uL8TM%zIs>a9g52V$DWwbAKyo@_Tl#j6_-D3~HSb8Z$r}A^Pyo-&PX5_&Izd~U= z?j-?k*A<*)p~E2rE!90N6Q@o3qhhS}=tHNnTT#0)Vt;fYO} z-b^{{EeaIKYrEZi%WZq}3uIn#TE2uFCpLBxw zG@~o^X{@)SD|+icz)wc1&1$Z{z8 zF5OE8uV)6@CxNR+LTf{pV>vXJ`M`zS;_Ha<0OVdOo+q#*+O?Yx_Axq< z^E%KH(ftZ7jUngIkRJPNR#LW6GX77>kSOK+*+E+DSUWB-Y262|yOu$yuAKCbFy@&dR^1ouCj zZ$R`EUYehvM&Ld#xQD@o^$X#0SYHC8nf0IDVAflBiu9wo_&6*=M!Q%gSjU70`~mAB z&6swn?yvZR@9Gqbp3k=`_qdrnjbt;_$kxxezW`p)O=hJS(R`?7K5l|uI}OK~w2uP_%nzQBe?zt|DX z1;F&&ZLkX^CjU)PSeuuhg*AotJlgi)FW z)a~?j2}rzl!QbMWy|KEw2!#7G^$&QJKJer=6e8b0jcc5CH>@tVM&*#w+r=l^Ph{=w zV=k-jebrX^TmA8?h|ZzFEZJSg!Kf?f3vZsqb8L^}-Z`l5TMw-t$2s<{fQwact#&G> z)zsC|`1|bY#q}@EhAOFQy|23~539Yt9;~!c=X@Pk{{ez0s~>*dvwjRN^%(V!uOmAq z%z<8RSP2VldZ7n(9IO8U zBYEA(YKTYF#()6}Xh0-JLxcfNIHP-UQSsLkPjok|L4Qh&ahrQUbtj&^|4$vgHZ0{T zyzNVed}nSCC9Bc)Cye&K3Rr6D&$0?TRlIKA!imSZxVgeCNZ5;5G*!P^yCi5J^3Can zkM%L?$ZsMLo0;FNR~o7@>qgc8`yYh&3H60_5fA^#6uj~A6Yrp!RJ_UuoRa3u@Rqj`3eIKQCXMdZhYraoa>Zv!s9}(0~ zNcvfiQt$uasm^29GL6e~Zz$P|)7(A`u;2&{FqYEFKB`99FV)jO#4E3>%}WR4f7pAl zG`i8bZ>j5xr}c|!PH8B>w@dppD&pXux~3j3?Wn9#@0GTVJ4^!8umvx)k7fc%ILp!r z-&I%pTH}dH8WsnSI^@TGeGE z;M(;q{VpD4#6}RU6n_1RdU^eT(1ZAOyQypY;x*imkys7KQnhgJ#8*9-Nj@?(9)A8YQb|&~ zZ626W27YdeMWJ10q5WmlbMEUPLpbVN?vAiYQ!opM!dJM)NRy`ce)Alsxo8$4HBi`B^qr9(1ZE4fA7y4BkDq<6n=3q4A-A}w`e}7(= z_xl?4_U356d!MVeEv;LtChmhYAl!@RTY3A{D_cf4?E)^e3rz4r*C&efaCM9~y4w$5 za$h{gaulx+pdw*on(L?N1HYn-%#Oo7hx zP*ZP^u&HQ@k*++>M$(^JRb7oah*o=8@k@2z!S4Agw;W>L zg?XPNg+sFL6GkjkQ?_-DyZtG51`0A3#qAKh_h%;fBM27upj4hw-`N)4k~kn{V=PvV z;$V#avHH`tBxRfG+1|ZPFG#DnSOh5zxs+x?z0T@`+xsd*)F3okx6)|j-p7X8HjT^u zLj8Sv2TFnM16&Kyh~`)$VRGCddGjn=0Xdc9#dp!ALaWK5-31OEWHZXV$DrLRis)Ip%(3mQA*YCcD3t|B9lCp za#O#9j7EU1d4E%3#S3DXZTb=956e<(-nR_4uf$qSa(_`ncDHD=%HU_shTD%$WIXyZ z*M4=-?r7IH08q8_ib)6BVP|z^WiR_D(&OTXR4lv3DA_m!z*8!aX>OgFz)O}sl9ZMAqZYE8ZkSop1-e@wTDDTSR_HTxZ|A zL~(umeswiUV2v>i%M;6wsAWC$qb5wV63HN~Vy{P`TuYLPnNNtvpt6AzHBk|tAcJB9n*Rwk7 zv1;PaT_S!QOHif0m`0)mUdEaSItK(?+otCH9M!P!XKHatU#YJ8d3=v&(GD+8HuepL z>NHLC<78NDjUmqb1J?YOAZe){taksUWs5lU@r`8dX{ z2GQSz@vtj<9*GWtXS1T{3!X@W!}@%+vTUI8o7(ladA98us^R`3eFJ@-uPO2RmqhQ@ zv(&x64QRcJgn1kf(8e{t+M1UL=MD7)_dB9S97s&-OI%bLeDU;tSbY?F@xo`#`^pbq zc5j1OhruHEyt|b7KmQ}f!G7@)sT>29kv4}msvjQcfL-IB1FyKkfK_bfDZbv1!c}OG zc?ePp9lJ(hoEy|foXIVLwA5*L^cKL3v*#i6K8B^d+DR06Ix8 zw9ki3%DYFGa0N%4z>sh%SaTSwQXei@J4hKE^cUJ6B}GDD5^a)5-Y3NJ*r(G9=Z1Ad zv7?DsgAaCT7DM}La^7t;f%6(4Qb!z&_d9V=edXZLaZ9iXrpl@K-oSi)qKKTb48!?u zg2Wu9Eo}kO4NwMX&VoPJL{oX2L39x3)A8yAHS|#TwzR#r=J&#&$+=sTQm0)WN_Sh% zt>^0LU#n9Owdwo!?@$cnPdxwc2TmCLnrl3 zYRRE6Y~%MHvbD%n(Iy@Zd9a9$&ETQb=v@>da!W(Z1m};?!IDHZ!X0qu068f5;sJH$ z;r>B;--3XW+Zg0c_3Oh?e${WQ`wzE#>NCK`3UeHz(}I|j^*Gu7vmleQ#^J0!5)lyV z++f=1v(>gWGB{Q2^|PkZ3*XVHGz0Xan!XeH;;j!-M;{5D_=CS?3Szoo43=iB=9q-O z09eL$0aM^$w#5IvSj3}JKZ?r=P1x)S_#FLH*=Yd+kNxNau#nB;GIi6Dn5N)5`wb7w z?yy@SKjP=Ry(gA2ey3tl9Y@2PfA4R}`UDdr552RNQ;?ripFY~! zrWavouNazFa^Fxm0WIjQ_`T{UM-!9=>fxguTo3^@^iJ?4%0Vm&&TD}9t@043q2p8N zT=AoE7q4$XYjX3)?2}SEYnT2!@=NIYBXm6iU1`EhWzMG;&aOVzo z-cuL+(W`Umn;7JGO3kn7YCnze30?in>ZdioCsjFsZ>lH$h)@=&cm9ZVjU*MIIaYbR zYQ6#i;k!<}njg^WJ5u2i**BErY(fEdUoxIE{#L8+CzKPOfb-vSXS70NUa~zjoc;#- zF5upzJ#+Fy$>UPpS80ECP%S>zq1UGvmGZb?HPsXx*5ZJc(s(-##}3fu5c52Q9YTHr zoWe9zy?HE7Ij*)p-qo-DFg5*nZ@&qn)z^-;+UF%F)@E1&^T|=)V(jtY!-8+d)?pU^%-Gh?fZi zmf$A{aEP^bC?a1NJPscOGPBo9~hn3LVdnQfK@Xqm-+Ke|72ntp@`aV@S_hCig-bldFHl6k=jVeK( zTz}k=9~GJO1&a?3DX0Mdq86WsR`#kpP9!#2wiP!Sh$4&9$7-XKF-nQri3?9x^B-IEb5Lksy@aTal+Eyy{_?M(HTlf42-Yma08 z8a1#yy2;Hw7!)VmY|w*hzw*eyHRP6vYLwvemW@DU4WbnIT6LAT^IJ4R{iwVrPCO2w zZ&#x)@f)c1PxWl_H7a`E8X@kyI_y-Rc2x@z|Hq8A;=Eo?H2ImW9q08jb;YSxO-c!+ z_kJm5x~SVu4S8Y)LCtv<(FhZWK52Bkt~-g|V9@9FM4~@5=!?FAsO&XmBm8wfPx4=; ziHx^!;4qq;>(r-CM=K}QH%@oxSo#_Syv+sBfUAhU#p%@C?n*Yk`g2`>OFe$Nk86^f z$;+m>`@vUS)^#XQ@f|D5G|>{Ndplv$mj$c@>dnh2S6IyQ`$NHse15+?5`B?FzHfl( zgFv91`JThr`c{Zx2Q0}rEVxcH|G#lgwdn75y*3D&3J%a@tULEVV4Gt4QW-s>`H<(F zK2bgMcbs3o~Uk2@a3NAAW zI-j&dzktPsp;-gjqz%AF&7>U%0+ZG+h4}<`dJ|Op*)D$31J%dQj#3KMlCuv5Y{u5a zPYpWP+BO%)QT3t|f6TB)H~8ZzhBx5=l^vgc>os-gIh!(9oqld$+kLOXgPxCzV2ZC3 z%ZeOa(!}_1mAdI%?}oRbE1e*%Qg5AW(|Q7$`}*?i!NODVsYo{*l$~$aB$BuVhz*si zrt0MLDZTbV6;?^x*R7=3JqC2d)U2(f0&i>B(I?#s$c2DT`yu_qlQ*Mj;Ns@EdXqs{=G{|rRN|LMcnQBeIQJC zBYH1lE@Cxw91^hJrz{-95?MHQ_BDkg6d1y>Es2N2>ea*Y$xZk{l?e~~cBq&CZ0mQU zulj*+eDnFq}F+1C9z^V4Zc<*`_Z z4uMWCMV%+x1`?BPnA)pPU+mWNAc}}HcYC0-Uq$vAnEbap;RxvIAN$Rr(W5NAZ{R*I8@+{m%$IK*UH}u+osC zv(*1xZm*=Ni!X;Nuc@D1j%XPOZ=*)!;~hJ;l;O6lTA!vKyPT|ir?$N^R{2JqeI;E9 zSGQg1)%ZO2UD!dAN@=R{Z^t;SNJOlL5aaeMgpc?Wnu@Qh?Z!<*`o9sK-^22?#G5uC z_S|y1!ow?T{&@@w!qBv3^P4shj_885sycx<4C$-@5(D9=X%;w0Dl{WD=FtF5+oo_<(NOHA z3LS6Q)q-msQG!aYwLn4Hc`d4Q!E=za97-V@(%4hF=YgcrU5@V#gOO?U@NTNw8<}F94D1 z!Ebzh!+`7h3jiuSWc;&l-Dvfc#!On;i7LUh4cw9Tq@G3y9(tZ;F^(^-`G2!nd|ru+yq?1gEbJCY)jc>yRKi&R(&_30`XN!(jVuyH(M%w)Q&ehDQ(pUZg#n^Y8C|C zrY*@ic+2?iuOC)bL@)Ynf6E3jnEpJeIteAu>A|;L@A^KyMv=kol)B|+488__=4Qt( zOPOmv?8tN-8(&w~$0@xBLNeu4U$xJzfg^AI573sI)%ZI0xeb&T__8$?DBmk&1X>*< z{`cTRnMf`?m<#vzAmRBnj=+8HtyVLZQTxZCBWL;YnYsbLEn#|jJHQ;)rO^IA^yf)P zUkpt1bdB-FGS3Bk!W*k-3~ffys}A-gv}XG8E_j@Dd$fX|e!JZv^e4E*iQ<*F8@eD< zabt$svHSL!`T3;sKX$+=D+Y-&w{S|SUF zF_)j$QKBjplkc#)kt`=`faa^5qx4Ba@M4_xBuq}YrY}`z+#A?@4`f&TEw-Ap zTN-#H}4H`y-DSHGOhIk0#=}dHO~q{e`yBH z5GQK$iJN~M&n4)7Jp)QWYj(?S&GW;iLcd5*?hQX{zRJQuM*Rps8ymJ)(8zemtQRu| zU-2gUsq3r9DDg{!l?dgzrCpSMO5oDzicNWP>1#@)YY)y-Db~@c{dUcJD7A1HHX}GN zzJ`~HEcyU)DAO=|2lf)Ba5v7<@JH zI_s0PR~4m;V%5G@l!dOoPeSNhz-fYZ22pVY^X}#%anlMh*){pq1Fp5{V&Z{UkUfKwEe+RUkOwEdVHjf zZ=l2~CEC0ON~DsYz1KkL9A1Aql_0$3fl8PEFqBSn|4N4r*Y(NTsRl})KbrC&ql#q0X5Y1)}aO8b!@={_!mHkSP#Bf>5&@T2YYFLRqG* z3{nz%F$3@tCG;}(&yUm&@;2TIgqdS#_2s%;(H&YyVaKR0JD61<~=M_*3>{Gx!#(S|1D;U#RIPrM3g{qVN(k{zi2C7`TJ z+IoiumRs}BHAIU6_U{PG`-2E%aO+06!4ZEaw%(xw%L;m=BD;YNY2nW#3G7iiB0E3K3TT7EDBzCc?^ zA4{~IoI1-t0-7MHf7d!U!Q@=4rP0Tm+W+X|S*-{k(IY2|sk$uy4iQsz6$*g3A-=A6 z(hgGl2HLeIn4G#6YE{-NF0IU}bm;Dav4z3io6(28Qyc7$=r*@4=;Tr?YgA|26O-)^ zW@`9VseT=-6H(>=;%1Xt_<5IAyE$Ra2#nU-xirPg5m9nWNoms>F^T1p&Sv zt{@<}cl=Nq>xD4I|FhO8Y8|zw6t|M3aV5|+>nN{yi{$bv!mJ&@VdmCWhC!{g=bI^! z2}d7-@Mo{Gg^|*4577@A^nSgR=za!$USFl{Y^HqGk)M+?;EI4z}x@^swBv5>hG&ZLA~ zL&RH*sMJ3q;vGip)|YBWTPV{*Ut;zs!LV;r6mw~!-bEV|s&sFD;XypR9vkzx?K@{KVd?qfu?UO_|`5XqZ5LTg_a zs7Q(MyY_BN6rAt0(w0i6(79}&DM4ghoG@;ocCDq-xie=)GFB(Sa$>lOMai4GlEjzA ztx~i$qLpG-E^0Zglop+7TZ4Bs=#HlzHavPFl8WEGWX(h=S^JvtEX$>BZly%nUZCo5 zhU5+d8Uh)npVn@-QrgFWLA#_tjF06h?;+fI=o>(o9_(DulEakdjnI-k z?9)$bkAx`$V-BRV_Y1;&Co0jWH_{2(_%NU04Roh5)@Yl76s@N_ir1+v)f*6$;NUe6$>VoTRrc~m*5z0+E0+wygC0kV*t|Nca# zAvn)z`&wh>HPrrVt#tJ-XyC6!w^4fgoo%K~Y@-ZUCTkzIQ5-IGrKG$L(~PP(l7AJW z1?@lkRax@})1e>sQCX;W{e}fv^XTi0;^D=0{o?)9lNZgg)N??<2((0f-iga7T;h~v zXrPXVzbHL8T!|qluLBY8^M3lT+SG6*-gOsYM0Njl5eB8lE){HNaY?r`#khDYSL}2F z>3m*q!&rQ8)efitK@5=+bSbkKgYDknzEi<(F@MOpmsc`ebZKnPsqa^&$S0?J@+tQD#FPKPe5Dr zfp59b%IMoQ{k7&1O0U>l!tt8Kb^Q*U2OSFL!&-W8)SCAI48evW=sE4_2qkXJA0rX; zgV>d_7ghEVwGy}J;>(l%cK?H)RgyC;bvJ98)sO$fOGZ*3-!_Im11oHeBzKCkF(aLralK z8*pi4(nEB*I3U(bP+lq#Q0poDbPY>^WcN;M-VmTE*!~oZ3DS;7DPewYw{|s3N$_tO z?5}mQDV>xy+JiQwW6%{RDoim=7+uS^DNzsa0pL4~Qo{~lv++(+rT#6jl2bB*gcg7Ye_XFfFd>w@3>y9R~5U$In>2&!axvvndH}H=&EZ%G8X$xXKguv^KIG zT>3b|jI79~|9_iYibK%r`W9_|J0%$_ytkb)q}#L1FFTNn>9}Nm!3Jmv1_U&YpuB}d zpkwt2tw(z$;eOOXeQp|Q$KMh83BHE~p-=}2cP^h!ZV{tgg5B5g6j2|+TJyW!Qf#MC zO ziSdii)qagt!UA8WaZY1Dc?qL7+`}frb=|8)bWm(w6{Ab9h=Rd`DpLNg^<|!+Wrp8{mN)9vLm*J|7iDhR66@rCu_4hDo?dh zlaZ2xurNSgO2S3GR|=JY?g9Sow0}D)9mB>Df9-Wzop$RfL`3394lh&nj8l5L0s*9Y z8u+lJv$f+!5*k#ZHECBtL@eX%*G~{}i}pTvcjh-D?y&WyTHis$@qejLwf+ML61bVs zmHKDIoX&{z`nyDo0s%#R{!qh3!{ifs6eyLO9sFz`z zuRhZbU(uv)F6=4TBUwzv>-qpd$(!&K1{PmAr1P%6L%1Yv1aD4}_4hmu~+58Uh{jG5KudPP`UJ%3>psO-Q z=o}!-cSLl#(uBj%-mtJ7FcoVm(! zLc8aOoETY?6Q99wUr)Y=<&v{&B=5uM@aDH%z{gy`LrlQ%8Uas`00#u%%!dp1ebfFP zH@gd|F;N6G5=5$}zHcCCeX%yVtJ3qa6F8~%{m6{mum1*uJ$OjQ(FS_RO6W#oA7cfBE@i%41HF#tOS^w@9Uwt}LJ)w`Qwi<-l!c&ZKj z6@|Qau_a?Qt6G0Y+tE$w=vpuU!cH;Ear8<=EVsDyQ5LSx{$&_!J54O44pdIM0^zW5HO=k$TA0#535$E+`L{P0r-jqg)*^zxz(YkAcdng@T z^wxG^=x{Ff)~&ip*obE1Da_>LO{1vqVO&C&Xx^zA@3fP3B0ksX5KWm3n*J|iXgYz2 z^UeMTSD2^o^aHO6G)vxeKMCwNnM*t{m8r4yGP5p=Nu&l`VjfAfq9slB@g(96?v&-c z+v~iWgLNY#dACtoQoNEdmbV$e+g!RS;#S-2Hj=G2ZfUoei7k1r0N$$Q7TR#A~~>gQVZm4ME%Av8O~^rzbr`{{d~V6cls< zpKs6(X%l)W5v|DWv__1k{jB>KJGr~Auhkaep~rwSh=*<;;x=bg{$6X|E~XnQ;x#=0 zQz+HFmDZQMt6aC9XqHCxvT=-lPP^Vq>0SR1KsZcI=&gix_+Jf|Y4iyOPV+R5A+6r5 z**0$0X7%Q0dA*fzWwG{EZ{@zeD@d@%@jpy-rpD(?4IDA<*AGC7Vf%KjMl}n_No-F# z*p;63zd|qU@gA`&ZQElrX*lGCL>j&Y8@{Y&_{G|LcBO;TNZW45ZNV6=!j4VeY|Y+B zc_eItza=Xe8~N0U{uca@Fm8g_n<4GBFZ(EM+AQo%{@~luEZq2rH$m`&5ZX!bl?Sz3 zeUyQL2ni+o2il;%O3Q{hbhm?jPL4LEFV3QtXzspB!h`LBQlmC~$u|5qJmllq?3Wy3 zzbBrj+rmtdEs+HCmlG=bkOutr@&MD$Zo?*!?kCE$;6!X&ztR#CaguXR8=j~{g+Bp} z>6SeZ;`5_$UzCPlhQWrlQs1a8NK^(Yv$V~L*vCDs1@}`XDOuXge#*$kaohr=3TnN# zpJMOyB-gf_Ie-uUI~;%nyo-?6Y+9>&Xp#Mu_T9$uU8{3IL^E6(+x-V9{RTv|A}DVt z5%kuLS9`X<(yQ}i$Z@h`D*8W(klZ*F4r2$xK5l0?%dJYix%N|koN?9H&i7aP2VsAV z7tzU}c3Rf~N^51THf#W*Ytfz^pd9u~y{x4SR5pxzrYj0hN9;O`7o#T;a04rB-CNU` zG0>tg^u;rNbZdI{ZsYI?tI%_aSj~td`tw9=0f7oW3ITh9=rN+@L?Y<|Ew9(PyfL2*o%cE97hE9J5A`07Q5@kXtR8{9oswr2hfv}P_%to zUXs#Fv1_GC%7ixev%w$oQUA~sh{5n|%xPNZWX0B`eH;dRfo=z2R;V^187IZq6(-{b z*Q%XJ#!@_r^04v$BkQUIvP_=8yYFFOfI+7M7J?X9h#jEVdFq{UcIT;QV1SAoo;`N! z*{!dMd180>si*s%fpMMAHwL)peh#&oI7hB{ki#cwf-?+M;Z3qXm!*<|gWw4h+4=<98^3 zA3w%z0(Bc|8vmO?k4tfYhT(BW`? zGl#r^516&SoC+r(QKF=nOE;oK`<&3I7;g+TS`-av!_YG|3^nbW(Lt@bc?as4(Fbtz zPmC6&^X>CP+izG+glyPEhof1=JwMlZ z@w<^{M1yt!3YtfTaB*yp_7D%A=z%KIhnn?5H@-eJu$S1}v7}$LPgD&Y%f}@2TKS9bYOu0EoC zjR#Px>ybS^^XBAW~s7LI>%@1y|=eFltMKMM`bZyp}@x-P7zBHzf@Q-Z+am`AM z*8JIq%ci_}(owtyR+{0M&W20Fw1aS=7g|aT?~S4GJkCT(d(4c!Hy7Mq1NminsKd-= zqJWVmdlTap&%Yqv%Uo8wc{UQsyRT^3D8>!#Ku>YUwLQx)2I#JXKsufyy$H1uTZzZW z=j0dES=6w@{(-H>E^R6bQOHtvtaMs2$%3*zsb=E0oZn(kyDQ zH+ASI>PC)YL~y|o@YBQa<7Qat|5t6*m>DKPERuoLJ?L{fmocC2P51i=|FU!S*chKe z=*@0K+yL|0$1<9Ce2^JPBM{Onox=KyLY5I8=~TbJ2(-MxL;wE5-BJ%(3H?QtYTt8{ ziQy&DoyN-DHv_9_S^B7pGe}^yUo$swW~}<74FqZUH6E&Q?GOmp96NkodhmV#5R;9` zaf*up+u0Fp%31{!_evnBF2ocMMghaNYUU0E2Z+Fc)9#u1@gMYU$PWyP#w2t=C!Sa@ zy9S6Ra!L!zK2QY7Z9OSupqT1*u!?DI{8Jf=CB706S2JAG>HI+9QRTf0lg*gM_84h~ zt#KC_OthYMqIz;JACohCC%dFm;Xz`NYej5XTGMCqxRYKaU)|E_${JR% z%eWex@J-(f=+ch0Uob>ZL&{9V-4#w|i91G#wUz88xI-P~{t}H19xQ&h3@MmS-a~|+ zr5PTo@k41mbQ%JYQ4kNKhlntzai<_^6K?m$zgKG3kFK%+@{^Ku?8L%0N` zo6r9@0k+4WyA@>I|jD}jnPk9 z)^&mB9VrtzC9V|!6^4QSHca^D8UW@2v({)#yN8MJfboSg7l9wlIWXvEtukr`byl4V zrBlA2Md6TskZnp8$SiGzos_|my!jhT##xm^R~>!^aqH(#rx8C3f6H+^Ec{t`THKJe z=V#HVnj2b!j8zy)BR(w~s*P$Lq#jd{`!H^F{28|$0;7>A7Q;pZ7KAE=Hfj$SfjL8f zr`n392MiaUE(M`j00lM;@M1VG-8aZuGF${$zBr}R@!=xKGQ>HZUJMt-EPIjVIzmKN zJH;(0hAe;%sPSme7cufVQMyXYpE*`%&1B)hC}s*2XLTQK%oqVY9w1e9LDL6E0FOUd zGceD<84qW*l6xHiJT~P^rvf8^M-ULIGE(?hh9RrnNYPwAjHb;av6bcPNEb(n_=X+B z%oXTbhF9Pg^snCK;q{e*SWAmj`~^)J#Ni+gv>h_+D>9F^UXiX>imIL`otFIqVHJpn z{lAFPrMqDN!%c3_tMO0}=NkEEBoM|0rnUGd+)@)iSeP~R!kp8bJVqguZA5*lFbXE+ z*dM6VC=qQbh}Jfa!aALSuHzxGf79L%mwj3Z{LeZaV3z1-l*n0{yWtWU>$DLXD==CF zdYs6~@(NRDYx&AJsO77Jw&qNyCZmOmWnRv7Ymd>;YfGn`k=fMx0f=#C-nkETxy)tT zX{|M;;en&h$@)nPk7_82lq62kmo2dPu81*CnII+?83Xb{i*1cu zh^M6|SW3BE+h>+pDmp_Pei~(I#Q-j6y@Rl0sT&;VNOE%sg+`OzMu*- z;H*uX3#t$UHdu5@4|AeuP~>1XBsrllqp_8Nch5AyJEgN)!u`ad!~Pbm$2d{X5`>4_ zaiU5gm7WGOQ5Sw-h=l9uR6JhPk~7n(Z@ln`8O|k+Xcs{$Vz&XT2LNU|q-)(!tvPLu zwlE=nAX-!%q_BIz**28X+y#W0q(0GfDPA;i?5yDr>Qod3O%he);xy_!35xH~G@3dI zhoFtqXe+}(X>@CnC?Z7~eVZiQOFYA2miDTQ`HX-hzBuWcJtY;^bk)FP=?%TfD~R%&bF7@%Wd*^xo>o4vZyFOf2HrHVCOrD zqeOydvh z#p4u*F9vDZ9i?vlPi75#F-QkDjmBrE4sR7~Ww=*4E>!TjrFl2XF!Mbv(5I{$rao zTqhZh#UY<)!*t>89S)pqqaNZ^%-Cu*PxvEIMMTi!=^_NL-%^1YqJ;C22(*Yj->_J( zRH245gs&`EnflCttZed*w#*QHiaz3*0_IcB0dp9t>J4aQe@kR9(%@Em8b&o{ihk~X zZ*YqD{V`J%&5_HBjjL}vy3h!2BF9-GxY&*T+S#z)2-erJ zv&neHk2hr%!31K8O>v>SvxU2(9UkXKX$*r}I{%d``Om>Q=Udu43#!zPw{(YpXS}6v zv%rDF-jc^`G1F4h)k2$Ri-6qw4J4i7au%Wvv(S(o&gM#Ev%>vdXc=JfhAPiNgXP{( zd;ZP;hQ`c6!|z|y{5c}T$zd({6Nge-j@@cO*XD>La&rJXJbY-^Tv0;ye?<@H3P0%* zN`geNoc@~dcFxO7;OfNQ(^!)?P-_mFjEgXY*$Ay=7&Re=gh-sPs zh8*UJ{xUk1hRqXY<*rm(HxKjX@PG{I5NB|Gd)T@WGJ_|hA ziSd)!uH_B&nJ@fZdifi1m=7{pBJ?#an=eWh8yv(+BV3%o)2>*I(Wt&P-q$GuP(o!7YlHT*y5!@wmLYEVY0!c@IY;TMg9v#Wy`4I z7V5ANJME8=G;X1Yi0sTQ!>xj0H)6KhiCVg+f7RSqnRqleRQ_Ydvf%OaEBd$)c#KBf zEO^X$P9+ur57!q49$#<(rSae`WYA0cVUeipRt*hBB`h*N8E$?jS(T|un-+=i$SPZy$sp(?il=`=U(}S!y zHF!aBi-D6H8v1`YIY!Xc#iFidNRWlRmq2A%RE26S5#2q`=HYHX=wrxicy@~43w+dK zMMsth7oY4(uR~@lcc-`quq87*1J9q)yCv9TPCg^orJ{EJfB+oBC68g5kvPIx4OP^8 zsR)x@{-Q-oMaRHT>#&d@kD8)DHq`?t_Npw@eiF8jaD}rLix}TvNkM zS3_*newhd=^{XTB)GsG@(N9{zU7$UIK|MJoE=hX=pQ!?hHZBv>+?w0@6iiqTOF?7k zW20uvg=^{6xj=%-O1Bs}0)Y9tbAaJn9Lp(juBLCX_%oWjT$B%XWJ!&kG$%W~lOm6~ z@?7T~>cKw_@7*{vZ)5B`YHu*TUoHZR{$%gA0h1K;tf5!NTQ=&AMH@NK-$K<_K!gmg zKz&z;4rS`F@?fW##(Cy5qjHAJkYi&psKHQx)UaptX@v+1Kc55ptG2}K@*g3Y%h49U z^}?5|6^}{S`Cpz*45t<=MdLzu5IptY9``<_T`NVP+ZE0Mi*MBu{DA%>xO!EdUau65 zEGgM7RAZGGA?rM(eXHPH40?zi*$oCCHnm*YQBylA;{unVwyiAUO6>P){7?Qo&bRzG z-vdOo*PqV$SD}(+slU!4Gk*%_uYT?PzXhoKYT*}FZXWx1m@>FQBq?uEfat#_*T+W2 zP*QloWt_+-fW4m3uGOMq4d(|?OzxW}w^;ny0Nd`rfEhON8^4!fC9atFcmBZ}8BU82 z^Gg`G;JIci_?4y+m0Ba*o%SF&-PT=?3}1S^Pc7Dn!cJ2-(aB64b)P1#5#>GVa^iT{ zPAY<*CTS}wMTOp{i)(~$WIiOuB=!w#YO4jMfmMsI_fV`TYcp^8X_`(>#ivRE4Cdta zm<=v?tQZCVj>UfuuQ~k=T_N9N8uz>C)VL5XdIKM;p>?5Mw1Kgd@p}6Om5p2Bs3U%Z z5+eBZUBG|q8OWv{GjJM!Nu5N8Kc$62*TPVK<6@!4{4gFV{p}Ca)(Wo*Ex1-u)Z+H) zG4PWKRAwOY3lHm=D#gGw4IrNDE6%AXwmYNSdWmOKvK*z&+& zhV*HOKtKC2X$qLekUeAWq|u7a!pp5X*ZZs|Q1ii9#<1iqy1H4Ea~#Gb1TOddDE}5H zt(UG*$QG>c4cDl}77?2F3}XKb6GHjlplMr#t9RRLXzwRea5Q9~z6n&jMu)eEO0heB zWpU=PH;KiWzMB6uqUFHSqi|WHft|nt(Dof(iWs)hHj&*w=dPlyPe3$zDvyq13e{dt zi_nacqBb(X_q8Pd$^}lczhEAxiHu4kk!Df<+@xQ&;$7N3mot^5lc)j?*LsLrdYKMw z6&|s1D4_WSJE`775l&{5Q@xOtVXltW(JCNcJPe(dK-OIAC9yy8vblT3Y{%w~6m<@u zEglk29pfhsHc@-|sU@CR``gIC({CO~AB+L0#SE8*-Doi9v@0}l8$#lT;+RPNhU1v& z+ptA0#R1O4ZKAu}eS@O53!k=!oFOQ%p^gK}Q4USvpJRwOu&9uES}TYPKRLQ;z8+CGNDs#)?LJM}l`WSbK&hHR@1^x1|lK3npp_8+|OF{L& zK6j^ZEwVW$bPAZDIaO6~(mEjPjIc83*xJg$tM%8Z(oRvj{Aw&R)+)kK0xSgW@Eyu? z=VA84(e;KFTn;V59j>liqlG&~koyZK-7~+Rl-YthaglEB6rrWE|0@keFy!uge$tBI zpYT$|M1(xzl(?%5fL^~|p|ZOWF4^ib_1Y!;>s?czrab9?Rx2P9(SjYb6rAd0d&78((M^GW+0oUB4oi9l)hd5K~Czmf%j8&8> z5_b;$Wj&c}^!0>5HQH{UHi6VF}dJ;y&zjZZo) z&Cp?5f~npjIVP!w1>)*BB6)$*d|f_#E(-~b%!^A@c8@p|oBJ%UrLKXMY{}TUr{lVo zhqee_L2u7+Mqj)P`Un|z^-bMDvh5=Z*!8Om>Le#!K@z%q!BMr?K!;+CaHTk?;PY5; zYw(YmIexCzelc(Si3_K=_IYXGUQrMM&=#7sS5%dresp}Vs2DO5&43NGm9q&!xK8n} zU?8dnnsR*dGMOzYeKGIuvs8Sa@b;gFfiQ>SjNgbrw84fo4?a`XFK4OiJ`wDb1am_V zm)W}#vp0{i25f+axh%ACpYSWP9Jhej?P|r$#En8_{}Zc?T_NKy(93-y$mLNQ0>NPD z>YbG|*PsR3&&>YV!(C!Odpbq;H^tzRScT54@ka=huXV?Kh=|)uf^R!^U=w9L*v-(wNCQ?ApuU(UdT|wUp73S%pbD)qBlaOQ zL}&=x5lk(7+%84kLMG3}T8 z^zd1Wf!Nd3@u2VySPsm9;9#~CGUNmw1G#FOIZa(WOG^&Io0Ep|G0`@krNakBs62m) zULO=e-t|u#B3$3Q5DHpg3-}p(r;wC9WO=z;nVW zs(MINt=bZ=J?IfX2co9nGfvyx4Z7!MBP7uw4{D>HfyQIz(S65h=OIzD&@1Q(nY3XU z`XYti9ulE3zjBjy_OnmZ-#jK3u$cZ?ptxZV;*uWI5E}>nLVph#FLT>bL$F2lJuD(~ zUqJ^z34IA1Jd|74Ll|a*Ew_C z;LDn=oC>I^3F2}`;*k%xc zY&3N)&PSF{oZ?~Er`ulPA0H(qM!xwA>@+b4VqiuGj<7Y?ho3+c_XZt;e1lr2%R&7( z% z3ml>se_(P_lga4>&X=wxQ^^yenq2gRdY=%L8&o!8j0=ors#a3%&~$Pc@P%y`v^2)F zQZhQ5z&cSVo-(-4wvZ3Wyf||{>(cp?>E#JgDgTXwENvi-ZF$-1Jb92Rq=@R$dVq$f zz!UV>QCgQG29$ep045xtwK?DnjBTAONZ@*vbVMd>i%wkqAX05hx&0jnsNqR5GJhpb znS~TC?62T?zIIXs<(Im89lSfTj7z`UPq|NtnuWIRN9o`6i$6D5G4oeT_EV=*qO|On zOfyc2a?X2zuW>8|B)o8Xd-W7Fr@n_so))Ejg7=!a*lBnrRdR}J1ot%~?7Wv6pN0bZ ze$Ri4z(vO2NaG^6k(HrHiczimAeHbaa6!C}AfKpG;SS41V-OjOg9XKDhoy>#{rj1Jemq|S(- z+yg*w7I`-$LR!v>(#>~)1h&51SFo8iOJ>?kCoo>y&rk*MO`AmI>;1etV5W6(h^7K#cGJM=|gl05yo8Msoo-%=BBQwhpPdfM4~t(y9xh zNb3riM2=749X-*tCL1LZ`(e4_Q5BY zM`5-OtM+^7*d-B>d)f%5_7l|_`gpF(qJCgiAj2b%E62JGc23jz4LdY0kS#Xlxs!%o z#>xN4o3!|{@F>{*Cfcq%FsD=eMm`=(42ge6r!I?#kf`mLW1L~6reCz5*9=1ml?~ir z*D!}7z7iMZ_uEQku82Z?hjZ`f$Qd0)gMAs1AqZ*2?PsN+&vI)2P>dpnyoQ@@vmS014U$0R^kd#3K~qHR}1J$Y>leYzs<$$vM~&8woOEVz@r zu8I0`!4~RzO%%^Da0`_0#2qy2nh22fw$Pqy*mYiSqWjliTJ7CLF4rMHpKqty*G2V$ zzidv+;t#>L{pVGhb6pfE{Ot<%|M*jdG5Q5tF+_dMTxWz8vPzk*vTvoE*Rg9o-a=ol zi@vha73y_Ew35}f(cv3VF_v$qdpAT~x0#n^Hhdx_0%FxE-k}KS0l${<~o5o$KiK zT~Rs5NC-1k{yll#6Kx74{*L9*4cnu!`>RpE)5LqCPOd*7=6YvSWB;*UyeF24+}_}E zX5}Y&)2*W(h_A9({0h{Z3Zctn8Uf(>KGWn%av9bh6o9O6CJ%EyIWHFajwU zzF>z_W7U-ZBwSD<7`|^Wa6|QE_@y02I2XfT?eGN^#c*zj6OG^vRf%Cw16IKdlm=iQ z&zha zu$JMK2CNn`z{;0_Pz()v0tV^3jBewXwmpHM3DiI}`D7`1{sjYQ=Td6$7x;4MQkwb~ zUIp;hzr`rmQ{h(R?h@?nI|C2v%_9aISP3}TjH!a_++Q77LbaYkO`tYlu>s!sMt+1-0@MnjNPMz6O66sV&rgNJZ@|$b)O)a3l)pqHGIzI2@^oTQPq{ z;5CaYe6Du^{rUpSz4QXw@&acn4;Rq=7wF{2g>zqG@*)>f;g_OVuDF93S5o-BcGT=8 zVv>HJM?GGO@v;!n-!Db1)2z9e`Rh7(?tz*#mqxr2#q#$%z_eHtI8M)B-f9PE>nl;B zelU`q;_hk};RAeONKf08>exJZhd9U~;UJ)XZ)K-ZHz~6I88$|;!x-6VOwRdFMuI2^_HZ_D<}0dyuhSj8Wv+xRMY$PW2$hg@;iFUoCd=w;$c{?2lSNci-X>n z_@qOC8lme>^Jq&dUVoV}&tR~w+BU`))bXYQuGpAQk5k1wxqKdtc`d>!R|GGq-k`O8 zX1;S_(Vl|$f03r|3y!Ofwpm@;4?WH)@g7F5iqEB|uSK=o)AwRm;CE{4@1xRhL?f4v zNWeTrSMD2#UHLT{Q~ii$yunU#aUN}cBgzI|-NTc-(H&WX=46>4{>jvr?OS#68*;#7z&Dh0C>g2e z`*)(E^H3|#Du2S5JM^7H%if8fU4KNXZ960auf=t{H4n*_RO%n$EsxKj2LHgJ95bD^ z|ARZ~^3O6bfu9T)#5+{s($JeePuFCR8-(6*F*D_VFaCz;o_b|I_b zCEgF>jQn#tC4EGE%$ljR=OY%##Hn-#zfPT|!ZMP~(wNjCk16*jQ6VhiD|c%M4*SH9 z0(rU2Ayi-?>G=eIrkx7|;p)t&(i4Mt|Dd$&FGUhazwpDq6`p|=OVyM|{%h@#D&gl*BBR$b! z>}FUonhZ(dpX)M@{u+2w^Tr}mVDlODvLqV&RkVyP$u%2k6ER)+Fi=zj7nCyt77ak( z1-B(W*_Fv0ByU%&gsC`4x?arl;p;O6bsB zj+f1;7EPuu-$a|@UZ`mJ=99vyj#1knXbOXlOcLxgxj@4#u=fZhNLJ^Jm!cI;l{k z!k+G=uPnm$0u=7V(_; z6}d^KJdHle$fic67$OEl&!*;#Fv=Thg>jJ=-x|vdInvBCY_F73T7Q|N#bm1b0ht*R zY#>hJFzPAA&CG{K$Tzq2jx7sp>`_c#AOl$DZ$83UAfnVT>NFOghyQ=9^$l*I|9zcj zY#xN81D=!9NXzK_9*&y0bBmeT2p!mJa+v#fi9t2D(81i&-LV)FqY~o+PiCVx&!r3f zlUo*YKf5H8l2H}xluR3A_0J;{h0BpqsDqZ{79sA51#+T8_Qlr2d1YTAYy3i?PSVe5 zi*}zu&LB8l$)VK2NfxbsdkFJ9V)#*8Yj{p^gEdVrs6)ug!2Gg4Cqg+!-`Ok?d614c z$->fenDvR143_fcV9H-W7L+#!Q_%wQx@)#!Oe~Y9K|U5FeXUfWpzJCq4WbbRWfivx z#G?sQwFYb2#5INtP`ifEse&>zhXZ^Z%6|`~6_ktQqg}M9ko?h0?lA~-7%j5LmR;X5 z3GlGntmNV%R~M{hMFC72Z|3}`n(d}@E^?3ju){jpRr&~db33hZll}aU;LOd=pphm2 zS0+6lMCIINGw-b^#V8K!XQKEVC_e6wxvTn|gtH8@UUZk^MbQn&H-{THhhG>-bhtR( zs5-q5R~;Ipa`dC&o^o+Pwavhhxmn}*VJj8$k|X8a&9uTxwvajc(+4lAtrN@EB4SUH$ak1S5Vo8FQqU#?W5m=_CDR^IlYy zf5UpwkNlgj7mdSj?iW3Q2(ykQKMgD_-Q?+>bes#U?MYAhcT!Kv>5Jd4J*g!B*6K+y z{OjM7`uhSw(bJmbD|6vM8aGtfH}LK(xD#CxSK1vP{JP*UbcLSc+p{j57`bLK5XPH+ zOZL)5Kk4llg~GgK+Fqr9{bYc&^dc{R=_Tter%L`1x8wHEkN&b$xsp>^efWw4R_KlQ zG?1##jX$Uq9*lWuev1&WM`Cthk!va)1io@hH+sOoN!#Eb4mbx~3l>qb5J*P$CF@bBZUw1Iz*bft6ryR<8%^6!|gl)s4d@7SJ6!9?B1 zLp57wb%7P4R8`fUJt|A~MPxThcnuj1-`7$vrVYMbx=3@e@oC=A{AT2RmAr+f7m=$< zujNDOq`a}7{1g655t)qx*!lykxd*xz^Ts~T(?WNDcls$nRw(>aXJA(O-Z1lH1<5mY z!N?B93E^ zHt$FqgJj`qr8}Z>Vr0}(aAjiN`E$`PA8{_!virQDyN$v`+V$V zFvY6cA6Eb3a-)!8?dg08c_Q}Wd|oy6@jGgKUCNEFKF5CL2bCett)7 z#-x6hQmq4ZDHIxy^0?>pXe_VZds;k(4>N1HWT~+zfE$C_iWYz1%>nv;w+;rzQ`^}w zjyJxb^Gvz-{vLvO8rsvZEMT1Ms8B7xr}e=y?584}n;jlFkRv;~wlgEkU8aV=t%W|J za4`JIaz`z0xd{7Py^w$6!Rc!#&GK)QDz||~vpMk&iNy`m&i2)AP;MR^aFw@Mj zGGH9YTu|#7sKPllQEo9q`OHew8F&P8fl`g|UDI(4@6hlKHH3lj8o(=M{Hk5&HnccI zddj+OXm5!0lYwpMc8Dw{zqBSxs4N|Nx-}~hEpbgc-hmnzllW`kI2IRu;JIR1z5&^` zWNkyFs3onbQ>d&jTeYTjp|X@~G0waQwU(ERCj-l%c$0rg_*v3J<;6U!;qbRL@;uS|_|tHZwFBt=0s?n_;ej_^{W6*XqM4JzP%8GrTcc zZ^Gx4_nOm;3bKgTx)`G#Zf!dZd4R93u)7P?njS;vD@eDWhND1+{xfy8S(^2qCe4@& zp}EJ8g~i5oo;O1CgEW9|zccX8;F|StdDs<`t32%2}bq z9l(A`d4KvYqfEvr*oeWL6<#7Iesskx&A4J>?tOzy=gFYK`uj*YHB7sJ+PKuKGY` zanOkFs!(Iv7a@z5Ed|dyx^lozqsspv>Z4KR7iK&|L6&h5K@wR`aoHIF+ii@Y0##)Z zA2mhSvyC@VEM7YUMW4phtg0;J*QQ}6Qq61q4^p=qQXJZG+l@kuTvZgvKrVkHI#^W( zdz|gZ0@83*j_c3c)W4v!`k?`dNa^BrB095!c`yMpwx&(dR3=gui(Q2yKt9}U3!ddK z(x<4INHHd)cs;IcxcMelV+PgtgX2tz%PnYNp!Fe=xzFV}c2Rrl`Z($rGryb8$KffC z2{|n2s4<)=cdDXAx)CY6#U2KhXp&K2zm<(o>7X31tFL>AYuXSa^44Wi)okqEhF7s< zU9M%=MyU-n5^kWiaVo~j;e1qK!cEhek=t3^f`X%YtI6CIDxhKP4;Qe( zF`EzPLrj+D_ELAjN`|OUQI1>|+mf6{U#rP-@+GKQU3#=C%i944f_ly>90XtyrVX5J z>TI;bIWIhGgGSHtmhDUKAa2%Ra^`fbGO<61U=v)IKcLX3YE)xz7lI0{eIDe!nHYSd3`!p3p?i8U+7gW=@xrK`>IpH zq3T6~(aID2YH{m|?wQ(GI0G^41C2`)GY%6XYcU-l1o=%dHf#Um85yHnegJ`~X6hF4 zp`4j+Sg?s;q6miyJ+MvWqqy2KNVZ6z4fypd#mf-`oAQ}`R+Vo$P4ni5g!??m-|A6n zZCSeT-0m!*nnEO~a(w=kfy|u31L2n&?erOE zrsT{>>912<2A~&a2kiB%JXWerYwAefz)Cf+iw=IO6}Jw$NG0{8CUXJ8N*AI_4EsWNXlJb zdey!ji9|=%IFWdY0YeuEj~~X&g+p)9(k`hl!A!y6n}cf%$1G%Zcs1%(UxqpN`H9z5 zJ}!ZlTx!y~`Z6$f3+TkHkfyrf1{=;>Bx)9n$EuD9FQh9T!43zp@B!sjyU9F6LoC>^ zhoj5-n$=TO6WxxnZ%o(iq^dJeqpjb;@;*PgqTu47`2&4)}aW_t&WanLWT5! znxh$c6SjYI^Z!tjhDS-?;yrX@AqtJ@Z(>s~tzyWJ7JW8kY0=&ikTpXM>Wk$rm@GygP zQLslzMcnS{o`T<1Fjy~=eVu`Xa8=x=aDBT5tV z)0Ycib@Ey2f1FQ4Ksr0^dvm44@P~+SF~f2HFbUh#$cYUF5_->0RU6E#3PzT9X3h*V z$J@wp%*+{X=184m?jMQao%CemOXOzuV=Pm)vHXF|RUpFysLFZNs)6*cWKCkqePDqy zdIDpgs)pj1=e;j}wScasH3dYtdSz+QE!Q24QU3-Y9t$5WP$S57fq$Rw2T;@Mn}fM zz3SaUR`DC#CR0uS{EGWtX$T4Qw!VTc=|R(4$gtS#*xQ&t*Hp+{#4b3N0wJ#q%(94g zm&u&<*%o8X$dLvu+l|1RSqaBk{9^Q)_|K38ZoUcKv@U{E<0c?om1VE4xoX*|fo-1?>>NgmzlW2tcnIzO9@4BxD0h>gOg3zWrXhE8mont8 zma=0*4A%P>9%4RU*A`TSPij+aYtedB)H|lsGv+X&H#ZUaNzaqHS(+!6YPLRdcmSHJ zb=+8du9rjI823ektzFv5s=}>W6Rw??SWQTBdHg_gqxg$Ye5?uce+Z}yvN*c4% z{;^c<2U)w+k1v_|_Uj3a3mj(*saYdTDE6Ew8svq7-K%eLTJeKiFSqum8J(qzEc}zU z?1DS|qSr3-jaF>?_UC~@Bd7_JkR_pWRHBZXfa$m4M_E?-^`{3v%4+gK3G(X*)38=B zRqZHC$3{2g-Twn>si(!zhTi?}0R`Uu(^Ye%8AKS`+cYK8RU{YG96PiX7i!cG@PE{R z2s0&94Je^gTuP!lUPK-7gBf`ziqZ3q@~+&~i%xZt-X5vZOw#aye=>oNYD(~ltA7bf z>m;kX9WBnoJUrqwPRPd;gi**;O$wrBon?T}ls-@wR%(Pz)mU%4FmqJDJ_h-&chux7 z?g1`_pfYGz3ib5;D~;kWJH@p`rRc<3tWuqcqUW7uMR~h71$L3GNlVk+nq9(owUM!!7>#s)MmrY0EW&#;-|+{+x@goK zde>zehTkqZh>>d-4wI|4#=vAEWPlfzp(Oka{0&FE$2M3)NBN#&{s#L#?Q;kj%BgXi za%aoIbW*+nw5*%-DO=q{F#dtwK_c6L9%?Ek&QA52*f~IiVjYZ0j18pJZZaUqQL}}i z`{8an6q#+jdk7!w_XbJZ1W?89(l_?TAU(gJnp)Idv$mS8f1jye@vCjUrI^p?#QMzG zVTcOI%C))m)|IXfW84{x>j_pigg^;R4rAi3Wo2AvlTK~ecshU_CZ0DsWNFYTLuGI@ zQ!*VIAG_&k_h@H~_I_+LwbAugd=zrG@(jOl{TD(p(Cg%tC!y|1F(lS;}oN>}QC7+SmDB-<& z(}nW(mh)oUbm7rQ=4=f<*wB^xFj&n)>+<3A%*kD*7p9SQ7H1&bg*U2}IR)-AO-F}W ztw2JA6K-YZ&vF`SkK0k65WFbQdfJ1|T2IU46t@-4F}conrm#M;s9e+8kdIESwR|kl zTvvj8G`E*7ooP`YoB@BRMu+;qz;y3SPy4{h^4Oca`eHY!=S!9PVmE12jdzoWU?=sr zClAaNg3g#wgO^$$*_3soGX!8YE^CAn^6odZ3-eKQCm;@i$0?1_(0?|VQpn_#%nim9 zDS5Gq0zeS1zJDOS>nmG0o@i;nxF5M`KiRz0)ZW}I*GxFZuTnJV$n=3H0Kfe8?Zzxe z!vWaJ;@;2xvKYPZC-=#7?P*(o>FeQ8HB*Nif5py56}{3(^_T5S?sYdt4f~Zp_WUS< zXD3XpaAg)9x>$K2p9w$X2kJLKcF4angN=r+L-`j~==K0OZXP0;_qji!c%L&(tqo{| zb>Cb#GT;{)@p%O7ZBguzx4H6px+m8%?;Tp4(^!pjai+o}dR4$qcc-|X$m6uzI?Y&i z+03*PNJ}nh7yY-eHL~d6p2MgSjn-`0xqn_sYTx&1ls-_p1by7CE%l4r6*n}+u=#7XR_@H*XrBU=?S;R7^P#X0g zEbA0&;9{RGr?@yA_oHjL-1Ur=E~sa$sak`=w+74EvFW<7u^e6Xg%J2c&iC9vs=DID zI2aN#JH!;H9nxtAbr}i9V{xm8)Og1i; zgjwaq$RxbP5k`p)+ry244t6%CftML)_XY{oJycvVOn%MrV{5cOvK6iVSvHmlE$IEv za&(TwR!Cgal12EmantwdVJcz3rmR`h92j8xv10vght@v>T> z$CAemeNKJvz%8e$BQX>*LEg*pHX38i8%_Bq$_~=0J`I>CC*{b2W^HG!^l-fNAeT5f z(D4Hr;9+)bK)=Po9{r5(me_*Rsb;)%Bma2$qfDzqX(-0aCLH{E(Mjv2j z0lvv>Y|$a#aUFNcHAzm-@lzTquH8*<>d0U^Gf8%;upB91XGD(Zb$b+`>ERz*RF%&L zjM(3AyLp@^2gjWN)Z%Z}&XeVD(q}aA;3Z|ae;4sovpI$K_W4W7I~B@)^>ivdRr*Ip%4(c`@RIAUlE=i8ewa**5 zN#V10Qt&icT<-lqt)|JQjyFDX_o=FS4camdru4y&)|$V`hr&77lR06N$$DenQ<)iZ zd6AM_9BXt6b24xCIL*?3EcBBc5`!iEGeeH^YWwCJn1GpnY^SJ+UvfE3p2iWhNiI1K z(r+`RM~>NhMK-mi9c`H@Yk8glxice*rt7E}Mrz_a$~_AN==_dC__z8yYQn!h?`RNy z1HZgAu!(E#kH+m$A;8Fd!!lLfdP}>whV?Do;9uOQ_h}YliGF@d?z80zOFLId2WLZI zb;ZNu*|LmfL~%)Ob8vW^?>%*zBRwqpOGsM2%Tkbjn*(*jvo;-{Bb&;c@5ptoEaJE` znD>bfs?OtLdDGmjf#geDjMe_;Rnqg z-oK@hghSX*rD!$DpgeyUFwb+B7nF3JAhxbOI`Op(=aiz0%o_&0x*E@YihbcddsFSsXu<* zF2CeO;}pLc-bN6@xFaAWm6k7%1N=9>U~V?{u9rG8B^VPfnMQ{I0D!52~U#YlJf(LQY9E5rS1ek)Hpb?)_Q&0ZTjM^n% z(kj%pY$+z^E{4WCUXjmYS=MD&UMAnLFh*Rz<`SjaJ*TdVrGNh7#klL3gzK!zM5$c5 zh8t}}b(ct8ZC#Zt)gJv#cNb$9k9f*0yHSHBa%s_mHb#sGua9A?=HclKQ(vEIg2g36 zjH>~PoR-RP$D5k_gZ?Zcsm4-^2lZGgor}Ex+gzo2yJDG}Oait(sHyNlUHhBnFO?9f2Z(dNniq^#y**pV1G?!OWi(#VwZ+jz4hMi6QrWB>8W%c!X)c)tTmu9u%Qh z>KAlv-?OJ+fQ&V-F2gF^_!kveAp^=6Ge{6WAq7l7@w-60Mf1osC{mT9nb{4QXjLE4 zt8UMz*9tk>v0@SKKRjn)NvSKqjh&uRzLnC$aSkS%1z+SdDzj1syCU3^VP+IwS^NjW z)8LgdLJoLDTUN?hQTrbMXMP@MH|A&MpT_*WY79Z^%4Oi#W6`1#it9ZxtEf?}9(<$a zt6{A!TP5?#(hq3MDr_&8{-S%UWMKI(#jv_jopJb2wMS+K)8;=&j(ox`xKYe%G`F%C zwO=j$icZj_BNy@@vy6zJ?V?L1m!M^GsK z%ZV%#<3Cjgv6CNCjkRDlw@1`tt*j|q2GaVqGNf1#`{e9YObk!<(oG!_X63l{hcJ$D zcPMSG^egZH{s{K9X>pJksUWCJuaiY&?xNIUoh;pB!2_OkLkQZ{>YfJ~^OkUXB!+(m zanxTZ#B^(7W`rSQh6{I#b48}wakD_EE)X8%k(mcxR;wS7ZJqRw4PfUv>gZ|CkWuMS zF>Vzxa=5ks%;RBXK#p=6=V{K}$Z3fi^_aBKUJQo~YiqN(6_m^nhAZwHqtl~LpegHR zLD$azQ;0?ExyH0@y>#=OU6_4xCwQ!H_A(JnuJA5GkJih8+yklr#uEyN{ zWl;tRwf+MtNQ$1>jBoioT#Jb)@}lnui+1ZSZg>q<<}?x>zY>a4_T zpUgel%#4 zbn7+LFAEZA@1cFN+H*!01oVV$|A`%V`mlco4#6>ZdU-tiD&TWj-5r#SyJbit?ZjWx z#qhbn?6^s9Hep#`xJgc%rJLV%U+y+ezr!ArxJvT~?=U*nXs<8T-7HJFC!&y+xQs^z zVlLMxVYBp(nv9-!$F7NG1EFved~FxLI?||znICbCCk7!L2syxg7gvoX5q;E{uKd3A za3IAPAOY zpYW69vNMT)!spE9M0OHBze(k`$(pfKu6#?+xN{U=!{*{ykhN8Tfo{J;3(ZAm$uEw< z*mD0jzh;(vf6l)P+bc``!ddbmP_i}DpOMT`AR2BpI|A%sv*F+iCpcd$Fa{PT@*wgq z=AZxE9maPqj83F#+o5Xox=eGn%UH{v{7}1h;G|>s2Wq+l;jLpo(7+witJTO4nM2el zp2RKZ7+U8n>5Lj%O_gkmbf(X*ShMyh7L8VaPKOeMLOGT1C3?6+mMZGcdP3ruoT~n{ z3{~@|qRdU^5g0Uqs@JLPPFcgF>Plj_40ZL%jh68JNi=fO-bm$(RCl+m;<%XcP7Dc!CX=um z`_!f@bY!f~Vd&hfx%#1;B?w+yx{%p)n#D*Y>rxX9|M;rh9{gYidDb7sKXCuH+N z+0I#pCh}CZyiC7X<#5XtH1dyC1{HdImaSI9`miPA9Y=NPEQRcm;e{6H9DZ-`@qNR+ zFy$=$yhj$z+h7vQfTt`8>ju-hJ+e}EFBE=$pWYmk{?;6OWu(mUaXR+If@Q4@_sax_ z*y6`IpM#2h`DrTKHHG*ok*^wC=;o#|0G}Ix3u-(AIK9!uhB8pm zEY^bocq4SNb`0oO3V>WA2H>_v&I470fdOWnatw?I;DqYxT9St}9I2{&xVk;Bwzk3`!o6KKr9D-)>6fCWL=T&z3;?j}%-fe;Pc zP@W9bGIR1W(8kQkcJy0px~&%=?MOR!mp$lpd1En-@Lv>}Krdh5H1Fg+s4H)QS z7K>zHoLMZ40sUH5l1t%HkY3=1JlM5)4a!TnMBymMY}wxc##bI_+D++f&3jxf5=DkU zF@gW!z`p`Mt`wzK9;Y*ZU^gj$nEw7lR*L?drpv#xmrt+#-|{7~ zj_is{&sr=13^nTziyqYhsOnwsLJewGNfh*rQg@NWhZCF}KcfVE?|ABYi>+0Kbj%aaoEgCDT`+zHut@5o%F&bIM$%zgZ&I2(x z^4g#a?a0fw0gk)`{Nvg4$Jq4btmHt-M28ZJlI}{kptjs_>@DEin4enR#8Vx9+L?@} zD*W_avUS@7*+S$gj^ILdXAP3RzdQ7z|n z>(h_&x(u$7z#VTi=J(rao{36FY>CC^N-NMG+DHBLLyVo z@WNmE#Doh^!vQA0L6D-(PDLia-LG#ES%_@ogK~P;rfs*SyR}$$%WUa%WGp9Q z2{Uy9Pgqaqv|xI6jkq*ga;P2!GM)}TlO>0 zW~w^LDOGi4hdhh245s67YZE8STPY*qGjJJda(E{if&kHf>P#PM;bSQw$8DjpJ{Hg1-mnR@wRI(l zR{B_a=I_Nug0^A1M?jW8>P_ChmLM6?mumZ3s^uEBnTg<2XbnyEwfGlqZh+6hJ**ic z9)p8=AQ+I2s}Qbqr8B;kVgZf-`Gc~|^gpvr0+g6G{HJ?go=7f!mNxlwa;ENm8sv}~ z-;2ihp?3r-t@5)}&hN+a8zW=7zzFWSj9&R!%6sq2GBZ3WH#u7yzK1tbHGfNhhgbJ+ z=?TX$Pg8br*bC^)WmMco>o5Kmf8pK*V(<`KO<6HhFFMoq!j|H)d3U;7*n*djyHi?W zi=TArPJTr!{)Jw4Gj!mJ`t?lSxDvZkiz1d{ZUuT6vsylTmfrfUrx`^oZnEWiT8~0L zeSR`{NTX{yg6{$eT=V%lnO+q^PpRw3DZt_>Yj>vL0KnVVQ4D@#XRKpF;9Fi#8DVaK z|J1%N|G&kYFg3LatE%#@3i4>`nf<-i@j@m!BG4+wTyO!*;(wa{ty7k!>F)qbA(^&z zZlI+|ncLU|w7#zWG8wqE!009(hR5PJV3uXlEK8_Gpe4||8#A)W@c*5(K5Kc>ymCAN z<|=xvbyuLJWq$dt1^r)T-vS@g(f+@?b3%klToVKvg1CnYaS2I9EP~XfrKO@Sb(d-s zEjA=kX@s=ss4CId+oH5lmyJt_y1#8T)mD|3wADIWTG1-ncVqwGXJ*c3b6)*@KL7oE z&d!;cXP$ZHndf=tnK^UjRPPE|*pb3JcV#_r$R3~7l{K)%rMq&vvO3n7)D+MGA)AkJ z44ReR62n;5bEMjohQ;MRLlgk%tC#q-bExa{ZtNthHVMN7_GV9>%?o<6higyGqGH%4 zB3?P2uP<=%pL((g+nszK*o!@Dt8`zW*Lz5@crycS9EVSKd{<}Wz_f<-!0<1l31jz z)iVBd5^HArq8I-!#lOqu5&hXR+fV1PPaiF7mCe8I&zd%>lC5l&Y8;6M{q#>?=d~P= z%!FRNy#tNhoy7+^fN)h7pY34vCWrNs1l2?*X6ks^P$x;F;653~({Llb!hQzq zf7*(TYqUfCMSnoqaj+9s``9_uUpMu=eJbXx$pDuA=*?dI zvwBb8Ov#)LS zVtCFF@H?|Re|HFL5OlhSAStKD-@H~bYwWr{gngmedN1I`!&$?AA3UsNAA5AjP&Jz- zZ65WX;NJKbB?p8mk~nTgGQ5ZHCjpQSlh)sl;Y~-dutq25`>6?Z-G^ewd_HUhYi@gK zzH8nHHkgG@!3R1qnr*ucb;B+id#6iq653zmmmWvAH0|v2KF*d{TP%zQK5Ps~5}T_O zLuq)R?Sm^n$3aYb_x7%Tk7li{%{sIvD#b$@+Vt~y3Mzmeg|?-(r`Oh3wsVy|!E&tO zf3%a$5qPLMn8E`z;9)!_;DwD@{de9S!>kF9w(;Skckf;pEb;Y-61)^gD(O30lbmi5 zBbNKoBOs>{At%%QqlM?cYYVMfpHjb!as@rb@a?%;sqpX*&6Wg|QoTgL6d_JA;mDNh zmfPF#z!K@W9Bk)2NJjk(rNX9FGblzn= zYZ84C-j1OGq(Q`ReeHCaC@Sz5k%iOAkk)7V1Z)$gk!Rp^zGpmSrp0vEH#oLFz?R>P zU!IId86|yPcP6v(*3gl5Id5w30phJqgnqbJ$8-B+*3Pwl3M*xemWIL8nm!0;9sTOQ z8H294g(D70`$j+QT04VHuTs@h3lK{ZU5}0vWx8aSl*~?hB+Gmj?y^|%Qfxo}4oEI%)k*t7~^DE0(*BRpg08R*( zW_5E~zJ8m=uxJZOr`j?&9#)^m3T(ghdW0GDY0W;ENMp5Ge^KR%UL}$KL69wB5%E%ebi<_PibElSg9ehHlb#MCc~x3jXM*|>*c^<+B8%=Dzp8&FR!@{ za<`%{pOeE{+P?goug_uOO_KT&Q5yu+FuVR8qqphn`Z9;z*SfFA8NbFTAu4O_Pg)Jw zZl5Ojb*w)8Hm|-415(Uw-Wp+#(A%C0Z@bN4sql${8|r!0T8s-sf?-_3IF}e!*{&OUf>(^+5S5J?pYa)zNd$4+AFM&)%Ha< z{_d;nRojv7yw7T|qyFza^%|?&xOfotN#kvyBvzxaS-WJ2?4)>o((gQHHH)Y-tUEPR za=gpu*ckQYjjBTI1B zU*sd>V3>n9A%iE#9M4r=GePN*UHQe=*w$u2zk7tOrZe@yhdNObwNI1}bmLn%8)EB$ zJx80M&ShPs?0wyZw_3+K2mh-J(M4~c_YBxn+sdb`W3B4$Y(%e<%1S==}#HVir?wK9 zdlQQcI!WViZrn3=*r83R6URY}*~A{TIb&rLW7^9mhPUTGY+^A%Kha2;8&{=?l}8k? zhRq^6N^TR6ga)<5<{6AIOPZ28-_>jV!p9b{;f)t|qjK`;rmuIz{Hb%ha=!6X3=j?4 z@i%usbS>@p{Q}l{HOQp1govI3B8P z%$hBD**oCjpOL)UR@NryOe8_ZeN@lN) z;d$1S2X15aIzA3BSg?JimDt`3i>Z2B98&CK`wLM-?!w3@%p+g@jwf%!fIss){_Hju z(PUb4ROx3e$+QSUbK_pBYvs4LVn^DsZJ4WMwBVPwu|_p=>!O)JX4j{d@__A7jPuQS z)OOa_cC@{qveCR|vc1LBd;YrqR5M<%owcenvyS8xF+y}zRwTc;oiz$tMHyHEnSF(O zfw}Rw5riq`M13Khm2h>s!sFg$gKS&ciGd~H2?FKT&8!a76Cxy=q9O#F8t}4ru?liG z%=+36m@9n6E@B7F!Lf_H=MI(>{w~Dl!QC_7E<`-Q0T;-?-OfcAJ#e&5gI`MSXB|dv6jG_K5KYAFv+_sZ7>)WZRzr_03@WchF z6jDGiz?a*pz)SOeujlsO@STNyIdkV>+x~#v({nhjpZ!1@_5|4GcR}_ zjErl{kG>D-)jIQw6z_YU|Choa!g+Whi)?-c3JJ4CN4n4gZ@@C}fY+`s{Cm zutRXWb;kw1uMjNT{EcK;Ss`l~^m8L>X`HvZm9|nfv^8tOd+lPqZ6CDc3wL1;&Xh*w zINuzu?NHnl?qY2keek&)9yUY74BjQ@{Hg~vDtG<4i#4*c8x6{No!x9~wH4h&^mYt)Xy&NqI(* z)b|W<+~J4!u+aV;@5tI|(>c5aM<(9Yt|LI>>LN?8O7~C_%BLUNsKJs_l20!BHZ14i zds$eGVom4=CW9-#eQdGk*6%gsdFnQ}!=_c%Q{c;|$k4<1H4_Wz3`&f%M`BjO8tVjp-RP%pC z&>98L?_(Xu-mM2w2V3F^cWz8!pllmeML)5XP_|xQ*iwQLD>ColXtL>e4UhcXRdqz! zbd>dnuki)Hb(Dqj1uj}*Qia?XobOV$TB#YL=htDw3Sl%WyFFm%vYHY(g0 zS^+(>5r6ol(8lqLZ8!P)6>i1fEUfoO^yFrR^) z^d$a@Uh@-PasW#B>&N^SLUyATWc(oOzz*Uj@gQs2_*Kli&j_hbWs?)=9*@DTfhEqSP%-#^5fgcafCX*f0Av(-z4a_s!&6JCA@(w6)o)^>A( zE4cNbY1S=f<*$9n+O!B4IxR(fC4mh-ZLrP-t0!5v z{}gX=1gu+kiYFmt9BrO*1gx8ko7azkb#V^~*8PU1LdCjLgLUT`^KXx^BewCU_?yKT z6rcZv7Z<~MXnBHPD8_iZKF@)*o!l9ea|dT!-|2T-bP8i8|pcIjLob2A*SmjV?GK#sanp5A7>M^lL6&ke(N}^ z&r6Q8iJHR~9dd$IZRq?-$eCqPB{|!5+#}0`9l_!G*p4-yz;a_EXas?{CVa%-Ji&I_ zJ~+mcPoj0Ri}~!6Y`HDqDD_IiYTv`OV4bPqRoOZJIn2A9qScRLKKc}!#`a_W4k-`i z-<@J(Y!4shJx{~)N&A66cAB+lxdbg%BLOW|ZO2rXMuKS{ig6}uKdIz*hxz8yEZugk zn1`N$7#4iUd!NBKh<6_0N6*0NxL)eIbcQ`;wLN^8_xuNdL?oHQ#mA#Bm`!n} zM}z&m#i#Jpjt9w&t1oyxa^vcFFCLp?D0g`IzE4@}Iy?7LHS?dK{v>VGYj|D1e~L*@ z<0n9XZ$_Zr!21hF@nM@Wzv${%f+5-F-N$F2V-byJ(0m;j$i5Jbgo%Fo?n3_VImpu5 zLVTTug-6aS^pmBRA%WzUyg;Q?Y5mW?c(pIs7Tafc`1@b5Cz^bG9+`PD$=D5Po`c`p zE9T&h_{pzWXP*8gn^MP)FFaOaQpdae=9g?g`|@@LYpYt<1c`Up&t`hP$o18E7RYR?-se}oLEEf<@JBH^IUU<1 z!vqs(G`m(!wo7Pk8S0MRZQ}g&7VmTk(y`|jA9#t4 zW;q@YU$hGeeQxn1$Ove83q_Ul{m?Cb>k>@S_hqie-?D=^SA09a_#Mk0SN^WYw?$+O zJS=kb_isfSEwpm+YGa^UP{*>(Ndr=@SuJyN&o~tk^#E6|( zNy9Qmd!)HP#@CvDU{iSNSzCas?G;wtYCE}&$Cg5)GT-J;ma;FJods4uNAB(!Vq>)K z+M+b7!`rTu?^zdHo%!y{Jk?*)UHnHZf7=fI#4$yj;%f5~>tL#eElX7k`El5xeTEEv2r@4G7=I zx8H^X(tNe+(ruPy3;q>b@_b9F@vB_3?y|1dx}#Qr{=zEM6F23K#iSLkqI>KEd>~|% zBryyK-9Km;p)y`}E&iLmY7Lr!kFnE(kuT4=8hY59R$IT9UHi+~Z`ebb1$=+)qaC*a zuW$FO@kaLEkF*)q;3nJr5}(a-#o4q3Ym>$JfRB6ak9hAco$juGfV*EZJ($wt>n0z& zHZX0Lwf-Y-Q_EuY^#rmFt8wCRR9={Dt?wL=b9>})$C4Wk5IfP4tF?$yjEzM?r{dsR z*hYyI3X4@(qQX*y`ZCZhCPF6>Bc+K(Ir`B?j=X?tF1pK#gx`e^B^T0?BVzYNEDexO zidt~vXrH1o14ZnDKp{6DI|Amz&mc%*siVC^#q1QzES^s}QMiY&r{lYKH}#Jek+#t5 zpNpsAcOu}yp! zgK#V|Aa2knE>tbld*anQf_TztfD9xC0Yq3#WD!B90jhU5AW{qnw4Vqdh+={WH6Sii ztAT@7sYG@WW#3y&GLM}u4pP4L3WiD-qW>Y7FtmUpkSGBXA!aOi4zE(_<^ABS->}nw zmg7s2Dt6r+7h4Y`+Z=f-Fb^)QBX6v7+cA1zAi8fr7|_-=n`}v5X@qymBEM zOObf_3Ahc!GAs_zBcw6_%G5-aisOhE1(LL902Te{e!08_l`R%i2_)t7@t5$1&9}#G zHTWl&Y}K4}4yOYwlAq#Uo#9ch&t(?{Q3eGCHS#dZE zG&x6n4P_$q*?5b0t`GuznKG*?06_F6lqFNs2SCK^4B&M*)jkFB^_wK>H~0vYlTdUN z-AeDziOFL*b%FRSb0!^ffnzTZBEhe{$vI^MGQHN|m488NF^+WJOTBU@I4c6;3Wtf- zLr;Qe!T625kQ2=(DxhJseajo$QDQ8-^42krJYfZj>4D25*>+OhI~2_pOhDW*kf{e znpN*`(&HVMFp5OvbO-@pRz~K=g73MrCa7+7nI}%#BW4O-b10CD2S=u-6D?Tui`{pzE6VD zp+2=mtey{t$O>;Am5qMNkta>m^)L)jNP+FS9-P&mc17ndLOmy75WVTYkfhDMs^6XG z@m~KIP@gM{uq)VE>yH0MtZxL?wWLU4Hqcl>a;Jh;N#OTcS4L z6~BC9p!*o|g{cz|B=&JJyDIdV3v-2I%}(iu7kZpNd-Xfg<(r!K7E~zvPNe$l>nZ&O zhy^#Tc&%`5twQRhe|;ym%X!%uW#m728mxY96A#ha%>KI;gTUG6Ofnz96Zn z!K5>O3Mvi23n)eQgn$!@jzd7ii1V5lyM*4(A-t9B;jd)ai2Ja)=Sf!#o2yLja4PKV z8Svx=NYI*MN6)xU;!b>B`$@=qa`$V&S;KIn>)?pbFw3I>KNpJI5flV&eKDdU+ew}cMwu`)OqgV9A^G)CMQnl=Ypgge6G>wt zQ-ZUxwUhe17O=@xqJyYMf0@W5#abw8l~yjf`@P_m*{FvOiq_FDawk%{MyLm5p2B8E z?38!Dhaz-VS&(G(H!x}#T}Bh%avNWKAAw(Ihyf*t`hqr(UDlThwDMGs1nK}ac1Ste z14S^7wV6k`es=$q-gQ37kux}Z60#}@5-oi+5V<%zN>)Z&nl9_L)H_a z&SQ4S|ab~rGk3YQ<8DHD8=5cjAmA;jocF$+K>p2~&@ zIV5Kc5itiq4s5;JXpcxNMQ9G7*AikNhg{iyoY4TH=zSTx6G~bQQl^hl9JYxZ34H`p zVxjibT{N1+>XYVH*rvFK5(&-?zIr`iVD5+?1L++`G!{>8^-F*xJ9QZY`hd*i5P474 zxfiL@Sasw!VF7>XJ5#|~!-e?c+tTERy+Z;?skxMjKBe~Q31BWo z@)0rmbQESvSi?-hTp6AE4g&pU*$2zxM3(AQ$OP)#BmGAw)vB+4uA*1}MfFs&6!oNn zWd9gQm~p@oG?I+kOd$&=W%td6TU60~dm=N4GSPjHsvIJm_)9d?PvknYj9oVwl!R;o zqn}Wom0+Dip1rsAHE=QO0Oj-yxU`lc;F?8W4(h1l3`7hU*(F9z8XV|basC}ol9`0B ziWr=93f^7<6y>y6IfAXk^pPZ7$fzqaj1KCeQt!{if*QGl=mNdgDQTjn0heG&Hjzh# z=gr0{LjfZA25krt==d8R9frV**2YZGCwa=emg(JPI)oigBLc!gQwK z=mZ`qI!YXHgb7FzKB#^4i@&Xca9us~fp-c(*T*6ihJ1 zzk_lyNr%C8VEe5j2hbi8Y8+8zPtN(%zoU~uaG-lMogEJ65NjYETLCcdp{59&4t(+_ z_|0PS5`*^@kAxQ(83nfGY1!n7Lr22bQ1qc&EvZb*ZyhL+HdrJ=W~twc0XFIh!|!!p z#VK9N_mdv6F-0a?9hK+SnB+PyP%X&91Xj(Q5s7t|;v4dYvxznr0*Zcc7Led?yut&P&q1Dh)zZF79`WF94I`Ql=PH{pQCsiM8ClNX)sO{M3fwj zn}|B7D)~9cjff^h&-5%>nSAl&dK;Pb$|}FjlD+l@ju^<5OfN#6p)7K$XP2&c$S`F3_9)vjkFB@upC zlzR}8aTFmX!wE~KubGSiz(fh)L?R?Inod`Xa^oQkV6PBtG{Hvcy%iozTo84a3Dy3L z0OAP%zS9`Oh^^du9R(Usps|DySdrmQMWF|l_QUOQ0^so&`s+! zAryI6$e1**?yvAVsQl#ogDkF-WQ2b1_rzW^*>igQcFK~Z&9D~u0D<`Y)Hc)?V`hoHz)#rOn0o$ zkiYyd`i^Wa?88H1VS|jrJo=Qn$v89w5*UZPdjGYPE}hcfs3%AF74RC9O7~z;OlI{SSX?o&G>Z6%gv2gucOS&_C5Y69T%sh_ zsYe1Ep3ep7mvEy$2GJUjQw)F^|EBS#I5_KLVfv&;`#(rgzt8SUQFK3v3M$CK#8|hI zyb`Ljfs+5rB^NPkcK-HH^!}Pq^f9>QehBFFc+f_=w)H*P*MvZ=NN2xYiN~n#`ZhAL zc@x3^yKqmaGcqW0h25XdNWIS8g#Ob_F>vT(DC?XXcf;te40mLm`%&a`O71DB zARm(&Lmv|M_bdOYAv75I@rVXycX#NueghJM38!2ynT*LTVKZ1`U~|7E*m+k!G67^X z0vXhv4=6I9Xxd75*D%KDO@NL>a@kIQ9vvlGv&SfT&H&&1`p}W%gn#_Mo|PP4iRcjO zDfbE(G^xVwH-y!@sdqxXJ{s4<_!1jepc*9wVMx+z5OJ^KE_hW(5<8Hxyf5A4)!W$! z?ajRBw8HaVOEY)h7xW6j0vNmT*kCG0|7&2o6oU^ZQPYeLXhtkk3gXu&pDy_^ot6mQ zr$n#+>fQk3s{f8&#c=D?pV%SUg^}GSQnm1vXE(mW;my}S7I0quVSpJD_b1u3Wx}l| z{mTI~+0JNvCAz@h6p@%L5L5vV^JstR>+N6@*wU!zhX zV`5_d0I49G2G0%)%mMoL@c;%JNOJs*8^Nem)DdFB{eU@Bw$lgGlVI|s048IBiTF`T zypfiHG(3=TKxWYfoOrwtYcSK%De##RHzO}Mpk4u)4J;+r$Mj1cdqHocj?&Y?8{>eT zVp5&lm`B%HW&q}zK3e3-g9UPsrU!jwq&|P%ZxXJGMGb@sWW8XzgLzSO~|xN)$$s{ilhGSPi0=IZ{aP`=63C=mr2q z`$iQ|(87%cThYFms9rpu`dQBd2h=KGR)nA`=IKo$SDz!YjI0<)X=K&g?gl?-C`Aq6HHr(A*92(- z$b6minE5r+&q!Kp>+=mmdM3dTer6|A2&7 zZ9iK~3=&U0kK+z~;G6NS6YWW+(Wqj+nC6==z6EfI?Q83aJe<$<9h7Gq80EyLyjBR= z6<=NXTF9MLJ)u#V^o;r`0?2JY?M(_=ylBY$LiCA}`AvT1+LLhQ#5~V}n__;KoFfmtAjK@BF2#>br|io;VtrZv@B3m%gW1rFw+MT!7a<%> zA6<$N-q1Du9VAeHhYiM~S)4HsW16nk*s&6yQ1l5YxiSmxygOovA>i0j<&BEL87_70 zrw9wZMocRXzL7&NJs88`14idL!rPw__fwa zl(q0jrI-UW@>!aD6OBQ`h?L!5}dge;sB30Rvaw;+R_spSPM6A!I^fP-+_eTC}Z z{{-r4N%eOm24no-LIz~_2rRv60RIf0hH`rIXdm*Upb;AhsPqJ|Sf4|8J#k0YeiB9M zQu32@w-k4B6w2-b0fJ$ndg3W-2c{`B?5{#d6ZWki#{R z&aJpfgq6aqQa-AA;M{XEY=Dp*G2=A)SIj}hY?P+W4);Ac5eY>!j$L0)v=od6f!Bn` zcuikTkwb_K$hkNm=Q@2rAq9_^U=`^J)Trc~?`f_}dLsx$qs}E1eGOdhA@EX&DZ>cE zZuDw}3;>FzDhWkj5?~AA&fL^ni+TY_M1)gh10o4U=Lm?-q92^|i^AcQ7~tNE%&W-M zr=w?xzhAs1Ml{idCqeiFsuhLar$YNsNIDdAzlMv=Cy`qjYVm$^&Twy^)j~yc=A{Oky zpq$>1tW+vFlmjqUQoH+_>zO zFu^683~9s{pkRVAoR55&d`()*X--m3I|eb1>q#g@Oo|o$+dMG}f2KzZTrsy+5q&rH z&oKuT8`XLupIFmY;{1FS?I;AmvqJih3AWvZ`jPn;=;w<*z)Bl!bVFwcXVLBf@p)$K z8XiE00*!RoHdBDXg^b~GcM^>Klc|d z2e}`ja&m%&^P9L;wGr+aeKCFnsupdgNPk4cHnAVTV$y-GNC3a?f{(Z6gtS)Zrv{ET9Z(FZo7}2l+shfi+n5Bpy`$e>`y4Pk;bHFZk;qfBSc8 zA{4=n_Qw=U{*YBc4YXQZ1xyzUyi_O}1TOyw1<+WUCLx3iL;DJdH#As9>vP|h1)|A+ z1y5q$kQGJ|gQ5_jClw26)DqK3L<00$NJfg&H&LZ5`q4x+K>uUC$d1gQ6#pr&$o)#> zs_hQOxamK?QzNY23r%si#knq|2WJN7e3no|rm#ill@hf^&a^ z3cdbom5X&J z3Zsdk5fS}hijiE67P{n)|5$yCNQS9$@4&N1GO@3wcyLhaM8qofGR*?i$TtZN!Qh-* zl}x5!4i%7{WDJr${t49iIGVMfxy0yKPCR^a5*;-_nfpo5D#jl9v|;{QY6MhG3_PNk z#2|zPpCH0Q5m>ZsuCiWo*vgBsV;(3~R6L0O9IUqm`bDG}lg|CnK^l=yB27HHr%~V==sd(?9%T#$Cie=k@qw(uXS|4fgHLrq zcmH}DLQ=qwuo>>cTGF|5gt6nPX8HQvK(m2zbuftFR&wGAjL^&Pxi z(S6dh148hho`2=NhLL-#U^$`?t}&Imi%`19Dr66+^6K@y6;1MM5v;9QOhQGgt^uLG zZneiNeUVDtd&Yem>f#Cy&@!}rA=)AvUsqfEl`pETHF9Ow(SBuoeW(`A=Y(oed`y@Y z?)o`Yi>bofHr5)rW;D@m+jv<+tvPShM62gYjnE3LPJO?}`zfyXaIM0XhO5gL9&bHd zf1LApKgabxt~^}RaSg!L3YX=IypCV;c;CX6f$J$;y>K6byBp^?a9s0oJ%+0b zt`C#Sy_<0@#g&>=?)3J?O*pQ<`;~hy;QI41dHs-B?k&c(5!XUoPvDBhwE!mwj>Xj* zS5sUTTrF@;dlg()&y{;W!c~B4F|IMVdf*Dfb?5VP@0Ykfz?F|H`)s*)!dWN2D1e)$ zxXM2*_g=!a9~Z~pZ{y0s^)#+{ zTy+n5y?-9`dcVN63)d=KGjR>V)f!h-TP@Q7GXTXWQ1u5QxTRT zOh;(J66|uyM_7O`10j7fpa9_~2ul%0VwuX0Md(`EVHoyXr6KHsFdtzG!a{^IC`?0(5JqDYmIq-P z!n#<>EJWB2VHv_igzO6#-cx4X2#s`Rpfg^iRA;Pk~9CY(edQ*9=URo<#d>!7Sm)6pGp`piX2lI>{+F==ks}$*lkVE4~3{J;&Z=A=AP8B~& zTaN3tDQyGN%8*9+^rNs4*A1jaQUZRIcAVbT2u$^OXNx46cmeklkp_N<9}#o|R~pjd zDP>Ge`-GZ-)DadOU}0H@^f;BCigZFud!&kyPB8QXFD>-S;wMP!NeQEChGg2t1DZb- zrv=#xBKWj8tw)1B$nQk?_@(07fHI{>i{+d9YhnDWIIVw!Ls$YwABZ2}#kb-tr;&zp z+k0zux<(^<1JU+=Ao#+vCGcsFw=<4r!73*9?(ij zcS6muWh|klJyYvnvy8DeHEY_D1acn%xt1;f>jj%uf_@&!H$MV8(h@X#6_Od83)mxG zi{P#Vtwn=x!Bazs94Pc93iW*0<6R>8{Lmv>N85^2-un>{RFcZ4J)(8C)fmq=J_5py zkLNY}YMrZ;!B~DP?z`G$zaH3Vn z1Pvjv56B~$N|4vNgw#YcJ$zhU_SkvwR?3IElYU z`CE`b2x*puGCvWw^tkfdN&Ir6mSl^Z%m?<<8uIvlTA(deM4H()Oy;xtp)$)e{6Ifc zRv69i_tSdYYCOYxCm|mD3}2cA1{6HQFDGeDsuaVheH$T#)};-UCr-@=U&X zp!S&U>P&uqAn=5w@xKS6l3r=N!yv$qdX~=_gz}}&^6i7P=Nq(X@A0BR;z#TcgP?E0 zkTQluqA`CgS!+|J2t&&E8h>vP{M2p9T65c&1^jHXHp~|G0&g~0>l#q#GK=pzXo<1*rVOL>Uhfaq{%K$%MtK zK#2)~B}wL2QwT03$aqZ!U#wcZ$KY5WnO{r6+k6~N6@*0hmicukpB@Mn`YMzFCH*Bp z6@?&QaVT6>kYVbQ(~6*n6#Tt-gFuPzp5lPfrLYgEmAW`p)Nd(O37SHXW~!j9pDbW- zAYXALP{A)Z#dQ+_MJ56ZD!PE$58?AwU>ETViXfwcFHHmvQyi+Q2ng|KDKHE&<(I|D z1_+iTKh@VjXR4?$MRh@_LYSfYw7H5GC?RX9x*$b$Nn@2?sNxNbxFV>wikFxONHb)_ zX%sL6JR&Ksq4F=7SX$!am~23?DV|@!V%b2WLWM6?;ddGpWS9yRnh?IEy1-DQrKWhA zsr(-*pQ#G-O&rOW4R%_LKFLsqq_aVx(tDpFQ5-N>y4569X(kT-$D~Cmru;;a5BW2M zXpyN!B`R)cMS+PUDLx^RU660epKrpKV(QX(Do^rnnAQxF8m%+6FvDDd3H~`#g>fb= zh&B;C3-Ch!j6N+?EDljpp608-E`%svb|BFy5gG|oO)N}R zal`U;^bwRQI1+6VswqB>$^2APe3U7k;xtw8tB+-}#nFmohSZ0s3XQ?VqPn2HYVn7v zydkvFrY^IaWWX71Vo{PpV8}$YVsR5yL4iV8Q^j{Hju?YghzZ|LQ~v(S1UfAVrV3+K z++g`~r3FTXDJDa+w=zP@d=vZ>lNQDM$`gknfA5$meAq+~+)@BqEJh10rty52iGU8K z@=>b1VP_)z<0SuKD#2)Rh^dc8nOabwJfNXUO)Vz)c$1-Vn92wF5g_!>ZW0pa2S6=v zZE8`pkA91Qx|RorOcP8ElcO8w6_36jnbOh~H573<6V4U9?s;*lDyl#RO1l;=nJ8V#AiF zm@HYmseH64zRJYm%@Uu}VpuAN31NsyD9e;#G5m!wFW(QcdNHm5>_#!s{lE zl=M~OzcKvIGqoVh)WWY#9I~sp;c%3g@&irf?<@R<%>32gDOqkLoG~HDR|pIlDK)Y5 zo-(aQ7o?at@R12$nIgdG(zV9;ZwSdNCKf$w;z)*xpc0ebTTEJ$p~ecsi`Gp9J5x+T zanM9@sfl8{ni)N+2A>ke5u-)Lrua40Aj57HS2k=;OPH$A=)=Ow0G8<{4oot|3r)5@ z)->8hnd0YFdD-QV|2?J_?=;27nJO$W5jfhE|F)@oh^c{L{sawy#&0qe2=oU){@;p) z#^ABe1b?-H#j6u+^?3YDEw*+>BPrR&l6U@#{NJmI7{nbP2~e- zX-%!m`P5n3P}?Re7KOT6SxTNpm|S6LnwvS-NKCZZkbc5O`TeDKlKJb5w0>C9#xGu# zqm@Vc6@DzkM`U2hD3mv1bse{5oe}EW|EBE2W0k+O2@2 z_etOk-syQQv{t!-HYI;(K&zED3qgk&6{e~Ri}@15@ufkWe{xu$N{#26pV#b_Voj^v zQJE?H7G*|Cl2Zt@X|-S#P2~ymDl$_=X02uh>>;BaN9Tct=m#Z-g{aiBgA&*-2~JkY zc13UjuQgu_GpH!xo#t!yT8c3R3dZu@>dnJ?w=GbDBk@@*8CH2$ysmOTmaZddtY1xG zN!x|ek5!wdDlXvz7HILdNFlM+<2TB(rLSv%is7XT@I1wngYqq(xS;Bzrw#P8#!J}v z^)fUBwcv=%Hv}<%uZ$aVm~ueI6J%gX+b7URca~_(=!;G0EhhA(CiEpH^!XC~e1+bw zYV}e7MrGfXD)f?0oO7zA5M7;@pii!Go6ZQEf>QhMRS~^E7AuVt- zL5}=aO+mmTWGnK2lxgPtQL22|Txnd)`B|!b!NmubcUrb7fQ)Mo1h9mt_NVNXQfRJV zZ8J$=G?q>2XU;EF_@jS$5Pm0~6)1p^n-2o8Sca+!;-7yo0Y>@gP7lsE2q>nH0peGQ zz+@yC2-0ZNml*)@OhrK2YY)yh2rP8D7iqH@#*EQRw2swMm7t}%URk2`x7teH Date: Thu, 16 Oct 2025 17:30:19 +0800 Subject: [PATCH 88/88] update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a28083a16..fcbce0be5 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri). For usage instructions, ## 🆕 What's New -[2025/10] AutoRound now includes experimental support for a fast algorithm developed by the AutoRound team to generate mixed-bit and mixed-datatype schemes. -Refer to the documentation for accuracy [results](./docs/auto_scheme_acc.md) and [this guide](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#autoscheme) for usage instructions. +[2025/10] AutoRound team proposed a fast algorithm to generate mixed bits/datatypes schemes in minutes. Please +refer to the documentation for accuracy [results](./docs/auto_scheme_acc.md) and [this guide](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#autoscheme) for usage instructions. [2025/09] AutoRound now includes experimental support for the mxfp4 and nvfp4 dtypes. For accuracy results, see the [documentation](./docs/mxnv_acc.md) . We currently recommend exporting to the LLM-Compressor format.