From cf87e8bc2ab3cca1b36cacb8bfb267c764dde533 Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Mon, 18 Mar 2024 13:34:28 +0800 Subject: [PATCH 01/18] enable special model Signed-off-by: Zhang, Weiwei1 --- auto_round/autoround.py | 14 +++++++++----- auto_round/utils.py | 12 ++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index 3488aa67c..c7e6dd0c4 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -37,6 +37,7 @@ quant_weight, sampling_inputs, set_module, + is_special_model, ) if is_hpu_available: @@ -611,7 +612,8 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de for i in range(0, self.n_samples, bs): end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) - tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen) + special_model_flag = is_special_model(self.model) + tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, special_model_flag) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device ) @@ -721,6 +723,7 @@ def get_forward_func(self, name): def forward(_, hidden_states, *positional_args, **kwargs): dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type)) + special_model_flag = is_special_model(self.model) if name in self.inputs: data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=dim) self.inputs[name]["input_ids"] = data @@ -739,7 +742,7 @@ def forward(_, hidden_states, *positional_args, **kwargs): if key not in self.inputs[name].keys(): self.inputs[name][key] = None if kwargs[key] is not None: - if self.inputs[name][key] is not None: + if (not special_model_flag) and self.inputs[name][key] is not None: self.inputs[name][key] = torch.cat( [self.inputs[name][key], kwargs[key].to("cpu")], dim=0 ) @@ -752,7 +755,7 @@ def forward(_, hidden_states, *positional_args, **kwargs): alibi = kwargs[key] batch = kwargs["attention_mask"].shape[0] alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2]) - if self.inputs[name][key] is not None: + if (not special_model_flag) and self.inputs[name][key] is not None: self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0) else: self.inputs[name][key] = alibi.to("cpu") @@ -794,7 +797,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output) """ from torch.amp import autocast - + special_model_flag = is_special_model(self.model) batch_dim = get_batch_dim(input_others) if not self.low_gpu_mem_usage and input_ids.device != device: input_ids = move_input_to_device(input_ids, device) @@ -850,7 +853,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch total_loss = 0 for _ in range(self.gradient_accumulate_steps): current_input_ids, current_input_others = sampling_inputs( - input_ids, input_others, indices, seqlen=self.seqlen + input_ids, input_others, indices, seqlen=self.seqlen, special_model_flag=special_model_flag ) if len(input_ids.shape) == 3: if batch_dim == 0: @@ -1358,3 +1361,4 @@ def __init__( optimizer, **kwargs, ) + diff --git a/auto_round/utils.py b/auto_round/utils.py index ba5f7bc2e..1ac3ae878 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -413,8 +413,15 @@ def get_batch_dim(input_others): dim = int(len(input_others["positional_inputs"]) > 0) return dim +def is_special_model(model): + model_name = None + if hasattr(model, 'config') and hasattr(model.config, '_name_or_path'): + model_name = model.config._name_or_path + else: + logger.warn("Unable to get model name via config.") + return 'Baichuan2-13B-Chat' in model_name -def sampling_inputs(input_ids, input_others, indices, seqlen): +def sampling_inputs(input_ids, input_others, indices, seqlen, special_model_flag=False): """Samples inputs based on the given indices and sequence length. Args: @@ -440,7 +447,7 @@ def sampling_inputs(input_ids, input_others, indices, seqlen): current_input_others = {"positional_inputs": input_others["positional_inputs"]} for key in input_others.keys(): - if "attention_mask" in key or "alibi" in key: + if not special_model_flag and ("attention_mask" in key or "alibi" in key): current_input_others[key] = None if input_others[key] is not None: current_input_others[key] = input_others[key][indices, ...] @@ -604,3 +611,4 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 + From 4791954e5e0bde971fb4c9318cc0b9b8c706a7a6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 05:40:39 +0000 Subject: [PATCH 02/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/autoround.py | 8 +++++--- auto_round/utils.py | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index c7e6dd0c4..e9f79b583 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -32,12 +32,12 @@ get_scale_shape, htcore, is_hpu_available, + is_special_model, logger, move_input_to_device, quant_weight, sampling_inputs, set_module, - is_special_model, ) if is_hpu_available: @@ -613,7 +613,9 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) special_model_flag = is_special_model(self.model) - tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, special_model_flag) + tmp_input_ids, tmp_input_others = sampling_inputs( + input_ids, input_others, indices, self.seqlen, special_model_flag + ) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device ) @@ -797,6 +799,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output) """ from torch.amp import autocast + special_model_flag = is_special_model(self.model) batch_dim = get_batch_dim(input_others) if not self.low_gpu_mem_usage and input_ids.device != device: @@ -1361,4 +1364,3 @@ def __init__( optimizer, **kwargs, ) - diff --git a/auto_round/utils.py b/auto_round/utils.py index 1ac3ae878..1398431ed 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -413,13 +413,15 @@ def get_batch_dim(input_others): dim = int(len(input_others["positional_inputs"]) > 0) return dim + def is_special_model(model): model_name = None - if hasattr(model, 'config') and hasattr(model.config, '_name_or_path'): + if hasattr(model, "config") and hasattr(model.config, "_name_or_path"): model_name = model.config._name_or_path else: logger.warn("Unable to get model name via config.") - return 'Baichuan2-13B-Chat' in model_name + return "Baichuan2-13B-Chat" in model_name + def sampling_inputs(input_ids, input_others, indices, seqlen, special_model_flag=False): """Samples inputs based on the given indices and sequence length. @@ -611,4 +613,3 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 - From 4af62b97a858a9e6eef6a29ac38fe0664c711059 Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Wed, 20 Mar 2024 15:01:43 +0800 Subject: [PATCH 03/18] fix typos Signed-off-by: Zhang, Weiwei1 --- README.md | 2 +- auto_round/autoround.py | 18 ++++++++---------- auto_round/utils.py | 23 +++++++++++++---------- docs/imgs/autoround_overview.png | Bin 0 -> 209881 bytes 4 files changed, 22 insertions(+), 21 deletions(-) create mode 100644 docs/imgs/autoround_overview.png diff --git a/README.md b/README.md index a85eaff95..7ff2ad827 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ our method adopts sign gradient descent to fine-tune rounding values and minmax
-![](docs/autoround_overview.png) +![](docs/imgs/autoround_overview.png)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py index e9f79b583..e2ee0f68e 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -38,6 +38,7 @@ quant_weight, sampling_inputs, set_module, + is_special_attention_model, ) if is_hpu_available: @@ -612,10 +613,8 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de for i in range(0, self.n_samples, bs): end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) - special_model_flag = is_special_model(self.model) - tmp_input_ids, tmp_input_others = sampling_inputs( - input_ids, input_others, indices, self.seqlen, special_model_flag - ) + special_attention_flag = is_special_attention_model(self.model) + tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, special_attention_flag) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device ) @@ -725,7 +724,7 @@ def get_forward_func(self, name): def forward(_, hidden_states, *positional_args, **kwargs): dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type)) - special_model_flag = is_special_model(self.model) + special_attention_flag = is_special_attention_model(self.model) if name in self.inputs: data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=dim) self.inputs[name]["input_ids"] = data @@ -744,7 +743,7 @@ def forward(_, hidden_states, *positional_args, **kwargs): if key not in self.inputs[name].keys(): self.inputs[name][key] = None if kwargs[key] is not None: - if (not special_model_flag) and self.inputs[name][key] is not None: + if (not special_attention_flag) and self.inputs[name][key] is not None: self.inputs[name][key] = torch.cat( [self.inputs[name][key], kwargs[key].to("cpu")], dim=0 ) @@ -757,7 +756,7 @@ def forward(_, hidden_states, *positional_args, **kwargs): alibi = kwargs[key] batch = kwargs["attention_mask"].shape[0] alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2]) - if (not special_model_flag) and self.inputs[name][key] is not None: + if (not special_attention_flag) and self.inputs[name][key] is not None: self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0) else: self.inputs[name][key] = alibi.to("cpu") @@ -799,8 +798,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output) """ from torch.amp import autocast - - special_model_flag = is_special_model(self.model) + special_attention_flag = is_special_attention_model(self.model) batch_dim = get_batch_dim(input_others) if not self.low_gpu_mem_usage and input_ids.device != device: input_ids = move_input_to_device(input_ids, device) @@ -856,7 +854,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch total_loss = 0 for _ in range(self.gradient_accumulate_steps): current_input_ids, current_input_others = sampling_inputs( - input_ids, input_others, indices, seqlen=self.seqlen, special_model_flag=special_model_flag + input_ids, input_others, indices, seqlen=self.seqlen, special_attention_flag=special_attention_flag ) if len(input_ids.shape) == 3: if batch_dim == 0: diff --git a/auto_round/utils.py b/auto_round/utils.py index 1398431ed..4bd239f74 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -30,6 +30,7 @@ fh.setFormatter(fh_formatter) logger.addHandler(fh) +SPECIAL_ATTENTION_LIST = ['Baichuan2-13B-Chat'] def is_optimum_habana_available(): import importlib @@ -413,17 +414,18 @@ def get_batch_dim(input_others): dim = int(len(input_others["positional_inputs"]) > 0) return dim - -def is_special_model(model): +def is_special_attention_model(model): model_name = None - if hasattr(model, "config") and hasattr(model.config, "_name_or_path"): - model_name = model.config._name_or_path - else: - logger.warn("Unable to get model name via config.") - return "Baichuan2-13B-Chat" in model_name - + if not hasattr(model, 'config') or not hasattr(model.config, '_name_or_path'): + logger.warn("Unable to get model name via config, assumed to be a normal model.") + return True + model_name = model.config._name_or_path + for key in SPECIAL_ATTENTION_LIST: + if key in model_name: + return True + return False -def sampling_inputs(input_ids, input_others, indices, seqlen, special_model_flag=False): +def sampling_inputs(input_ids, input_others, indices, seqlen, special_attention_flag=False): """Samples inputs based on the given indices and sequence length. Args: @@ -449,7 +451,7 @@ def sampling_inputs(input_ids, input_others, indices, seqlen, special_model_flag current_input_others = {"positional_inputs": input_others["positional_inputs"]} for key in input_others.keys(): - if not special_model_flag and ("attention_mask" in key or "alibi" in key): + if not special_attention_flag and ("attention_mask" in key or "alibi" in key): current_input_others[key] = None if input_others[key] is not None: current_input_others[key] = input_others[key][indices, ...] @@ -613,3 +615,4 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 + diff --git a/docs/imgs/autoround_overview.png b/docs/imgs/autoround_overview.png new file mode 100644 index 0000000000000000000000000000000000000000..ac13df61bf1c62b6f77efe83d30a2f68bda48129 GIT binary patch literal 209881 zcmeEP2S5``69z$u1qBsBY>1**fOLXb2+e{b77(#uAORvF!O&F{f&~@ZQ?V;HtRP|+ z6nUUv#oj1lFJMElpvb?M+(Fxr&O4 z*07;=V^ma{h`{?2f(CdkT9Mrz{GV#zm?1VQNmn|(RB5u!DahU}$Un@R%j2jRQ*0&g zjLAe_L12(E#m<;aw&!uYf`kG-cngmE3)oz5E{84oj!Yy|O^IYvB5fSe(wM@eGQbOo zY(}Bc+$HC;d^mjgLL&t;xI7-qnCw6^BZ5nrjOX&%g3v(l)@dAgArZl0iVb)NzMxYj zZ^P)+-rxu`Ab`gi&+&BR20_24SeQ}h;IN6~&~YP27?W+mdmh)11O9Ia$IDLu{e&%K zg_>~%;B+#@jA#bkysfWL;15o6Adx{^W^|&Zr5Op_IGI@ekPc*q8A*J9V_DuTAy=w7 z=n|YDmXGAw?S?t}3&T8l<3eeEfnhw-aPk-z$uoy=gn?YpVdCyX8i#%y6du5l9A$Gt zxL)8+prZkxj{@P7B}W3iSUmVdlH`b|K*;6@B`1NtBO6ofjfqU~W9UDMEt|{o5wiTj z2R;itptxaYa9LCM=UiA`0wH54YnWh~GmkZcKZ+U3WIBwgeKq8o{$WFetN`DU0yc*S zF3b*-T$W@3br^Kt>~Q!9kxY{u@ev{oiSLn2gbs}5&On|8IU3AmBTo$9V30t-3*rXA z-2fl-67cz8?7^)_nqUcq(2x;7A$*#*fCoPwd;sdu8fQQU#-jQZIy#=q4)T?p%b>xx z2OS^E;rjR@Et81wH~uW-WcWa!FAI!d$?wI7q{SlXNTfT30s*+5l+>iZMo#h%v*qxh z-mcY;U=aTI-&Xa8uu28t+y1S~1qnj!T`h&ov6RvFEE>l$fa~FIN+L5PatUGag5gg0 zk{k&P3P%R9P!P;#i){v^;Gw?UAkNqT78w5&yHGI3z%kz-e;)Wm0&m%@Kwt5>&<9|b zc(wwbKr-|x-ZbbBIxgS`Ngbz%|AFIyz*6~fq`skx|AB8LHxZOri)si57?U7jI1m^7bBcwe6<9GS44@{Eri50HGCB=z)>mqTk;$-1vEY&IQ*~+R z=k*a6I0qg)Kfis*#Mc^QbqE&LX(_=<9!3e}N+Q-2EQtnpj+9_&L=3@_EMblND}r?% zKf@-7@8jgo_H`n9hD_%VnL#Om$5am)uRm zDkhmiGqZqY1}i6(1owlK3`t~I&eFcmdpjoE96%#Bu@tGT|@$xokFs4K_j!xFMFO7$+cT2oP}jL1G(7voWUGfg`No zAc5qj#o$oWzViiq=ql0!5I)oxgNjCD)E-ncnuQtKG+4kRO?p&OS6FXd^Qtu zz#!rBSb>3DFVOu`lCKH$(6Dh}`Ez-Y%8ldt1D_AL`kYYkI!55n0z*VRC}9VYL;;_{ zh!KXnfzzSUApS4``T#l={%8k5ZPk}{(3PQ2b;>)4CG_Ed@d-sTKevY*b`7ub=P>#+ zF93uVBr1qbY9T-3UP_Mw@dAgM3Mq2CGfCN(YOtGKqWD-B#I&l6E>#O zdPy{cT@Z!0uozfL#VuB%op56^P)j&!S9n=spe24~iA8JxA698hECycUSD0|*A#Gww zW^k0E@M?*LmiQGVOtwpjQfym`nZ^_Dy>daxR z)K00iVvJnm>fctU71wB^+G$48lcDS%>GWkr9qG&C!4;fIvLz8xsJPaa3%pY`RNUV1VhO^OVYFJJR8a)sO~nhT&87yhYaiU?WFDW&Cq>>Y9Fb!1&ySbBnGkAElYl1x`9 zNuv27(+1#%S8&V?sN&ze5QH$vCYhTgaknHU%j*yayUrAyTLk*DK zC7%`+11oVXOKe_9Sv*(_w8XJ2F<2p)Yo(l8SS+l>u_VzMLBQ`(7FlA&l@_=~OJp&6 z4Sr8~I06i`#IY>Ve2=NjS$8?Luw)FZ#IYnX*q#QM*py2T77HtJEJ-xm1F%x&>xSf2 zE72S*jfh{Ygzvn8vC`XHF7*0tWskoZY4=Z;wULeBFSJ- zUKL3H!gppK{1DaIEp?^-A<1+^aZaFg${tmmvVzXn8|E|)z$(y$pJv+C`v$0WWb>Jf z3JBu*gA%K3P=NlYO@X97Noz<_A8>j8pS>q3?uYpb)vZQX{3BDywNnzX()UP6I<9=@ ze36mz!!gqT33v9e%6ry@xcV-8e_ ztSkm%;x^9&cyp3OubP@klKo10d9||`n2BR)qGv)IqTaB)$`S)JaV$&pOepYWWnL9Q zPGyOOm^hXt=1iy(Sy>Fs#BII_8QKRCB9CffW&>28kyG8IVqhkYrHP&o1-18Nzk*Xv z`5z3-#IY>V^P!;j9(+*YxjZb)#IY>V{EwJ3ka>=#yjoc_X5u&NgiJ-xi7FKXgn^lO zmL{43QZC&@qb8oEiROTmXx0P+HSsJi0O{O0$O5yqo-q^UHs)96SO&(cJ* zK+4U|;#rnx4oJx{6Tex0R5SyGNsGEt^M1D~ z-_KL;q*MH)s`5$l|D+xRNeo2p4&1(YOY?eE+57noNVQRSR}rD~KcvS{34W4vz&~#0 z|LKK|SR9Z_w5XjaL1Dl?tyD!hoTL|0h*%_6t;ACfkH-xN1RIgt{9%frHx2s3N-?C1 z|G}l@Yy~`lP`o`mMe+|EmrQ6h?8d${bkIx%Ygb{XNJ_X+HY(Id(y61!G5OismqNm# zw3Jk~bRsG%7~itRBDfk{wh&ZdX(uXWxJsWJLvQKmjBk9)7RV72WsRU(S!GLCFug@j z07`^i?MIVkkGJxvw={+&YSV5DJmc+O@ulQbwsaChp=67=ca3Y4>&Df(q~0iaOI0x0 z{@*5(iDBY^i;NKi(MUM9$>hIcljT%z(J>INWsA9JM_K)3bPR-V+5SJit+L5OE{j2q+zy$4L4)-pdeW21s=c- zoZ&ovhD{LP$H|@T>qPVnna&@|4=}CbV1rmfA5PHE&LqK}3Wx0jUs0?k6nkSLG-va( z|7tCkmtr$%7;GkTOvvG}g18~ljgg616lZ;IL2`r8jW`Rq;^Gvr(?JIN*wxxaz=Bxi zbRf-1IuiaVWahzqit!({OCh7$rPEODO2qOT?NV&P9ZHob`1!rUXWN44{})dM^`Q7P zGLZ`@L7f8&DRJ=Aphbxk4I#wqp=hw%s9=glC!q#B6;wZlXE+*8(dZc4gaGoVR=YS> zH1tw92_8tZddj3|u=n4!Us5!5oieIIDH=MfNx2mbb_naCXo~4N($KCW ztub6j8UwZQ6^*P&h=!GNR-0Q7@Mo~06MZp!VK9dg(t7r{E zYAC8>hrz`)2HwyyaISh@28rBo96{A*o(6|%KvGO1fo6${ZyE;;&;*7Q6^?&3pw?&F zqhZy6BTv*gLU{k$?LsDc70m6T(NKzp5zxz?8mH0GZWkKo@QXLp|Ib0yn1e#xHKkQF z3~Ib_5DiU@D-@AH#ai#(xI{wT@y+jIorH75SdH@Q$zD>usc{7R6b#WY6s>XQpwQV* z)aq5MnNSkYuWoh6z0+kW zaL*|kqM;QHgED9UM6*CuO;f164VrAHlVu-oHMXL`uJo@tL1af%MN>44SP9L|$zIWD z=!GYWrD(F9PR6~WAr7GuD_VULl^Q_NEKxhADpcMEO}5j?KT)(sT)zJ&t|Ls^*F(`1 zlisI;s!z2R>0=;^>=liU-kO+JU6mF8c^fp@PAC6F(S9>}lUvtOQDd~fqBN|y6 zK+z~@MMF=N17D;lZZ6V{5FTaR8POhqG011K65t!RprTTlOQuA>)^6&T3% z0#yO)ELsTFZ+2k$b9qo@!Es!FP9P{~=fnvGuVV!MEI#-xbjn!n45;=Vi2_wNtcu@| z58{f2lB$Q$cXiY({C(1cWKmy*B2(Zf_Rnd2xF#Ucj7UNfK;^1fA%%tlGb)2-29FSg zkC45NQNZ*;E&c_g-QNZ?$Vma!FQtkzB99{Q_6`JPDaBh-)VJi~UoU-#Y{^v56cbF1 zMTifm}!&r6ODN={nzacC0qO<`|bq>06ZlVt7|bLdjUcSYbv#m zU!8je3WT97Ase|K=v~PrYc>N8Ll;Cg7?en^x=}Ku=_6R498lg>+Am=7xIO^x?YscK zbHKJ|HX$5g5Z8;vW5QqfbJ=Vj@B(av9B{8JPw2h4P%%9GKmk~qjWNv*oPoA7*cFGX ztRrZO1CKvgMZE-kKF2EvZU$UXtVH!2M^b~Rs(3ZnK$d6$&m}>=4!EKz+*IQf7dC}w zjH?=leHuiGaS3m9*tGo&-fG|vr@CkeCf3y~sa*rdu>DRkDE^Mu`O673l7)h%?PIL; zM^?upgHiUth*9I3Mu+tgX;r>~kxC`j21YO^C^2>Q^+%NeBogqRN+r+iE6t3X~z^d$D!vh$=FGODu2!RJ*BjFd4EswM=LF8&L6psK#B{$&E zw&P3>W{Dj!9@tF*I7L7A%YX+2ohC)q$kHWa1qTTvE~a?vikciNytRclm&Zf4)kF^c zH+~32AQ;Xc>}Np^WeqRy_q{m^I$@rQkYmzuD2(7WHps2Tkoa%}{v5zs)z^^Aet!fi zl6zXii~5iD)gl1+&!F!S`-`~$HavlsUwtjB+z!F9LE++X1M>E2^N}D1^5XJ+;9vy$ z_7{YvMA8^lb7VL`X=s*=!hlDf^Z&Be z*TdZuE2-QDR_bcxC;#ED+-w$y;SG`1HEokInL_tsa6G*+MBf;aq!4My1`#mwkr=Rq zqy&k~m`V#0IVL}XgdGMcJ7Wq5aEdQRisFb74x6OK2-^|G7bBgjFxrUadMb}uwNxBI z!pbMDkIIXVWF)5ncj$$?7PLB+&?oUrC~A9pJSpjjw|aBk?OqG_!-`{wZS|iC;Nl z5<5!c?VwPSLc}dfqM03KvaKkT#IGcgEU=o4fYSIoD3rvpBw^P^N|H*9k~o$ln%R-$ z$w>J+|1Ax_9ywLDld>?^tVx!L*12k}&TO7PUz;ZHi#f>@Vy(0WG96Y&BzvdAtFm{rPS&VGvK_vy~$pf8E|a6Cmqxc zLMK1gsWURnWh8-(v|v`FDgzFVjd3@}kCE>UR))bGs^Uu$Y$TvCl0?%prd7+|6=nx27zK{J^L_Ht@p$wUQ`C7PlE_7y&;_*#iVN-_zzNQq6+n8`e}ms71Ip|BFa zvP4!QNQL%d4^~+$ED42{IF=Vd=EfN*vcrpB++PzV@aaf z9;93vl+*@tm;P(jXN(HZ=(=pR+Nd5fNC7vfap|Jaj5-RujYqD8sxK(1AougC)1e%$ zk1r@Y$OO;H7YO}XP*rFd+-;pDtev3=QOTXval_#7@07JxfZBD9lFUU#B^w|+KT0KY zA$3jVn{=n5={;m@DTM065Akc;fM^?0Y&4{Gno0zlX<#VyFsE_A?$KHP0pM%CXCU-{ zNZ-}1my#Po%FP3JM#|KY|F3|O>dsZK(B=w5s;Y|;;T&WNxoR&GsOUThkN!KQ*SimDCCAbV0k<=4NH8B1kQP+53*U6O^YoR?w^8X_}RP9;eL zo6gqGjK!@aF_=6$vh__hnEY|QR{7LP8Y*uKzx7&?lIv1B*)UlJ@~V|!A3;UYN*bEU zGbJgFR-%y-zvawmb?|t~;`N}h634Q{#2yiS*G~c64GtVfj#IYpNY>!f7 zC5|PD=6aCyFxjQniL=!tn^HshrzC0}DXz6zuBMt~ohu7JS{)5Y=haE&%1V;%q#gsw z#zXE7+`bRAv$sRNa$oNLC`lMOc1Xl6-#K=cSg~ZS)XRo1=gJ2= zq`SdeY;&Lw6QB=ZN`_)1{%8kfH$thVl24FNB1t@j?i6FIZN)F?N+r{g87tUBBhx8n zL^zskxT>yHM(vy#takrj+Eu)>xg|Qd|Ic7km$|p&Ob=#33rOl%?NCdHP*Pjyz)Y*v>3SP$u{8{{3dADTnxiJZp*c%feHwODZC>dOQ8kxvtgU$rqgjsCZ z5*-XldbHXpA$tW2LpabyNMeJ_W(tLZQ1H=<#|jMOLIs3I z>>C-h$I1j3^d*gTqzG0hH~ zfi^rpw}qf34y?4O2I|CY@r2po! zkcqM$cw{2gjBZJ`q+1dh7M3J3qvkk(ym<?|Se;B)-JDqLgyvBKc-2U`*)9(thCACV|E^d!0nj{q{*q>3mt^5F4- z3<%i`JY;upWaQNv;rJYRuol-Cc~zG0|0B-=;H~t?gJq5kJsR}2l+mjNhO!%bifHuE z?j_>+{hZ#Gax2lbCjN5~3<>2uW}Fk!IB+}vPi+GWD^~vcKy@9<$0@bxJql6=K`Q3P z2enNiqq7&0;nMhH6$-+}Kx7F>3PJAT|ASVFnfZf3ORUX>(@9mqzSOWKA-;e#SA;Bd z!16pGcUBf0ojA%D3It%94n>?Ge1VV^;5$;lhT>wWbHv}Wxhx+c%fHqa$kA%X&v{hf zIG<2*$S}8Hfy=P5B$}Uvr^M;vv;1Ka%$eoGF%<_+(#=%+?SLmh?t@HZBT=yh5mqHO z;zkjXt*(582;m6(9nb+d{DJI|IC3TveS*1cK<|o~N%+Jdfq)mp4XAk#4uEQ$$btyS znx}aScreWgUn~GTSB)P*2gU;a0DW{kmmTCQIhTPjh0xhU;bKJ)c?yOKeB=HsIQzPt_3^J)urzR`>i+$_7xjde&fF}T~59oS|x3`y*(L5?*JflLBB zQP(zrjH*ggh-H)gG#8nSHoWL||9GaWdQJ~{6glUGFb5xcZg;$Y6(5OA*+7iB6jJ{M zKAZ>5{y;e4BX$!I=;sUgz*mDOZ!l~xHZ@*TO)LrB{By7^8rWE9AzvG4#Bnj-1t1yzpVm;+* zzvBj$B}IW`iI5D^YAH_}9yhEgF@TbaSD+-p)SQ$g#mhrsdQQqE{{>yf3Jd_LNgxLG z=7vGJmNqE<%$w{5Woe@LGdA6m4l3rs5DRLsGiiv@j_?0D^@R6%6 zj~iAivCtB~qQoG4pLxM6K923}I|ikD778Us+;uJ`GT$SWbBYjOdbvUtyw2AApxId!X=YiNVFMDHYi-YxC5$E1N9%WBtfDn zKGO!Egyd6@mZ+sNRNQJLVmG8T5{f-#&F;ymaix$INt77up0WrM3ni&|MM(-awnr6(vR{m&}=VdDTe_oW!pnG4i+?U?x*8sR1mU#IGRH91oQ8EbA~t zUUd?UlQbe;agvIe#U*oc06!5Wjzx)Pc^X11*WvphL7_lq8px+(?~w0NfqvN`7*?!=HR5>XAW(`&yMOu`=~YNDj4p=XX)j+#eL2n!$Q6 z!+A0qhry;osg*%OfgcAc+6d}3GsvD4`roQ51M+}sOB27TK~yyVCs+HMH*5wim8w9Z zM5^;jMKz{W*)RNK%c)jcP!vZiY1sUq(wJDVWK)5JiNye#$()y$SG~kQON4*H9dSg% z3oY?0N;D4$qDv(*u^4!X z+w2k=78eMp0-3KOl2;>(ftUCdCYtvF74?+HkHx@C{E8BT^&y+V@>O^*77H(ND@tT} zw-oCmW@BZZjVZ4t7LAwq%`u@XIq!qP$KqI+Xx>M;G!u=OI2II5)8z|u_)2J zkFrOY_|5;KqnRJEkyV<>Xmo^$V_~AXpN81kO-zuH3a+Emw0DNG@IN3o+{0G<-aACDd-=^~ti6&gSC_ zg#Ij^IE_z|>L>LWe^5IQ?%Q8KVclItgfjj9=sBexrLJcFpGnw=CI6^Ii`t116b2lc zNX@3>B$JSirXf)aPdPjuHy{w&rTI_ysH1syw`?J zMWdn7p9)r|!tSk{I9(n+GoF*(|!7i%$Ut?XfC5iOmFO8>>RXt7Aj1{ZBzx{1mS zuF__SDQQ@(rK2;uaV=Ub%2HWHOIIwdMH85ffVJ|ewKRrh?aIki3mjuD7NM!ET1#Rm zlxVT&%?1~3-I!XJJ~s;1QWZ0zO1I`Ud#}THg-Z z>_sRD=Ce6$nCaz&af95T_kf#96=vT|y>8k^wzM2Ak20EADicJZraz!X8&s7As!bf9{sn9NaT zErYGlueD4(B55je=z~8ol;%5)AFeua?O;b3*0|p?+sMmJGG5{*%Vy84J9}WmwgW&?G?v zV3`G~^qC^1W?0HLHKT5P`CY{N+gb*D)DkoLlT~OI3XyP*RamMqw2X#cPNGm+Cfn2u z#nZBSr)DUKmenT_N%mMqvqbH0sz|9BmaoJ zrpR8)=;&R9Y1JiFai5w&BFZ*5<0oqNn;{!+Tt>wR%rGeR#&8+wBs7>&6=&9%M3ilE zhT^GNy_W(j#v(Q`scL8wrq1IwSwgDL}#w?klZv$C%{W+w#BJwB#Z|^`*L{eO&tiB}^|90s@5)t;U>RD9{Q(qCf11cB_ zy{{_oS!YpDHY?CqTvZhMiC3^Nr0PR`RS@s9$zZce)F`zin!zDAGNfd$9W(|iXM>8Y zbwXPGW8DDtWLe?BJ|haE{RF~LmXM8H4|JF0 zk~NzFhatcQXThxl&$C7mM^M>!1j~~HZ7nF}6|i_*9{~1tUSJ}hBLv?;I~;MnSUe{D zg+G_g=0Wv~g&eSbAIlSZFD^3-4?j@Am1bj1vjb=Fpv&8^y!?E`1&)!u&*1vUlCgxb zfs&Vi&*yjr!IuW7!9=ul>0!(Yv9Ot~-zbx+WLs6a7iQQWaxP z4P$XX6&{<&Sd=|#Vid5Z(P4E&T9t3qq*95sQ4>rPDg|m{rM}ca&FTWe{}u4Y0!SJn zWU)4jRbDATIfcuWl`{nZdlnUG8*`Hoaw4G?6&}j|i&9Q=m1d-M@&xWWO~6iG-D_@gE1FMIBq2+wcTl ze)YAxaytaa28D|w704B=O=N=b$&1VPf#aBZ4A?qD7a0yv8k%6EFfaxve5l3%g|Qg% zndD)B)+5V7RxyA?I=`-R^c%)QuBcx+vL_FW<}@m)_BIr<_t8lVI3z_HB~xoRD(5ab z1rA5y2LSm+zvG`H+!kv09}-e+1?eD0l~jHW4U(%O*9Pa~h&Lm_14D{8LmYPmJ^q;b zGBLKbH?{$`1nfmb6u&1+hXgfwFbIvSgAOue5EPB@BK@Hk1o*DM8xoLfvTd-pg#Z^W z0|LaJNa?|Ne=bD%_Nf}Q5{uKggHzh;fxPSnWM}_yRFa8QGrA?&l5R<4SXh$CjGBXl z3KkJjh!i53NTSi{7DQwk5*anp?oNF$9x=BlQ*H|TQLrT^$>r!2M-xJU07QPMee;lc zPa!M7ccg#~as5)~h`(iXSw2FRKl}y!0}#dmRv$XmdrZo36%|93VRp=MVN*&rHJjDW z{?iwaU>ln=(SEviql0vFxAa|PlCvn;C?+iO*rV1Q?>0h0OiZMq=uB6g(ZbvjZcPPe z)dV5TImHoC7KV!nqL?NQ?N(YR6mPk+&Y;igH!t73+`O*G*ov?3x}6_)XWtB$f)9zG zU!5{CG~c6I<1cE}O_gEir(AS$)EZjjFH)Up*=&b~^xv3qYHifV1{HCR?W}RU2jPOH zAx-*k`8I>)hHJN}nWvi8IBv*vzEZ3GcDEwcGum|yyXRlyzEqougsp1c(T?33>-!Kw z)FZQ1ij4S^25Cn6sT569S=&4IPDhmhHI(%R|*h>Z*^P_k9g1`b-(C;8WAc(0>Q7Q9Bc+vis4;LNyg9^+>%9*W7Xv=%(Pe zcMhMkJ$i1vMvBVRZYt~+XLHpey;LGi|GAg_J@r<}scDH#Q&!!KylH*ty7gEM=C~e} zo^29m-e6obJpH8bDRtS@FMU*q0X)}eqmFrO>eV*Sx(@2K@A&iRw{6EB{__29XBCaE zVOx%y92%o%mcO}4lOL4o~0JV_1`q-kzT33iiwtms8aasd8xC?ahJR8S?WBaC+{n~QjSg6&T2pE`^H<} zHs}uQ<{TfJ`)}oT-Mzgd4bvmp6-26_urz$SrD?0UE-1)u`SD$s?_@8JjV1QwQCIW`N1HuO zj#$VaIc37A8#f|ve9eAF6s*3nVQ|a+0gE%wK0FZ<7(3xmR`}&-KASVv(63ycouAJ! zx0=p8o0)XWoD$OdP5M*Q_aiwU-8}BKSj&8$XXe&v!|<+`?`N3XWF^uZmdAW9`WC=F zE*z*wxlL=W(nZgp=$TRf!o~WZeq;}P>XdkoFN&X(wf6F~nOi=5*mx>0w&l>0fd`J= zy0rOb`StG~dNGOZwVHo^q1EY7dByqH;ViY+O(UQG8`a%ybr%9zXWQAq}aEJMb_f*(jFe zoM|4^@+5g?@As}5yE6IiL7lhlYp>`2g>tCVd-L(UUZ)vzS`D@!toxESHRAXXt-+a1 z^?oed60TB|^v4sk0onzKc0zhIg#7(b;on*YvyuF&8Mcd?$&E{dx1mT;Ei)m zJUsZc%;Zg?_XmyVIvc#^v`BAZHNCLSF}-W=CuPOF{nGQ($%VWH!=Kv6PW0NQDJqGG zRo(SHKeVHIxMdsl$@-&zIL1z(5Vh40Y|3BLzE8(=m8=TW0R;X-*EHYohg+APy7X?& z(cEF`%-K5^wH+o{wxq@6I2UR+Eykuf4u)TPtEguh#(1QJSTB$0;H5rLzzolFLr!%wD2c~t^E1o!Bv&l-8soR_?-1KHI?B1hj^I(nTUqbh{ z_DI{VVyLNdrJ0ZFnTWKsrmdOFC)k}m72T5OR-~q8)l<+xB|o>R_PY<s&wYz$U6X*=t@W-d4*9GvBtUpXa*?Q<-xapZcY$cMR+jd$8-> zw_ce~Jtt(nD?fX3(8Fh^CS_#}@SmJIy0v;);d9>(x7C=_{gxGeK2`Yd+V^VdXP!N9 z&cAAt~a&ex0>O-~9nxETvy=l>@NB*370XY+gW<`k7UXIUan72Ai z5Ph*axpqs-iJk%R@BisN?Gbt4h7miran4@(PHDpY-sVmF!hMToHhJ$k^UW9YiPwyB z-bT^xcaM2(@0=Or_+{sn^s6P2hJE{vio@K3 zfBUIL6GUmpjHfL2y>@F_s7_{kQ-Y}E;+o5}lHAG`#jz9SPpEj5f8=Bg>#c@{NT}n* z8q;$(Vdk^Zoo4Elz7@_MZ6Y!(%uL_4yzjek)(7>6D{h&aSx-3i=tptS)SU?@A`M3s zd}+EUZg8Bz{H|`>O&CXq5ZBu;A2m4h*`6+h(j?E9J)N$$-NYt2J@cSyXIW0-n+Q$h^lFi8T{t&=bZ)P`J#CT!<>ng0W6<(U*?%lp35oO zaJ+TQ+rE|8op#RKaQuks!%hWdX}ap&bdIGLo!ync$XjRO9i#g%jpiSl@;L3`^U~t} z9?u*MeS>wn59-TfW}CY|G3se)9&pV)+kIQu$-Q2+?vW;7Qe#&>Sg=GH|%456P+{&7wlvG4on%M`*MQ%S9?tZ`! zb`K9+d0XY%^}|-s4K8R$YO;2j!R@h2K~pCGtnj#&V!7|L#V7l7yF4D6@=Ln3?7?k2 z#qtBc?NOiHy)I`jJ<2H+Z8qM)^IUP0cYwGuJ5_UCS?=o>S9%VdFz;Lk`_mtm@76Qr z`Yel|xY&2s;5|F6gd(b8$lGBLn)P=IxK= z+s(ZF&n{yXUnj%0KI*Zn{!_0mqPBuqK`@9Z&Y z<0ob58(UlSe?++r3A*LCl&5;6S$X&J+joC-FgHYxq`&dYP)7eklk{zqvU!`+@8$HfyLe?uci?c$f_M~n0NxeViI&Byeq2o1V{HzJHt>#YYd@W@Gap<9c20468 zPM&#ooc#=+2#XH-s|@vvsVhGZ)1tQ4(m%0miMzSOecHM;F0rrZc|IMyj+7o{OgnSl zC}C$5kaX7cZRhtE(;yd69d`Ai5_YSmF4n)+|3;7QRDTn9Gf$q0WBVnOft%@dXH(kP zY1UGI@e};4MNH7hIDVQgGg?rdPWDh(-?rz4+nEC83ULYbMRQc7My9 zinh5^=UE*X^le4oE8qGis6^^~KK(I$;+!9yb5mB&5k$?NFFfR{$+66D%!#10XhMOyQ2ws7kBsbpd^mmo^pklU8e%hEr6?^3xj z;MTqME!?K`PEg%D#d^k)6}dwfX{DAVL0;)8jhu4b%G8A7g=;Dj{ZskPZk?asda>=0 zkNR!98c5gGV*@u0skX&qV@K^Q%}h9$UfRS&B^GDqDOL?-bB-S-?aBAQ?qZ3QIU2B^*@8U|swUMfEhHF)f zt+cmlip;-<4VX0OML*3W_tVRsSDtRZz<+~k_U8|I>6YiLe&m<+{+@EW-O@L0V-L>1 z%l|el(>;tc)>pA+){LX$; zFJ3vaHEP$SEpaC6(!Zb9JzZEn=;Pp*d+iq9e9~k40TaD%-!jgPHzxE9Iy}~Pt$K!f z>>==CaxaQ-jTYOU*-X@`-L!|d6Vyefz1{3Y@$=3yKJini}T=szxDUY;?*-FZw)@abGUb;Gft+K2ZLC#0Q+hH#VlL&3|c66d+<%$Eis zjCt#`Ubj(yJh&xzIpEd1+d(LHQi*UBUomER(!-kt-FJ`N(7t^?qtAr?VIAC#yNKeO z8lN}1JRB%WTjC_}63^B&VuO3ihyyz6uHHF~(66u$^Xj&3?1SFh!6N|r4<{NE`rp|) zL5my_yKerI=Le^qX-`OcklZ1u1FgfL%F>BBr#Xy%8?97RQpmxa^lur@0}A7>-S^Sc z8lcx@-*LVEAq?u6+kHA+UavRz<)Mp*zZFgxts0SQSun2f<*rRCqXrJpo6%`bP;`oG zo8fQoS9HuLmYaonJ^i*~W>>8%qK-*jt`FtAxM;<0NmIF?H+WZs_qZ$5Ji=9Xr|%qf z+H<$p!A&zBcWXO*;+%f$F$jK8F_CLxhgc*A_9xR)b1uMIQC%3>}OGJtd^GY zH!-@lzr1C4M%N1y%d_?^DZ0^|;5nJ{EY9{?#Y%(7eVauG{qv4i?k_9O-8)UoaM#FE zs>r2pgl;$OZubdCzaF%%IP@Sj#B#*ST(f4P+g%rjMNJJ3*1fcDkg8dFsl$u#kS)8l zgnhSu+&_=W`1B$vAoJb1#d-%-o!Sf^?{eqNmsCCLuh+}xX-}c_dUodUd(CI@qZbs6 z=;}2jo4hqBQ|I2{_cPZPZBQvrpVE5#gCN?7^_I=tkMWNmKai9Z6@KG+#tp~K{D8n^ zV`rRcQrYCi?0nMEmHiI&H$NrR-#9(wV>CH=iSdWUtXKJat|nDP4Y+sk`20Jw4r>h8 zi_3^G{$OxZH{3^9Ui2+HG~evw$(>%{2_8I8*)dS-g(!YAZTQ?o-}Zgg55A@Q`>7av z9@>9KN7Q}_!J=h|!xXEVSNr6!WBfUDjx`Z^$1z;>+@F*RwrEm%`dFk4aDTaPMW>Pv zDNpxJy&4%fHTUftyCWw{g&(I*6vZD|&}~W*OT%Q8+560u7b{n4Yugbv4oYv?#dN)H zsCIr?v0(XNPM1r^2X>s%>CV-1enR*$>#Ipi?X_>-U$<>^X3KL|N($QkAbqsa`tp4~dS)^egu)Wh)Takl2Ml@BBk|M> z!<<1w81|j@Rek$K1lwk}Q6IT$XFu)j8vUXT1)FRaZ~S7l(%gMt?0t@g$idaTd*2-$ z+C6;kYJIhNmgi)LiaW&zrg$u|)~@8U?U&Hc0Ti3nMCHt|h#jI=J*9Cmex4)Vc zkv+8gySH8ucQPJa8K8Nz;@!X}OVax$8OAt=*cMK{`r~T!5&9${EB$zMQTdV$taBAl zovA);f_QVL<}FygB3Vtuf5mWNYMZhyZi(0vb+$Fr=gLh>BIi@P%X@c?cM$X|+ds|q zVN6uPOP|fH@@C)F@H+QgUsQDQ;*|Nr!b5E`I)61&wtc34*bCHaS1KAEw3wh?>TrtL2auXJHjE^hB~w}Tcj z&zw<&%o)p#M9bFYwaZ?0%&0WZcAw#KuWzmVr?HP- zxw${^nsq?I_~N3q=^+b_c+A>feC3Fjj-5)xn}eKyH(w))9;Phtk1AR*r0cyi!>4Jg z7u-Ck&OCH%5o9Iotdrdb9v;11eP6q+&CRV0zf?SXy<*^u?^F7>pX7PVC?{2P*?Rer z@%h_xe(*I!kEU2{KNWh(DCXN2{qG&lXQk{v;NAb38f(AE(69Rio#@aW1oxT!HXfh9 zWWUP@;nM@DZJLNibjdrmhM+yGx2W6G57<`PaomCN&H@n=@Q>WdWWeE`!?btO- z$2L>HxA|D;gIUL}ITsZUNGlP=AGcWUlD~?n%__+Z9dvW*l}oP{Zg~*)a%kd2{Q-ej znf_Z}GyS!g#1;MH41Fsi%pC9Eh~#Luo1AgY+Tn2i%M&Xu_Y>XKI^D%8!^q*PueZ+4 zv71H@E-PoV4<0@=WciVdOGf&a-Os;T4t~r6sxm9_d@|utVVYk`ugvEUX6>Ur-K5ty zUT~oN@bLF31c#O9KSpu*r~C%>I~lr-`!1#f;hbakb=oY$K6jJy$nHB1bRif~+ZT>+ z7u9Ew(K^*tdF`2_pB?Z0*zmZCN~STxTDnK zm{*<3^{=@n`1C&1M07A=S;n{0o|pVqIbLn4es79ZT$6LYZ#R9K*YQfX9MSk#qtyp_ z1FW5M3(Q)bQfKD|z=s!ad+JFIUek2X}>5_h(rn_Ww@hg*^ors+Hx zd+gn7+HkGRH6)RXRTAr(ZiyYFxTkVCy}ubGZych%k3QTa(I@e7+u_AcC%LIIy>)Lo znwUC#E-j_@UFcNGrD$h;%d)#n+4aosdQlAd?A+@P%?>1fTHSRJ;1`O&WVKy-tsu9z z##4Z)pRBXGyUiWO-viny&6)l7{#W9i4}_E4;MJouHT#J|Ze-g}XV|u$~EuFG4?nsLvPEQ|J zglwB%usr+KPzR3*a|Yl2^2#T))s%J>ChaY(`W6pwCUyad5{GS349zWT>s^gwj@m(E zHpjvDJ55rtIeZP%p>MM8k!3D`` zJN)i`%`|3SJ<;m&G~YLdYd39*NZY&DN6p&8W~j!_pkv!i9t6F-{`7JOiuZ2sie*nf zuye9P=oTjcCa#-If4joacPy)!nc3vbJzAnY0k;Fvy7q1+dVf10VpULJtihG@=R4dw z8C-0oT9osqAaz0i&^P%#2M!z08jm`F(HqCp@nl*v|+m+{+ zFMQ2U5*iaul8Q$F3jhTc?Yc$VkFkJ!_RIyEqek67sByGpQb&WZ3uW9ZeurG6V-1cb z>|(FB+2`i!db{nW{=xnE?}{>BJN19rqadN=-XYDxnQsS%JzipA((TEo($Z2*PyawN zo7`gRwr{TzCN&csyu#@C8T8rK{XKM78?HTRXfiqTO4t6)L|1}+U2OwX2<5X*7k22- zq3`^b9fU(Ge^gvPnGo^gqIvq6%+`Y^o$$Mpw5xfG7J>Z=vUG>fAJOhXnd6!a8S9=i~m`-?(3}>o$mcVzPfesu*WP_7ng@y7j)U1pTfBqJv+U! z%RY-CF808 zxnASm9KD~O(ZZ!g+dhZ7oK}hKF{R(|0mp+4E?Lfet@_ZknYzcP*s;6+d8fM5I;uyQ z^~!FCX20rr`T#G*d&jY)lYu$g=;yCvQT*x&&yr)IU@P{+>ph)lX{;fH&6SJ()z#aq>tkxMfA-~* zT_5JXog?zo_&j3wr10ATvu>K2ir!!A>){ueOz`&cY0bNnR1kP$*Qmi+3mpc1DN+l2 zx_W%!%%UjG^7Xe?f_4((dVqHB^$UC$*i2+*LR#vwBgM~6RMcvDhS8O-z;BlbQzc{i z<8!BoVeg7AmX{U#-0plMcbC(qlc8@5Pn}YWI&=sMa2Xenw1ZQGCbTICZAQVf|A}pQ(sWoH#La*T@aGpDnlT)IptQKcM2{4VSm+ zl|QKI57G_Sd;im|ZQhN4o&Z&>{Ln%i<1H}iK^T4~IkWf89{e>vTG?~-|4}`4cV9v) z(b|(wc(zPqS@w;$ySFcOpZ)Fi zu~hPBI`3}eOZK3McNv;mZAVyJ_TO z!xvBezQCx%)x1vqMCT*7vvWXv_J|VKl5zk0*|t~rO!V;0OWLq_An8D#4iRTv{oOwd zeOfp$?*PO9?jh4hj#nn#IJg?AW4T(P`M?cq^~xuBI={K>YRk)r3Rd zou4>t+A^!O*?`LL9}m<#4!VIrx}jeGCAS$tx(r-)(--m?rb`i8d9lY z5qr8tq1)HlH&YItxt7{>Rx8IXvzoTQvPosagmouNyUr}99lu5y`K{zyQNQC4Uv*m{ z4CyB-w&2b8a~L^4H~oA26hQ~R$=H$uQzlqNB<|fC^?s$^nH;|b2Cvo%@}vXe?h!*5 z$KTsmcIG+Rtw%6KE4a^biXA>&J1EKakimL2LeK829m(uRZby2Yh_lvKZOMFhaM4Vb zICOn;_u*BIz%|Sf>4SWHRpx(wI&$r<7JWjB2=k^+ey(NES@(kWawM=lvbf@s;rTv< z(hB`Syhsi2{i9|rIQt_73R+txj4ZWV#EYWl4Af%B6up9y>ttmoJN+@RdG6I
pw?&Vn}^wz^b4DXp~5J@n1)b@a-)*H@>%5j8u} z{o!ki@xJ7wQ|C3udYeVwPCPWCtY_Qd7UM<-2Wws0FsQYeIFS9+VRYFs?&HtSCUe`Cc_-TW1;Ess6*9evnC5Ye@KOVH|< z$JY#W)|mHMm;1EK9e&EsqrZ~bCb zNqw2IV#V3kPt7NuZmyxl(ERlM$AT+&d$&e%0k#IWN{kYuAM06aSs2RuoTRfaz6;^i z`6p3rx?Nvn{7>0Pp4yLs<^*!z2bQOt=DoEL+zCqb34XeDV^r5qfoHy+GFZLF(6_uI zUN>fe-`c2a6=xmWXoym0S7`BbQXSROPoHwq{PZoSulf9r^SdU;T79|Il@tTbf}ZK&=yKTY+-7y94AKpd`h&b-6)g$aHGi}%D1D8C`}4lBI>vU`;^k{>Yq)+@Xs>u(co{{Dq=+dHLVc3boGd zz;FQWJ2EB>cX0`Ov2{V8W4wE%@ekgnXTQ1(thc{kZ=q?lVY<)A!-p3}pZ(O65<7!AHzS9@^wJ2uf}_{s@A!&_9NPA1;3XkrdPxoxgOZmX z53*JZJ^U$U&FIXy!tQ(rt?mTZH-$|%rFOY>M#$OOEo4&W$foK``UPF`^tt&qp^0dZ z0WA#*+rTy_cANhGvv2Os(Rup;pYZA~L4DbP@}Y^BOQx9_>TB$%;dLdZ{U}=FF*_qI zYISPJ>lu5KlZn~aM^5UtC3j80y7RC06zgPd@S3&tj8PR#0g02rc`8Iyw%ui=# zzCEQWNPDkgJU4&Jlqo+VSAX6)WgR@9Vr{$1dmOgry}Uc!z|Cf7K`!_&@$<+oez7i+e}I1It-MI*mRO zTzq?dSWe663DKv$ckDXmLKt>2=fd9mDI7a3n9)0kgY65B5=w)Is0{(^sYmaFk%n*9bu3P8bvvSB zUeB0iaZ5A_rKc7?GdS8j>b}ughvb;TE*W8|^Opp5YN75qXDn&j+GT5X)hL(tyL{j1 zIN#oRmgsBb(&$!>0(Ft6NA}1Lnys&dYkN-D+RmFjr<>p6E(E~Su3|ko&38&-ow6`@ zFCOPIaD(%VY4f+`tFCHexWhoDP{n1kQ}4+xlSUuBS3WT>t;foBssVjf&y0M~t@%5n zZfy4z57Mus&A&T5_Wg{&c9D~8UM{pI?AiH|wLb0>dyU!T!i^-a`34gc&xIDe`mkx& z-dEO!>b8Z;&T2FEnpl0_^Xk;1ZL_?3-f2T{AK7*LbAxjget>AFE%IHmz@8ZQku+nu z4=ti)^b`NQIZM~)T^*@y*ZKK_L9M+0WYbiZLqtiCJ$)oS0n6<{hK znMN)ov_IIf_odI{H5OB6^<3YbkP>sh;^myR?&=*bk;W%)nyS;?aPb2Bx=?;Y+k-gHXZmry^!^v7c?EiEVC zu-%$youg&VC25yZooCQg-L~%Cn|^$o=gzSmN`A~*_T1Zf)ui}#QGth=TwU5Ee&4t0 zE-o|Ij2rj;j@AS({)wGKX3Qv88UE5*#abob*2Leo**&M;74E7pob?Hl*e-F#+Uzsi z*EqW${`dX)rSAuAe{v(FC??n;zx!^VO|hXi+$ql%_Ug1fHU0gzsOQrx^(M^Fvx>NN z=-tBp&ps0trF@BkI)xNK0UXST`S#0CEKLm=)NilVU4QKn>~ejL%~rF^uccT4`ssb( z)m6-^CRZQ7yRzWkh3@)(<@#xY+fQb8eA*_cqx#u)mfNg0E}7!kgjq6nb2ep#Hu>;K z6KYBB$B!gyZY%B5w$6**JiMYpR56KYz14a2s@03X{|McG>RQLRKJRAe6?rsQi)uFF z*&rLUOX@iZtIAEzytR%=zx#GhkGS$|k6;&9|8Jj8th%0d@^C^iq1)N`y)z@1kGI)- zzwN_i=S`cd@3+4~&;pFWh?yyKD$h{u_beDZ{E$hvw`x1rS?4@xwzJYfjjnn(_u`TJ zT`wB?9E4}?+g=QFhrucr~+ z^~u+sYv#BD~-bHg-(U6=8tFEuM8~sCTZ+UlZQQVmg zs`0B#hIe@N@6e=moxV>J#p{|{_Uq2Py0yvUSu+Pr$~;KD=X5imUs-X%vtg z&J1~aVK*z);E-!!(&|g@iPs$OsWL5|9PIzkwehP8+A!bk-IsKMkv2KCbu;T>EJN+7 z{T=R*5@z>MIpq>b-g)%sxM7ZC#~p6dq-A_r9LYY~#>RE;%a{9x4ED8Y#K4 z?7~;?H+InY_nJykuf4C#_wRP=e%tcgWktWUqT zll}D13^H2v$$qhcdAgz1w;&gnY;BdU?&;c*i@yIbKR6;OJFtzazID5%lXD0$54yE$ zW8)fp_teb)(e)M%QGWf__W%P7B?HnqfGFJvN)9j}A>ECHbSd3KgMxHPccXNJNQ0Dw zbcaZHKbODzzRx+&bKdtKfVnpN`p({Keb!=9iCX9${Q~27N?g;8FU3?Oqf0Vk6-{nT zuHh+e5a-6VHz`!^9hSIQkXTnLy^GxBd>%TT~Kc5yB&kOxenUInOgK7%{&>13>$ck@{D;G8e%$d>FxgnEtd7 zIx%r3zU-=;RfaE$yeYWfAarre;-s*3d{nOH(F4w&qIqe%Q+GvG2=H3Aii34EpD(TL zD52Qn=-oj;Ww_1WT09-yU)Q8RT5Ann;~X-gGqFAb8&0K}L%v%$4_g%e&xh+}$+JlY z#GHbs5bFsR#0Z7!NlYwkyWsiRu620D3I=TJ?9F?-q#W_(Cd=8OldB%4_)X(am?+{f zZv{3#)`F7kQ$18@Ui|qKQVF6HOwgZP5P~?^l!-~%I=T$YF~&|_((tdx>cA4W@+)E5 zfMT1}I0dQ;pc@{r$o0u1d>_6+mMz|9e%Lp}=2C_yJ%niSEV zjJC2wz`)Q@`;ug{11r*5ZdvqGky~PWgt3)u#WJ$niUb)O_kXdZ8!9VdF`bffaxk{H z-&d*f6KgWA6$!zn^DBY`lX*+L63u>(KETGrBp5XH)A%|`81;}Ds;atuQE6-k-_=Xa zxn#=!MYODwQ?h{|CO+8oYJKNF-oCFEKmXy{+xLvasD;U(nDDD=ak+uc<%r+?x8JKb zy31!7N+^e9d$6rw`3j188k)hONSZ}U0W z*~43Bb7{CxM(g6L$_K9PsuEGfk&3?hDwC^-&gDYsT+c!FdaC*7A_iy@uqSv-i0Pwg zapsKN!o3qUW&<56h_{`(OhaYnGW|_n5HIal7S>4o3G2z<(gGAL>3f{o?UvUTQhZw zC(eJ1h)DYULln?fOW-&c@{s_&!PppvOCmcgg0v(6w6NQixPJ*zdDswZuI(cQx*&aJ zB8a|a0qOm_eWDm0@trfUF+)Kn^CNJ^)*$hvccA~cKDOgiX%K6<9%-qD#uZbwoJOUU zWZ0%e$Tc5-D5%3P0E!FUQPgnp6U?OD$7(U8TQ8Mp_bVYOLyq-;v`nta6UL8YjUCzJ z(!5TFWnwhTonQHM3`RJ13_f&^PXi|&s~jdu2lssLyXpJ+cJlEP5-Xwp zcB;OKgG?q9BFl`NCNi^Uf3s;>SF41_CSq9^_4H8xh?2EYB*(FQeS(`Dl%^Pp`Abpd`?DM<*Cqz%I{-<_)M$YO&T#Q_Q)pRY}8qS9J!XWmqE|o?NvKhZt%Qj zv=$_nP2^4=z%=s4*D`c(GWTgz;UX_~u3GThzb10^J%PR#vM-y$3oJ4Tf_rE49(HZ5 zF@v-drO3WN|Ezt1IDW%-HyAW-KlEeUS21~xos3+{ZA2D|RhVZiVZfVB6whPPRv%k|)y10+73fYNka1ri$?`t+WXACl<+ATD0H;=_CSk(fX}@ z;|zoCV?&klBOp}pvjML08Vvs8vMSu&{jx3Xf!L3oOs?bkP(^i>2{e!u78ZNlx+sC_ zXlVj-jLTTB;;+JCd;*R)j95aHgEaThE;A30?2=hFO^LG=G_**x)Nh}mI*Dc~;Vi@Q zB8seaOt>1pzq0UkJ^v`aljj2x;U1QU`+Tz;{-#~;cW9!aA{TjgKS_{<`z(H{O^F+X z&}c^pI8)GN205C@6c=(dtnYF`ts{k7+ajD7XpFFs_BQH;ZNNVJlmD&Fawh4=g$Ycm zZ_Vf>B;o+n;Q{iGqcKpiB@WZX3F2iNdHBGIek){HCVG42BK({Xw2$AM?}5hyGw80B zy#3r*b9CS>WZN2)D)ATQyA=EINc;AJ4%CT&R{Jg|!H^VBgS+n}-8Hx2CR)6r ztnMrCx=7@1>9HO{MksxwE>=lj-bY`oRCmu$su=tPY-h31R4zzQ`^hHZDk{=M{98w) zRzzUul{YW;*J_8Hl4}T~8KuhP160DAneJ}aza0DIDwxaZeW@m^yu7^M$EjQ4=q(-z z$UgZClo39AH9fuSx4N_P3Ksj@V}7sI;wQy-lyjJN+%b53IZaME@q*w!UbT#Tc;wQImhc1d_%QI+Bk zn|l_<+9+#MI;&{!5?U1^-oqAxdrr1|lqk74Jj5ClDtr5;^_eYPD$>v_uKXii=X0(` z_q2xTF=q`zaK24)8k;nU_46fdl^{_~5>g()%oP6A1RstTag!<6y1sAd(CW`w!Jl>J zLPN{{3^_+7dibt9VMm(m4Hm6PN51=s>lJ;RL8z}#hQp~!hHH{%G5vCo+&-k|UENqo zFb=-{r{($Y9w9C(LbRmM+*g2MRmJYfq`|Qlh+ELgSQ{p9ooKYS~@^{ak?(2%2+vdK5c21HP zn)`sIj*x#e$oCj(#`;3|fS^6-FcOdR*4c}SR&-xp5zIl;cX=1$7t#*ac~9(>Qq`)( z9jMn}$CT8`j0QjNw(|IyXCvrx0kMwDbNX2Yle@bp+!WMGGp8$*;}Qz->=Ikn_d^;+v7UM7GTY*45JC`&Wga zG^`G8BIAZ+*3n~9e|RUiX%AHz3HES7d=yA-0t+J^dTfNMtVskRdRcDNVkBf0^a*9< zu_^~xjyC|B1)@Ah8B!o?oPUnB zQ1iBA!k&+3=@C^)@;O8fs$c;90H}}n#@{?bNj_g-d&9D`mc z%N13F0(AcnBb524_mLb;VW&G2n2%n7@h>gC%MF@I4E^rT2L39yWhEs|Kf)*3$a%V; zfUSO<8czZjcr}t21w4&sOpGm>@Ha~QQ13-J!!%d5?eeC_yn6IL@BD@Tjc^pv_&c&a zRz6C)MG+*&b~~VX@Fq}S>K8;?z5o7s+un45aKbsvt14MOYOnCy&5X#o+e@ zXb8inqhH2aRX@i4Rf1;SBn87F5G&QO4p1bVsd$rT&f9C2PJ}n*e9+2s2@tD0qi({PThO_}Fb=7IVmhs9w)~AGo;?h4l zCrBLL&Yz+>xq`L6qy(c!$q=*=Ikx7!{PM2m@p5tKmPGVg zbjYxgFiLvUS#aH+BZ|GZ$oZ*_V{V~n$KU0B zzIj$>HIXXAaT)R$v^^>dJ zcubFq3_WuBTOcPV$7#2}=;_x(uNGP?J;+1nWY_t~n-}xD5jT&WOo-G_T{B1A-0JNY zdOlJM)urb!)nS~jMT>=)4<_za6c@jkJh)&O)olV;2+BmZ#vD~!JurEY6xtJTalTRu zcC^Mk4z&;!Hbl19^#EHy|6@?Jg@a?)b~9htAFm%f9lo?43*8^D!#mt}zh6~Z-b>-xBMeTO0G4ov zI*Ztu5-eb+kN=n3ZK`Y^{Y~rDi`&wFd{-SJ zV0TON^zv!Pb>(!|q<{f+oW2lAXHrrtZrt(y&smDsG`o~_m1e%&3h6?`v$YP>#w~!6 zv?QZqvlTxMnMSxZ06`Ei>dHoZ+jZqvN2E!oxV&M{Ik3 z*m{4xd-y}>pry@mUG`JH(x4Q;|j9_8gS;)HKc-nxA&g}^RP0I&zo!Cq~ED66uHiKbN}_^g}l(jc7`?^5)vR__>zYOMprbH zAx!D~Y!iO5Ho81<_!V2moiuQO*Xtt454TkrP2u(`mLQ3*L?Bha6c%=u#(!l9Km=K} zt$zYMm52kiRhZBk6hmE*H*%ZZU?lFJo)05q{!w?nNXZv?|6u`k61phP;-)cwi`#PA z7_s;-+tn|e?{||L&2P=W8>u}J=3^0hw>cz|kdV+dI5?=4suBMeWisoZoCsxyN;ODC z66trE&?Fc1z>B zH za-^&H=qrxE>h^SxN47UBH;@k$!{*OO;NW)J=YDUkcYen0fx(m_tW;#FBGp)^@M;Ue zD=LOW?rrP^L$+03)l32ih0ZNBLbK(VbKA?8_Hr${H6*dol7wf^r_B`4_E+O`zQg!k z?i?QHo&N5oO@(wcH4CM&4z^T&o_|)@NBDS~!3C~H1+jP`WwZ4=dipyq(i6=VoFzTl z2aBDY`~V*Xw(D;H?peloo!uP%+mVcgseUx%Mcdu^9GUp>Y9PL-yGPC6F*6s()+pUn zg^nzXwLAP!;}Q3;8v1lJ(^9!_)`u3&^pL@eO(;sls@N-ofexQCoG6MEz%A zXQ$q{UA`w&hz_)ZatR{jf6T>-9~SPBZDXb)qUz-+T0w8n;TA5!r57G5<)5XU7f|6A z1X9zEONN^_RFAhQT(1pC_U<=aRC+B%yVPluM+H!WT!9P=F1sc~Jkz&f778BL?{yW2 z?y?wYlLW9Gx$%Bl@7&%kKahNgtKj*W0MW4s{Q5s^cdJq9dr_e&rF&5zy;u!7zP8$g zk+}ei*SmG{TJU22-CF~nv&pdqEQ`Rfx0(CD(5fwludDO&p0KC`ddhP#9p~3@fTCBQ zD?INRxS&iRh68SHcgM%$(1MPyq*{pc+9QDOB^^8tU==o5eogcg_Q%qPl0@l)go8ml}Dw z)iOMtv9y-?xZ;;G{>VSLUBYoCW`h8tnYBdwgSe6!v1mNy#~&y2}IiR}xjmRXNJaha_Q2 zp_;wh+m5(OyQ(k4xa&YncF8o<=fDfTgfYd_3?v|AK3JSZKAnyLOuEESeNk}4CTZHf ztl1g`o}wuVeA7PT`9+1a*F&-bxcl2p1g^p9@C?0WgX2nO?q9mlv71;+-*rl7^T2?0 zWU0n#;(n&t&0)lH$L#Oc%wTIQfLU6ktg&*T!Bvtof`3FIJ}vFO-K4O9zC51I{#a&4 z4>^t*w!BJn4aJn+8jb+l8H;^b55J+=^Uq>RLKw{jcl43wz`uWe1j!-B?W#JYUGcT> zk> z9JDA^X{DWM3GbxRb63~aDO2eh19*6t^3Ac%1%vm^os#3Qu|hZT9Z$7XlN|w3ZdQ`; z4X3-y6>mp#$d{qA;x*&hKbYA!PRX(3mtTKV82%S2S8zWw>SkDGxsKYyQ@_YhOR^wD z;7R57UcHh1JO2lxGQlKnSs_EGdPsyEu}OCYVzk6YFm?W4WWZ2K$c4#!#~+te$*yyJ z%NKKqK;fq&9FPcSozq%!Fa^9YgE9rJbHFiz#5>6(f@FvwjFjW_4DD>AGIUQ5xEoDHTCvZ=xJ(CmXLS*4=e}Sj`p<| zv+mm+aF_5RZaRJ&{3(I_3R~%!HwLsY$K~13E-+{oj^6M8^&jLUx0P%S6$z0g6Aq|& z%`ly>0j)_BeM-TU8USO72-Un@ux_FdB7r-JYtw$zANBTVWwI7sz%$Zp3Zb;5I5Auv z#JkYQCWht;)SS?#OW{$Xq)fgtX$yW$4n8xaw?q;Cy=}@sqRcC}vHN#385;hn{u?G5 zWx3QoY90O3dTB6G6+!}KLsCmwvqRpbL$z;>s=4Lv{aa|^Upx0Bq1>*{VNPRtxj$@|R-S2i3u7HcOTFkZlqgQN zx+%rws4+vd-#?Vv_S`~&ZxVxeRQnRjUL zAv*1r!$ph=lRwcqZBlG&VDQx`H(R0c^p8-T0md0r@zKd5ljjdP0a;{29ll!&rNEiM z`kIWlQS%(~=Haq!aZoTU{55hUz4>oJTY9KCjrhc}%0qq89L$pDCv z7mk(Roj2r)#`J+uZZ5`#7_9T4Eny`}@LXc`FC(zG@D3XZ5C+e`Bro3I4YX7A5f9d9 z1&t@>hQ!6FGLrK+aRA3&Fgy|~f9>cs5G$e4b*WT*R$+3XYvKTO95|c4rd?BSdklR~ z-y96=(Gpl9@NrK2_iJ&G_I))#jIU4L!bruHw+^B^z!2v1{mGzY=%c-NKVEDJQt&im z2EqJUh<>(&M} z&6|Pjh2q)Go%?lEK42t#G3k$GWkDdb@~u|OlzFm9^{BdNI;~brXKEe`fI#p3mz``+ zelb(6w6m8)zG^0pep#1AKC}?pNuK;B>t4#mMyuD+8co4QIsYAw4JF|arHeEJft|_EFl$BHAHT0(i@&+K^dGHJxWpu#?DyBc>00rXyB?l7|)9%cdHGW|FTVEu# z8*|lMolWUEblOz5*1cV7^9j85eF)uj<70~fj>$-~{tYiq_~^cEc(ymtuM$*j+IXwx zv+FGV8qqP;y@6f^B^}roXJQ6PX(Z195v~*4_5>wFhUTi43Tk@I28-fcrP0@nq zjnr&}rKo^%ra7$BUWqP+C$m=uhVQ>J~a`N zdFRMdH#b!z3rISV%Y~G#{_pYTJoVq>0B{7589#Bd`OR_TxnShWlC2#LPBF=V?Ip&PRif<1Qs1yXsj)qg;kb6ZZ3JO13 zOXtGndF7pkCjB13NRKNlz^xT8rw|yP(V@I6rXik-B5cSk%l;bvl7D5@0}-|&9E9Ni zjz6}#f6sh}kS@+9m%$U78vMUS#zu4qN=Zo&^eX|AkNQ89lke@l)84>^dsAS}PX0{mUc0@Ab=0qgOx1crZ(z9KWV6o`kp`*yy{gqDt%7;)7XUdyWs zB;}i4#A3McGeE!7RQVt?9W@1lT2NMU1GQN|7CVwNT_VArQ{NBX1BXT56-0zbMx4JEz;UE&A2 z#H@;3DHbRr>uW6Z>({zESB;(3Zq4&50adTR;&Rb%GT1r`>R}!>7ZRi|30#Qg5Li3d z@veD6@%Oj8mF6SDC%8f;-L?=)Z{NQr{O5e8%hs3TD)A`ss(cZ*idZ{ROx;iB$cgHT1jz$>*vh{U;V>gd_x~?~ay*sAF~ljw!p`4r_Pk@@>nXKktY&d?Ym{C*o^r z)R^<#dybhKhQmNVRpoBb=rka~bQ%T-y;@xzUEOONKI$zG;UwmWSV`tRl%V+9;&S#o zUiLl4r;p)$uT?bB@+GbAoV;FK8ib3#lE@y<>5lZ{qwLc2X**HZpf2+1B^<|Wf2$$q z#}IT116i(|PJ`m#DqnMoGzNFVC=B)6fzk|tJMxlp=_F$bPDb{}KcO_{$qj$Un_GWu znkitgDsr#t|(_6B|iyO+=@G+VXRU zLkW#QhRnxPJGryBV7Je;TmCJH@uHjNBZ9#!SXyl8*x*;+dZ6fxk`_E0Kvl@;l!n_Q zou5y$BNV?@p;k`Iqd2>%Yo7aVro25r8t%XFB-p0m#0C!tiVFvT)>pr5iSMjJ#hqli zE;q^z55(pA#3h&|R>E26^sqVzYG|NafJsWWtc~wDW=?`8UYaA`K^o&R@tRt8cj|?> z6%C`AQ1<(0=z@Q#0}aH%EKC04wHsK2s0IH)BM9YiLVKf$pN|l{Fm4v_nMw|MqjWm{ zB}}EI9|Nv0dq)Zy=ql+AGf#XNoho{sP_$z9L`NFu7w4}0%C;z6@8vt)%Thd>m4w#| z#+SyQHN%?^pCpWERt6ju+WiaybZNKuob1T`ZEjumVoQRbW!^z8rOKY?ImTS53JOYB z_4CdodHJm$IaXsMux~}f^gTR0s7;pIgo7lo51%vUNp<=o?jS+em$?s_tphz<`XK+qVDB%^|JseN;YZiNIe`oVnsT?O&Qfhs*P{ ziVKthyj6w8B@fxgj+je^)osVlk!S(k_CEvxqtkjKtHJ_;#2SrYAb9h$Dt91N?q?Mk zF4JYXV<7^K@1##uC#?k}@~xMArF4_xhvzX2bDH(`cXTKX?OaeU1-!U}lLcAOHPQ9m zHsMtBtG4MIC^{K}X*e&aMf~D`R`2+*OAO&Zz9mU|EH2Rp12sVKHM2lUKp6%JdY}V8 zsBSEGB&4QBg-9)HQs(Sl=G{ezL}>&I3w`KQBD1&nx!I#suyruZ3{zv$qq`HP`as=>eCgTI03BlA%N)MV^a43Q=*q8tdSuqkPO$^X;(Li3l zN2b!dych#{Wq9z|<;tS|@AfIDJOiK>Emb)q0OR6jMXUoztV0&a6w=YjLT9e7!8!ry z&kv$f(5TH5Bs)FXNKwc-HP2x}bMm9Z$X1rPmxo}X%Hd;}0>e7M z${5BzEThg(3GtBEn5-Ws+Q60D`6&L%O4rkKM>0Pjk3QAzY;ucIiHTa146@uNsRf+> zDvG#08UJ@E@LA;de}@9qQ9FgB-(u6PBLQ|!f|cRr6hL4?=}I-^lx)=0s`H<+;^g{q z`=haY<#z{koZx}RwkW~zIfIu9u)Jh5i^QV;Fh%`O%s?+bO>IEg{a{be)Z+KEN^S_i z0pkhrpLVf2rUAew{&p<)MitTj6r@EcF(6s;d#l_P2sg?w-&~lcjq=}NY%;Lng6~J{ zjlR%c(Tq%p_xY(*5Z@&y0G4F|9qC`*3`w}A9sC}zaGwTlr(2t69Frt`_aCLD`?EDx zb1aI?u~|$Io`VGXhDU1>)E}lXx!6rioK}uKGWa7Sx9E2*(K07^*#le-d?K&0{r1VV z7i#6vd_5wY$?fJM`KN-o`&nem*-WfBgM&w5lYI$WXBjzA9_R&F74+fNkF5B8P0;z| z2A{X1gI&4b(Q0=)W&hcgcR7z!^c#_))y41jStW&YJEcIfHXi8h$th_JCMM?8nT%0+ zfrV7$w_NeXLXyPCf2ZdG;>NHY{(?@?tS5>xGCBE((U@zth0nO#-68o%gtCx5-B zC}UN8*_;(EcK7UD7OgKEq^0biE~S#e#KIYE&CJ^OLzEg#7-SNqI!~Fb$h`0h@&bd& zM<0c03P;gouhVSQZhZVOS)G>g$w%ecd=8YeY4IIdw&Gg3odMwI+T@B_a+o2mK7*E- z4Vl_)+jHXI=u)uWJ;Kb2HzRcJL2aj1+Kae~g^^YNE?WX_7)=$iDyl%Z8~HaFxfL$| z?|kKSyfrXuxnqvgfNyH5YFA+CwV2L8?o@PneAOi-e@2Fe?6K*L3RGrKvW*XMG%=gZ zW$pGr$gx%s8nMK^zoTvvlSAp!~v*4dwoZmmom2z#H`FUOseEs@0Q^&yD2&ZN7 zgdc~FE|3e(ztG-Zwx0QL+o^yF&_7&Zy&XN?KeR{?~wdGf&?_ZH#Ues4cN`up&wZz&xImEVZ}xR5y(!Nt7G7!l#A~;D#xQMJDf>g zN=7cQL==8dZi>iH4S&ju|M)el90!0eC472-3Ofaj->a((&=DO+R?MSTx)C$3=hL@~ z#r9S&Z>A}QXqMOfW`pI^hRR_5tRYw2CmiB`3lNUEta+vY~-rIe?bS3}Uv@J(? zrXEJ;zwADrA3gsHpq@tOdH&g%UY5qEIac-~PpJK0*54zXEk*ybvLWgJ1WU!QR<~+K zEEIO2bGEL#R|+J@?1cV@M`-`E@U;z1@qF{!8{$GduR26S zs{-aQ90fR`Hg(biLsMArzbc{F$-yTg8EFBZu*yqIy_VYF?-;6(Hs(sZX!7vXfS?k6 zLbF(~fR6SnXRJy?R}6;n;gKYZ~I)Jjsdj(Phhj|aCrTI-?yLS3+{ z(0@4NULr2tMY8*W@_Lerr}2i$WVpUr0vVQ9YRu{W&(fD;oRgbBsW!zjVP`PjGB&@R z2W-{zdh6h^A7t9c0CLRhoS+wD^SqudVEJ!K7sCA4E8K0n6Fq*{`zZ8zQ6^_-=E?zJ z2<{*wGz8x`PFIeQb9v8z@Jil;!#1fXN;Gorgb^} z;LU9=Rx1Tx7^EE&PTaVlrMZ-^ z^(c_{N6&nB4{6<(|27KA4klFL$~ZHybp{D5BZ91&R_`;JhHRlaq7jfqM-V6|ru7mZ z0>8Xb6zY^z(PYUU-PD!Plw|r@X?e6NO;T&g)9Wc5I!%ZiYu~;%$bEC_fasQQ6kS5M zI*aMKV-m+QGb17;B|TV`;T03QvB;{)>N_|n78q!me=pa*keR62E<*r{Q>mwiganl- zs7)rYt?)_kl#8J%!9g6@7_8r$hSyU?V2MCdkqtwZp_1C#lGsdwyatvWP9h~DC_$!s zz0Qiq<_mdQulN~1a#QK2xP0XYm1kG3rsN{f_bj{=VF<%mT9VMHZEiO9RL1d=D8!63 zx%&a&Kk$!#0h)n!j2=9IAVsN|Cw>I!A`XsR;x{+SDH&h;X&R-jiS~I^I7YyRd}g4V zLe>x-)cay>1O-T9uL4PIzxL?c`7b_Q!SqWu0zhc-9LVFo!GM>){u`YIZ451ntk6nw z{uIaeYcu=K`0=UbFSN5y(T7@BHjMKzLRe5+Z5<(Czg5gY+o*vu(b*XC=r($Bu_kXT zb*Nd^>`s-UqfviLJgAdg%OK%PwSkZVg@Gy=1U%!u<<*+}lRVnCD-I!-FgvTt$o^WT z>+QL!KT13lG3OV$g;`iw$->MW{v@o&b2M0=6y4iE1$OA>OGDm|D@lR+EJJtWKhcdJ zo!m4d4|yv`E zIAlvCBEK}@qfrX1#?}@DHMyn?P~keSX=zPWETkq)K+jt|hyQ8rU_S@^_{hzXpyn+O zRGMR`YE9CZog4hW`_+1iG~q(92H!P{WGBBDM8hdyVaJuF%Ki-8EdX=%e&}*de!jSt ziWq(MR03wmG+4qojs?-YZ^C)H9u*|{$gZ-#b@^L1A-IcolF8bASjNWqs+yC(>XOcV zDCH$)!B7S&kPF8+9Uv0{3A34yGXRY*98(%iRcxrT(m;km;ZNFRTSU_vW4G&+!mc{n zy3XXI7a_1qAn|OXx|p7t?*fE=RWZQ8w;3bRa(#|?7U)cC`Dk8>7#)QR$B{Fm#J)tR zF?VsH6v)ct&o;Jh%dxX@KJ@%SPd_+eNAn+v04|E5hRsqb^`62>#b+q6LJ-}(g1J~Bj z^-X}O*WUNLAXo8E^$OB)%=_UjmdlMFuK_u=d^>p$|FfWL9)|Cq3xcFamQ}Qby%Ip7 zhfIeL)NI*J^WcO}oWM`OI)M__;Ns<@tgE@J&_H)k(GUnjYxiv(@uv*I5BG=NEEH8L z!mFGwIJW1{nDDAnGRi-=mH-`A%dfz!M{W#0JdrT`=dM6|R$~hyT8D1W-2O@x+ za&4=(Lz#EGN*ebln6p&*siwbpgWb&tffg?sT);en=ljx^*Y-%*RPXypF@`@h|CLtO zO+S-(0Z7#%ky#n&qXUM|f%)WOH(lAzNCaH!!z}59fi6M{vK}d9#zU_MYlI*U~1tL~osD{+agt&;Rm*}_C7#hS~S zL~@$L)F^Hh2C^-Pufvoe0d=gEl$=|P4JZZ!r_g6rc~d6F_vy;12&`X^;@^mNMIkz2 z8ELjV)?lRfI*UZl=jIcW0TDzD2#>YGgcrhxWDpT3r4}fkF?I6GrjCRi_)fj+rLfY9 zFP_#Cct5l*Lx~Sa>!-Mu!9Wc&7s$g|Hqex%*3+?lHY;0;&lw%)p1WNX`|>uLWY`*H zXdjT3GPFX9Z%GtS4~Gas4B32TNRolzYJek9eI=tmEDX?x726rz zu7@x({1nY=YxCRFvWQ{`^4G6ly`?3d&$CZ|;vyQ+m>9)h@zc~}lr&LIG|jv)=!)1+ z8PWu)5rc$tMWs*-%Y+2n{|V>HQGbaiUpw1mI%|X|NsZXB=i^V*)I4i`zjM3^2%SH* ze}8ZtDHmPxC@FdK=1&aKvi7Z9ZqT23nVo-UbAQr@ee=+&(y4k?1NdePng(4tdQ$@# z!cS$uf~7$Eryzs!*ATs>R&NVU?p_ygjJ|WK{#&L7SeyNhc-3`{=tys)vvWw#9RG)F z>VMMA2APzJ(QU4S^#AF%eX(C}b)J+ig8b4XuOhp8FVRc__pQuKk)LJyJq6g6Q9~gk z?p>|%o<8PNk}XOf+7w-&L;oMWPn(02r4#sKt6`}N3BXHL@8FOY6LaML`t|UifACJ> zpq!y7>X$WFD>nxd_(H2;C4gNq^&L>s_2ScFG8Q#e;%h!km6z7p{)&Y*0;Hwdg_cI` z-h6H8KiO?YXie&D(4H2nAy+?N&I}7=R(2zY)MKu8(1akV7%(PTCYbsHlfvV=umRoR zOLB$u2=Oq@6CliO$PCy`Dl85t3PAN}4qRF9yJN+&6gaXXG3?^fWS3v1$%^Q?cmzbo&Sxji1_ zJUr~e*V>=0-7Wt1?Higb3wzs-IJ(DaQ-tff81P6_lg4D=r|1{U!b`F87h-#V%L47g zbu~cio9r_f*=q@&$8tc60#$is|0||F3`oJXPM5T7BEqdK*-YoX-)uiaQEM75q0Vk& z0C(*`A#H_y{GVAx8n_V8!hUH60(w*9ax#S>+em(T%wc<!pxpa!5f;$-8=vH_m);PvV4IO% zWKzH06r=nyu})By_Hm@Js}fL{CN!3k;04US1R$Y2gAT90F7wR*I_&0BaUfBY0{dsa zch?4d?uW*uEOyF|47veYtGEOpTA3`l%@qL5GgmJHXb|Imt>GmAG>8>en#KRM_=f9% zil6^g9H(eK{No4BOoc)7R>8x=`48})l9Kr~&M@3J1)DB?I;^2Qp;E75pv7eGU@AmX1n5$#1eoAjXBOVij!d;*E70~TLXXTbyQ z4<-sn>b)-Z4Fq6aq4-8MmSZ^!l}*1(-j?EN1}Ws?4f&Mo^lQ*-UpRfF$qEl!rV9b9 zt{qttH1d0%Z5(75tjK=7$|#owvldnZJ$Kb@I0+DokB+9$&c{l^PGdmb*_5_Tc@q(m zn9HH3qr4zg%JVSZvwp%|fkeD6C^McBDchGCE`#YIh3`cc)D7Oq-U*0`A(9ZkXiTg@u# zXvQ?UAFx|}dxNkZw>^GtNWhaKUkyA#a!>e2vfOY!w&I6pReKcbk!mv3Ger-eWm zk>#ZXwr52veEYOL%Jj)3u`6rw=@1&Xd8bx^h4&Lyk|DS{LU?>Yo0sIvl-_3w$yGKp z?ub4!j^U?T;l>ot>iEN!+vuIs!Wv#aT(sn?wSUOaI2nN>n(3Ow|U&^`$e@N4DOsma0^% zNWBd2`%V8fG)!1?dK=xStdqWchapT_Io$Y8Ad|53n)?Gq_8$uPQ2s)*o8KzHA{y{o zvpcnAzIazK?>c7}Q?Sxvt~=q#+pqD_`150(x0&>>IMoz?$sF2AUfYupSfjt@4>~j> zXdJ(>f1l1p?Ab`zImL*i;MadBFO;hC#m$vw5+X3WM}U=y=&FlqEg0*&PA;fVI=@CI zy^nkuWaYQyRLe15XKOXvrAb{0Nl_?bYV5POv7x-p4=hkDdTcq~wZuVRm{PQ2^8o{j zI&=6MSQ8tH3M#{o6csl#n)KQNK{052s03{E3GxIyzsi6uLDAVaS9$;Fbq>M&_mdc( z;jAxkUMZZp^fSyz;ltUN&b#st8V^0ss7)HfJ{5D1Xa)^oU$GdMV@xZ#7`*QJa~`dv zZ2y$recCDTCC7ogQ-+LXw<|}|qZhP`^kyWrn$4N%LM{g0*Y}U7`@HyZOjcTQ5*s)S|Hy=n&99Ix7d)-1pB){8|U z62XW2i}rrX%Y7s|j<45Fei9AsE`6aI(3bcvN$02TL?W;-J65IHesP4I)lzxUVSjYc zp5!-6g=eLP$h$GwB!Ai@C_||UN+N=Lro3SwfN#pbOtD{Ry0d6Ko0@A<$`rM6sHg6K zs%eKq$`LO^;~}*V{N4@fm+#4VJdW2>PsU09?z94C;NAKrhFbd`SS1M2MB@^|Oq0DK zeuIJ^uf8JMcmwyjT2)_q?_R6@1)jsr^?7Ka>H^)qh%WdAl1$`g_Wduf=7%VuVfAa` zFH_P_kR*Rl&@gV503$slqVK7medESgJb`(~kBUIeq%J`Gt$LhnvZssr`Cj};7kcpA z8PCssrj#N4?Bi<|i2^!{#!7xX;LkQ8=t5x1$4xMV()tHsSrtau`A9_ZQvUoIyBAgo z!4QopBeK@yLtRCAxt##PuQC83Nl)T5x_9l3qRyPN%>771_QKgr&n{Q9y#$h?sg`u1 zyBhkAV69&l;+P8jTa~ko6P-y=U{c5kWXW)6}Qp@k4%*>OB&)&b?Qv~mSCR|TR zMA2j%#q0Y`EI1#{sO9O^bCmWOw73`Jimpkd+(r`T`TjgyGyguiHhaTZ%s|S%cA31V`-lmDsxgY z&fHdIWS{w_^uXC+0W(l9qzjaU0u#=%(POO6eZR#f;~UrWzSQjz~Zf!$6?H~IV^&9qlYxF^&XoaaU*3nI~|OA-+0!( zyw#-p=+qxWO-4AgnsvzBKPW}rj-@!CUf^3klXVKuN_RUDaZ8lMU#yql zxca4Zc>4qzOC>e1T9$Ce?RWV3?%Vu%W?)tR((cJp$)9e_$Rzct{?xng+|TGCxa|Ur z+fM~Xe;HYoSUv7cGV(U`oqf3Bx%oxx#3v?Vmxp`%D@9{Pme*$}mNC+NNT;yR?V6gg z`40)rjDZ6{>D1rqhTsYpe67ed@=d22zX;QoftA-9`i^;g-PzRZP6Q5Xf%RMcGMVvK zY6tPzxgm1X6{T%voH}3Cd(!z8?c1VM*H~wj6oq7?9)3lz(E_7jkn=}^+I1%Vh}caF zmFtj6fODW3y-gaYJp-v?qwxev9wpeI`wtSYaVD*ph7Qi|mQfw#N^wnPCRQmzU)&2)Gt-xP ze0K?aBwRHo=nv&MhV-&bjSCdEk}&Q7N)k76tDinuErBI&CWBRP$2!~sy0@k* z8$pb=$sCtLm}4c0rqcy)##5{&j9N0MjZft2j zp8&+8${o9S*7(gWzPS-2InD9No#_L_Lp%<9>%8&XPGQf_ShW~bIU(k}bEt&J5F{e@ z{Opdj_ZrH`JF_o%p2$A;@z4zfr<#>p3Bv7T*&1hAqfAX^R0yW{b~MLCQ<|SQIwZPI zyYA&WLcP*mee_@UH||l#H|g^du+ue9%N`bjK4+j6i$d7Q8_yX*xx)#)X+J-m0@d>L z{6ZE8Wx^_tsm=uY+oeEm5bh^;pn#WFVWxX{Zl;!WbV==B{?zjI!+G=Y%mqii% zr$6AML1pnn*#kTf9J1~ti82equm>;I$moUG{A8(7ica5I;jqks=i%+Q_QPH3iyRSw zrCuWK7BqvZzhC@Ww>|~n2U~zDt~&lXT-Q_rnOUI3g1Oorq;&bpiw#oVwo!`F1esA; z;u>>*VpDI`Boiy#(j$DeWLG>K-sFZ|3cHw5xl4-CL42$)9B)o?+Lr2zP{3ArYM!@0 zJ5_Fosq3UAc+Qw}us*A||9RH1y3yc(2rPk$v#C}ev5DSAEp9i;L4XXP-Iz)GMqTg2 zm9s{&%*O_iSn<&3VL6v|InRG=L40ynVc{JB44q=EIrw zsCzDp<_K8-4c=p)L@ot)+wW%lpQvS;Cx5C5FgO1&r_Gxv+}z`~f4_xj3MqLe9)MCS z-_*@JzTC`iht#|6k_jN&TP@t)V9UWRf74et;@m^1;=IZlbdte{*B2iiyZPX@3|ur zRwWx@M6{F>~CUmbFP~|s)MowX-^pHxHd*2x44*%9BjPl-v zc$`D>Dx;Hk=W(7TX%ghX^=?1yvx;LxZa#L*Pq_UZYbw~dSg1%1LO5W=`@fjjOahZ$ za5Me=;B#CNHhCA;NVzib5Y-0GiWEr}FQL&nRFh#MR3=6v-L<~aIjpy$M%}u{c{JBUir%DsjH*Hva zn+isQ&V(wYnmpKrl$sf=|f@K?yTTRwklrJ|0q$3i#Y!aY~WA&@hyn;@cV*czM*sqJjAy8!$llQU) z!i+oc(^&_`I?f8F#A%%+E$hN?KWJnA_;kunEXe2KVp%mU2nr!c#s3(U6x8^_WFvsd z0H2)yorfwm-j#ANx6B9zldivYyiv-)2erwk7Ug0@fplv9P#jiVQ!muRa=w*1q}RgX zal(0FmZr6PnkHN;v4IHi(-URCQmu{?Ia@0h>IBEUoS{MRgZih-Rb!aUmrh)nW979W zj~PB0aIK{NIo#DOgj5n(v4`)qnyq}_D6%L@%HU>G6`M&rf`#cOf^-saAx7@B#8k#b z7Nx9jN>~VwHi}EaA<*~|HusH`mS3+Xn#!2otDQh66P=WIB?WK(#mJ+bU|?pT6Zlud z+G}RrJHud7mFZ{o11IaXpam$wc(83)6SR;YyWRDPp}~w6;`G%7K{4byiMC=a7NH~L zVD>WtcW*F!c2^Gq>$f@t6wL93E z4bOyNu&UKx8isAh-m~zO1yE#g2c_nnTlpnLsbG=V^5DVZ(+{Z@E%tpr=7p3as|j&un0Z9w$uPfko5OMp#fYiGai}*YkGSKt;Hq{RvL! zJyR-pG+jPqwO4MTUZYn@@@Wi|(0%zNTM!0!IU_f6XS2rzZXf5dv5g?*)g97~lhRF) zPB5GVrA~aMeV29pSMm=-DP}FkM#|)29P%viP5cSk-b4;1gw^qN&tG>x5Ne~;I*cdD%?LAnvc()j7QIdGaZa2U zQ{&4X;14CR#h@wZ6k(OV&QOn(YKG@>`Ez2i57Pm%RsXMpi^)KB6ffB!Ykb!bPA;&2 zfYAW0Cb#XYhgq`htTCw+kCb}0MjoD>@5$2zb-l;ew)?cgqBS2vydI(iwluz0d6D1W zx_(x8*F+;e@Y7d^Wzo#h-b(Z!~K8j~LC(pys9B!}HmW=$i9`2UBB+CPbq? zFl5Np_mmYDzL;tCvw9h7j({bwV1u#2Q>)K%hlF-nNlbV`TymsmaVhxTaQS+`xv zS8c`7B>hQ<4>EyN7wxyTlFOka8X$KmT9r8Sl*c6Oii~DwpiPGONR`DXHp^IkpoZZ7 zVaE5W#uADLYAn_>l;STR!3?=v$psvRU})adw}?Q-#_xhm{OmXHJ=;w{&ZJxZoe;MM zC4p@zg_`d{oXD6-!kRb9>0LUAPYhv$)C}R=VmW#c;|l#l2r$#Cz1gu0ci`VyL+2z? zj=&_ZZ=;JlCd+asF?&$?OO>k%#7o4w<^l3ka0~bjM2pgBhaw4F8q6HiBt@L)QAdvf zD2oF~+(9bS1gJ~MNJ%h3S}so`5%!|#-yvC}F8*QN2-fs4?Og+tYaBU%H88p`rW+!) z@0XX$uRm5Pu>V>WZh3E*rT65v-c?!T-g`Vn49oN*l({(J7W~z8$Kr8wpG!UC**3G_ z;Y{FKyf(Q#7ch@A>RPKFj0r3f|6~ZYc&X*4tqUBACT55ie*+o-^FoR*tMy=p3L!dN z^e9lQsF&s4g#%3p^Qw^twa86*+qBD>i1cs17S8}pEte~3fQkZr8V!VeoCR|d6{H?6 zmEQ`0C4oce2v-@-%c%x~LKD4U1vo{2!Q7*o&u@eP*v$TmNf(CTZCE9bH&nlbDoEZn zLoe6Ygw0;6Ia@em!jNyl0;jKpd8bWH+D7Wozbigq1QV|_eAcH}t9a^sy5_p|i0x(9 zTa%0yadz3hx>@g}m2dm)S$RG0bvi!Eqj@@fJrO?xxlEj3=B0oX z^P%%XR;LZP179dJNh3Q^TV1es^Ta2Daq-Z0 z5niRN3~Ms+mr%*k0_QN-Bl>>KDH33GcFqY3h$M}^(-<+4viqCh5f*K zxsxvqHYXofuW$2@7j5QvY~OiOVtd^LX#P#Qxi!zTYA*u&r6VIBw_&RiTWy-68TyXK z1tM~9lj?f$$OiJy$hT1wqu@e#+_y_y{Ts4~_BxmwF3hJ7b>{m3Wgap8Vts%Qymu9hyP9rY zM+;qY$1=8KPCCsmkh!T6j3j1S3Cm_$9o6D;(KBtf;i%L33{19PdfzjK_%;=rgpSKG zQ|yXEAuI=_O@n5JG?9+C)#uvED>T|mxd(9HbUpb{(~1A&m;5$TAA4=g{7XjtZYsHm z&5uWar%NK&YEBOZ9eFeYBpy0eTgESsGn=WCfWXsIoki2c;|tf*LA{kjRL2=k|BiRY zwe!xKXJY9E;YVYm^MKblZC}%#U(!oyKfxn!T*fV#PFcqDEhW8Ki+pyJ7xi_A`q;7~ z6mZB=k@M7#UH4r0;)-1S(90K&0a5plwU1k$-EmU6?ftBD?^hNjGWvHCSYl(kYPMUF z9%7zwKmBCglZle!HG4#Rb}fH+xLf(S9_stwbNNA^17(QU)DU#~ci_?A1Z7%nSbkyoO-EK$!gZ!VR*atGzn=AMGeAi13dApbQnqPi9_|_nA(DD4&Vf}r-evRzS zorNy4i}O!*W6c&@Vb>0PwobmpE)wez=0+qLpAr7din{JSJ}n#gu$yqG@UMKDG<(V^ zx)f+ZAT{~SdnMcCY@X)Jbi?ZOY4badf$}IfYPR@fnn<){9wt|bx^Zk`O$t6=MYY}= zT}6#8<@(5KowNUW*P4HA>3crSOl|b0eRb^Rsmw=wDi6}%y*le7!DKB?!sjFT)~jLM zbKyTBRYB1iJ&iq@VmC7@?~()n`W_A%0E3GclJCm&*?i_n4yay(G=;N8U&9kcD;aVv zH2rC4IL`b_^f};bBF_#9BzGQ}$B-DxX(ti@yT~?*u$`PW%GY2 zyIe@4nWJmEGlz{Bi^`DU!w;QmDf}xcA!3%GEZJ;rS7SSc7{iT#yiD)=z2cbPYT=ec z&z$uHDBJ8Vqy`TT-OSIw-!(AGQh76jk4KAexhXw~nU7ub^P;J6!9{-3@DFF##j0B; zJm1}LWOsd}?ONro?|9&ztxBu=BBS9{;XBYAFS`% ze$w_lAtHZEpKHv4c4~VvR3tlZ%9e5?&0R}Op5{*uS!p(#(rdrP>m6CnXxHf@$voM! zo#MAa`)2lM`vs#wqSBFWUuvSoM2#0ON6gx3ZMD%uYI*Jzskfs7H|P$}5aVjxf$IGl zpUa>Gdctt7n~J~W)AuZ`UOv&I+sQu9Lt|L|rWK7tkp-60KZ&uRScuS>f;6#5{F5VV;bDy>!^E9krj6!W|eRqYs} ziChM3er%T62(c7H_nU?ZgGzJz0iFyZ@|R;OpiW!5<-G|-yIs~l+1l=!{Tb=jP;3fMD%Jfz|kL1viDeLW9H5+$*%>+V6yg^Gf$ZJItp z%b|hW6IvMezp?l$G+~tsD63Gy0&y|TKA>iR$>*BHs`i^l+CrnXwD;VLXcZHz=Nm&` zo>(BpZhHfV45rJ9jbP`5%7!>XU3C0;-kL;yQTCEFw;YHV05|>1(nadt&?}Zf(E{z> zudj>PjyvCCyl$>vtXj?j*>ePyH2#~xRPGT^&-kmfrVf%S7)gQ+LZAfPw=$FvLVKbl zm$r$F0+TQzAU?eKq9PtV>-Uy4K6Irj|4>j1Ns3?Bh15>?T!_JQ7j*CneYBigTogG>ig*Fad>DMlWH5KYj)N}YVI1ItV5 zcqjonr6ZH9Mav;~?L(q%n&j&X4y@w?5IQfQ+Lj-3Tx0~Jc_shRmlq+=YTp|a-srd2 z>=)+asMwRO)}LN{<^{ZfPmYi9d*oHF(`be2@1#g19A0oxYhD~t`>CY+`LJzm4;f?6 z#?Ew_^-IM?XP~r{Lw9tV*Cbo06I?Tlm;}LbFTfo5fb)+!x>Vf~qWH*R)y+Sy_bgj4 z>o{}aZ&go!@$vJ%iCVS3KRubDf4Y|C-RK>pfa~ObwwCRyJ}paT00)(P4+jQQN&jaY(lCC#{s%b-s4_^fr@K zteo^;&t*z^nU8Imk_~*mtcy3oy?c!f0M1YV3;b9LI%WFd8MZ6^Nw5Ec&S%eTS2|s8 zR8v{6j;}XRZ!IW|u|zz$t@_<XA{iBbdcNlC6Ti@%_nA_YoE$;n_~kp6A?TLzN5v@tKcb zpjK0ZrKp6)xhFN+%l8v`d5%KN1&DsuF*N&QPk5+(b`tMo5q7Y+l{#A#3b)o!S*Ux@ z@lrb7abxNFyqq9^r%xhdIYVl&Td{`SfXG$alvXnO_JmeyShinSeb!FNn8Y~(Nl#KQ z8@I)Xp{KIohGO&LVYz}Rf7K=^K4ec7hj-%W%C1KuD&qvbQO|M8{>3ym;biVrZ>+4G z!VslYw3?L`V#@)eQAzOncEeP%N-G?y#NypKdqv=+M41MSf=S>{V-B;ZpjJAl^1^ox zi(9>iSar_lmDybi@dq24Tci}eqxoPlI6y~P-F|xvGQ*H9cVnh-#poA8 z;>t!cm9QLe5)HBB;E@zv#|f0*_)642jCClb4M;_w;4_=yQ%zNS!rsG=M+0x1LD zD(9R}RP_>t#j^|Pyv2hC^1*jV#yOh0m=tfpck^FIHPC#r)7%sJ-f%UL14PUn_*#rU zz=<|zTDVG}gG}v~DT}%XyD0hdFY$$Wyd*u5L`&ljgu0Y}t7)*YOuTx3rY?b@R=_L` z*4rTsQiuU_h!Q;xeFRdX%^5`;w}+w6(mCIHa6??uc)3?dc}Y~bkj_|#ot>y3sj;g3 zRwQ!mzuPF1d(MsaOdo%T_4i-Nys|#jA$f-+`{kgHc6dMp^&?Y`yIXOjW{D3>AO$yx zd826KFUdrS)QkL_0;M?V2 zOqnjIGpcDcMt_U1Y=_+T>Fv>vGU{cGE(zv|&y>lZq^&Q(F-6_A{Abp|NZewRV++^j8yerGA~p(h)IgPk{PNG#eyGZIJVb@lrezfjXOOv z`a1lMz7MQ)V}!oHbfyGxc$2Y~2+QZ|C#t!&vO$b~YYtW99kY2}^U>Yj_bwGq`S|4# zU`9NE0j&t!fdMF;hS;szN**{%49#h~ypO3O&1k0jvb!2WJNxG;mqG?jbgLP}uP?mm zL+?$+njG_^`2XoTW?IY_>6un8cVxJ5`n}3lNOUpG@v0^a!_$2XC!wc z=LF|j(=%a&vb>IfaNfR0ZlEAf4i&1iIGn#75WZngJ1%|IpKI zTgN%5eMgO|V@xDe+9@Fd&;ta}L-bci89S96S#aDmsf9@AUq#Y^JA=OVx-+y=uHVDX zfXqct(31_()`o#OI~EG`MeL-7ixZ%~g7Wt*RM(iw#0brqgKiB0msSWcDz80+6+-4X**r5!Coc^JVG9_ zGGq4u@(O?|j`_;xgY?Pt0n>rNmM3r>2^JKThOS5Jeh;XMwuvOIe#FPOYWrz08Y4y? zc^tvP6o!EBAw#tjdTlTe0^3tRnH1|aBxg7r#(7mO5{(c%XAT_dG ze0euANBY=^8N#R9u0WdTI=?fsEo5nF2fZdfmIse`K z1jKB^!u9@+|GNi>$_|LS+icAJpWy%f5O6A9AnK$z-1iXq-@#XNECL~za8|Cbe&u$Q zB%q+E0}_Q0@4Kc9!%}YnzAmyfhVJ#jGe`ERU%FLyY#S_Wy-xdY?VncXtc;C zIS#-(uf_2;790D&2nQ&oUE5{Kz}n}07d7nOnf2EFmP2^nO)cdG!!DV978N7_#eb?m ztem3#M-+h^;v(ASwBPL@HdZh&k?1Q@@%d~3vG!Nr(lUDjrd#^MfvIfn@eES25oOP< zKpIoAcH}X~LY>~IEPC7WCCL3&m6mA#G-4>fG_u&xmt-BZcwF^@_F-Wh(y6326+WQv z%~YA@3tJhJa>Fzjx8*CK>X5)+xWfYo(^V@db5AA535@i8CY0!gsS$uojtHmeatC*^ zrqqvTUoK!7nK!U%tSc7olA&PZ_oQ=Q5aycgrLL_$KRlx~`_F)@Sw3OP=1HjHKkm!U zA}JAhBwgY1Tt%i~W-MTm79~JVK3ruY@SO={I>bBib-dMY!>7!+jU2XKOtSBVSE z8N{a($;i)c`&0qq(Xng;$=%psveWuTUfKxz9&vOzzD-X08MOT;hR<>EfGoOYVDY*= z=%G0L&`b0#@XU?8LZUKI>MQ50!SIxT>0|thqPgPUGr+=L_U~7@hu=2cQ;UonN!o{xk6;s6?_@eSaseZA)3!8Ar zpW}$;FqZKv2e(i3{RrcjK#hS+d&J(Q#Ll|hN_5pmYO?h_am-{fD**%*0|B~aeUjmk z5P_Tnf3-D1&?U*|8<(S*fF3eptA6d9xVHmG143LT?q7bV=FoyUzK88C`30iS2r%C- zr6au1ekF^YOGS_483(5G0Iru&e(ANsG>zFd4PG}WW-K$WoTL%~EMnq5^I{|dISNLW znl?5RFFj^eD6OW(u-k0$&RFWG=Z)~0E^ZV$0roDS0Ak+@!bip{C Date: Wed, 20 Mar 2024 07:02:05 +0000 Subject: [PATCH 04/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/autoround.py | 7 +++++-- auto_round/utils.py | 8 +++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index e2ee0f68e..93557b775 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -32,13 +32,13 @@ get_scale_shape, htcore, is_hpu_available, + is_special_attention_model, is_special_model, logger, move_input_to_device, quant_weight, sampling_inputs, set_module, - is_special_attention_model, ) if is_hpu_available: @@ -614,7 +614,9 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) special_attention_flag = is_special_attention_model(self.model) - tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, special_attention_flag) + tmp_input_ids, tmp_input_others = sampling_inputs( + input_ids, input_others, indices, self.seqlen, special_attention_flag + ) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device ) @@ -798,6 +800,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output) """ from torch.amp import autocast + special_attention_flag = is_special_attention_model(self.model) batch_dim = get_batch_dim(input_others) if not self.low_gpu_mem_usage and input_ids.device != device: diff --git a/auto_round/utils.py b/auto_round/utils.py index 4bd239f74..0b2e069e6 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -30,7 +30,8 @@ fh.setFormatter(fh_formatter) logger.addHandler(fh) -SPECIAL_ATTENTION_LIST = ['Baichuan2-13B-Chat'] +SPECIAL_ATTENTION_LIST = ["Baichuan2-13B-Chat"] + def is_optimum_habana_available(): import importlib @@ -414,9 +415,10 @@ def get_batch_dim(input_others): dim = int(len(input_others["positional_inputs"]) > 0) return dim + def is_special_attention_model(model): model_name = None - if not hasattr(model, 'config') or not hasattr(model.config, '_name_or_path'): + if not hasattr(model, "config") or not hasattr(model.config, "_name_or_path"): logger.warn("Unable to get model name via config, assumed to be a normal model.") return True model_name = model.config._name_or_path @@ -425,6 +427,7 @@ def is_special_attention_model(model): return True return False + def sampling_inputs(input_ids, input_others, indices, seqlen, special_attention_flag=False): """Samples inputs based on the given indices and sequence length. @@ -615,4 +618,3 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 - From a914264fe500dd7df2e04d41b1b6b8e38316cf25 Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Wed, 20 Mar 2024 15:04:50 +0800 Subject: [PATCH 05/18] fix typos2 Signed-off-by: Zhang, Weiwei1 --- auto_round/autoround.py | 1 - 1 file changed, 1 deletion(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index e2ee0f68e..33472ebc6 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -32,7 +32,6 @@ get_scale_shape, htcore, is_hpu_available, - is_special_model, logger, move_input_to_device, quant_weight, From 0c1e415f10d1971165a6bb0975e2dc6abeb87f65 Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Wed, 20 Mar 2024 16:03:22 +0800 Subject: [PATCH 06/18] add model_info file Signed-off-by: Zhang, Weiwei1 --- auto_round/model_info.py | 2 ++ auto_round/utils.py | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 auto_round/model_info.py diff --git a/auto_round/model_info.py b/auto_round/model_info.py new file mode 100644 index 000000000..667cad054 --- /dev/null +++ b/auto_round/model_info.py @@ -0,0 +1,2 @@ +SPECIAL_ATTENTION_LIST = ["Baichuan2-13B-Chat"] + diff --git a/auto_round/utils.py b/auto_round/utils.py index 0b2e069e6..89770ea02 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -22,7 +22,7 @@ import psutil import torch from torch.amp import autocast - +from .model_info import SPECIAL_ATTENTION_LIST logger = logging.getLogger("autoround") logger.setLevel(logging.INFO) fh = logging.StreamHandler() @@ -30,8 +30,6 @@ fh.setFormatter(fh_formatter) logger.addHandler(fh) -SPECIAL_ATTENTION_LIST = ["Baichuan2-13B-Chat"] - def is_optimum_habana_available(): import importlib @@ -415,10 +413,9 @@ def get_batch_dim(input_others): dim = int(len(input_others["positional_inputs"]) > 0) return dim - def is_special_attention_model(model): model_name = None - if not hasattr(model, "config") or not hasattr(model.config, "_name_or_path"): + if not hasattr(model, 'config') or not hasattr(model.config, '_name_or_path'): logger.warn("Unable to get model name via config, assumed to be a normal model.") return True model_name = model.config._name_or_path @@ -427,7 +424,6 @@ def is_special_attention_model(model): return True return False - def sampling_inputs(input_ids, input_others, indices, seqlen, special_attention_flag=False): """Samples inputs based on the given indices and sequence length. @@ -618,3 +614,4 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 + From d72fb8f519a70de443ace2947218f9e062e05783 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 08:03:47 +0000 Subject: [PATCH 07/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/model_info.py | 1 - auto_round/utils.py | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/auto_round/model_info.py b/auto_round/model_info.py index 667cad054..6f4697ef2 100644 --- a/auto_round/model_info.py +++ b/auto_round/model_info.py @@ -1,2 +1 @@ SPECIAL_ATTENTION_LIST = ["Baichuan2-13B-Chat"] - diff --git a/auto_round/utils.py b/auto_round/utils.py index 89770ea02..a690d2f2f 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -22,7 +22,9 @@ import psutil import torch from torch.amp import autocast + from .model_info import SPECIAL_ATTENTION_LIST + logger = logging.getLogger("autoround") logger.setLevel(logging.INFO) fh = logging.StreamHandler() @@ -413,9 +415,10 @@ def get_batch_dim(input_others): dim = int(len(input_others["positional_inputs"]) > 0) return dim + def is_special_attention_model(model): model_name = None - if not hasattr(model, 'config') or not hasattr(model.config, '_name_or_path'): + if not hasattr(model, "config") or not hasattr(model.config, "_name_or_path"): logger.warn("Unable to get model name via config, assumed to be a normal model.") return True model_name = model.config._name_or_path @@ -424,6 +427,7 @@ def is_special_attention_model(model): return True return False + def sampling_inputs(input_ids, input_others, indices, seqlen, special_attention_flag=False): """Samples inputs based on the given indices and sequence length. @@ -614,4 +618,3 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 - From 5a25c1bbf1af0aad9acd97bb71561a543effd2d3 Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Fri, 22 Mar 2024 14:25:28 +0800 Subject: [PATCH 08/18] refine variant name Signed-off-by: Zhang, Weiwei1 --- auto_round/autoround.py | 20 +++++++++----------- auto_round/model_info.py | 3 ++- auto_round/utils.py | 17 +++++++---------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index 7098c67d2..ad7d33992 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -32,12 +32,12 @@ get_scale_shape, htcore, is_hpu_available, - is_special_attention_model, logger, move_input_to_device, quant_weight, sampling_inputs, set_module, + is_share_attention_model, ) if is_hpu_available: @@ -612,10 +612,8 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de for i in range(0, self.n_samples, bs): end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) - special_attention_flag = is_special_attention_model(self.model) - tmp_input_ids, tmp_input_others = sampling_inputs( - input_ids, input_others, indices, self.seqlen, special_attention_flag - ) + share_attention_flag = is_share_attention_model(self.model) + tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, share_attention_flag) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device ) @@ -725,7 +723,7 @@ def get_forward_func(self, name): def forward(_, hidden_states, *positional_args, **kwargs): dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type)) - special_attention_flag = is_special_attention_model(self.model) + share_attention_flag = is_share_attention_model(self.model) if name in self.inputs: data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=dim) self.inputs[name]["input_ids"] = data @@ -744,7 +742,7 @@ def forward(_, hidden_states, *positional_args, **kwargs): if key not in self.inputs[name].keys(): self.inputs[name][key] = None if kwargs[key] is not None: - if (not special_attention_flag) and self.inputs[name][key] is not None: + if (not share_attention_flag) and self.inputs[name][key] is not None: self.inputs[name][key] = torch.cat( [self.inputs[name][key], kwargs[key].to("cpu")], dim=0 ) @@ -757,7 +755,7 @@ def forward(_, hidden_states, *positional_args, **kwargs): alibi = kwargs[key] batch = kwargs["attention_mask"].shape[0] alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2]) - if (not special_attention_flag) and self.inputs[name][key] is not None: + if (not share_attention_flag) and self.inputs[name][key] is not None: self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0) else: self.inputs[name][key] = alibi.to("cpu") @@ -799,8 +797,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output) """ from torch.amp import autocast - - special_attention_flag = is_special_attention_model(self.model) + share_attention_flag = is_share_attention_model(self.model) batch_dim = get_batch_dim(input_others) if not self.low_gpu_mem_usage and input_ids.device != device: input_ids = move_input_to_device(input_ids, device) @@ -856,7 +853,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch total_loss = 0 for _ in range(self.gradient_accumulate_steps): current_input_ids, current_input_others = sampling_inputs( - input_ids, input_others, indices, seqlen=self.seqlen, special_attention_flag=special_attention_flag + input_ids, input_others, indices, seqlen=self.seqlen, share_attention_flag=share_attention_flag ) if len(input_ids.shape) == 3: if batch_dim == 0: @@ -1364,3 +1361,4 @@ def __init__( optimizer, **kwargs, ) + diff --git a/auto_round/model_info.py b/auto_round/model_info.py index 6f4697ef2..3085fa003 100644 --- a/auto_round/model_info.py +++ b/auto_round/model_info.py @@ -1 +1,2 @@ -SPECIAL_ATTENTION_LIST = ["Baichuan2-13B-Chat"] +SHARE_ATTENTION_LIST = ["Baichuan2-13B-Chat"] + diff --git a/auto_round/utils.py b/auto_round/utils.py index a690d2f2f..7f45c43a5 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -22,9 +22,7 @@ import psutil import torch from torch.amp import autocast - -from .model_info import SPECIAL_ATTENTION_LIST - +from .model_info import SHARE_ATTENTION_LIST logger = logging.getLogger("autoround") logger.setLevel(logging.INFO) fh = logging.StreamHandler() @@ -415,20 +413,18 @@ def get_batch_dim(input_others): dim = int(len(input_others["positional_inputs"]) > 0) return dim - -def is_special_attention_model(model): +def is_share_attention_model(model): model_name = None - if not hasattr(model, "config") or not hasattr(model.config, "_name_or_path"): + if not hasattr(model, 'config') or not hasattr(model.config, '_name_or_path'): logger.warn("Unable to get model name via config, assumed to be a normal model.") return True model_name = model.config._name_or_path - for key in SPECIAL_ATTENTION_LIST: + for key in SHARE_ATTENTION_LIST: if key in model_name: return True return False - -def sampling_inputs(input_ids, input_others, indices, seqlen, special_attention_flag=False): +def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_flag=False): """Samples inputs based on the given indices and sequence length. Args: @@ -454,7 +450,7 @@ def sampling_inputs(input_ids, input_others, indices, seqlen, special_attention_ current_input_others = {"positional_inputs": input_others["positional_inputs"]} for key in input_others.keys(): - if not special_attention_flag and ("attention_mask" in key or "alibi" in key): + if not share_attention_flag and ("attention_mask" in key or "alibi" in key): current_input_others[key] = None if input_others[key] is not None: current_input_others[key] = input_others[key][indices, ...] @@ -618,3 +614,4 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 + From 8ab2bd85c046f08d552129353442fe94d28fa93b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 06:25:49 +0000 Subject: [PATCH 09/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/autoround.py | 8 +++++--- auto_round/model_info.py | 1 - auto_round/utils.py | 7 +++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index ad7d33992..0f2f47a2b 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -32,12 +32,12 @@ get_scale_shape, htcore, is_hpu_available, + is_share_attention_model, logger, move_input_to_device, quant_weight, sampling_inputs, set_module, - is_share_attention_model, ) if is_hpu_available: @@ -613,7 +613,9 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) share_attention_flag = is_share_attention_model(self.model) - tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, share_attention_flag) + tmp_input_ids, tmp_input_others = sampling_inputs( + input_ids, input_others, indices, self.seqlen, share_attention_flag + ) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device ) @@ -797,6 +799,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output) """ from torch.amp import autocast + share_attention_flag = is_share_attention_model(self.model) batch_dim = get_batch_dim(input_others) if not self.low_gpu_mem_usage and input_ids.device != device: @@ -1361,4 +1364,3 @@ def __init__( optimizer, **kwargs, ) - diff --git a/auto_round/model_info.py b/auto_round/model_info.py index 3085fa003..fd881e814 100644 --- a/auto_round/model_info.py +++ b/auto_round/model_info.py @@ -1,2 +1 @@ SHARE_ATTENTION_LIST = ["Baichuan2-13B-Chat"] - diff --git a/auto_round/utils.py b/auto_round/utils.py index 7f45c43a5..1c2012ca6 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -22,7 +22,9 @@ import psutil import torch from torch.amp import autocast + from .model_info import SHARE_ATTENTION_LIST + logger = logging.getLogger("autoround") logger.setLevel(logging.INFO) fh = logging.StreamHandler() @@ -413,9 +415,10 @@ def get_batch_dim(input_others): dim = int(len(input_others["positional_inputs"]) > 0) return dim + def is_share_attention_model(model): model_name = None - if not hasattr(model, 'config') or not hasattr(model.config, '_name_or_path'): + if not hasattr(model, "config") or not hasattr(model.config, "_name_or_path"): logger.warn("Unable to get model name via config, assumed to be a normal model.") return True model_name = model.config._name_or_path @@ -424,6 +427,7 @@ def is_share_attention_model(model): return True return False + def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_flag=False): """Samples inputs based on the given indices and sequence length. @@ -614,4 +618,3 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 - From e980716b3abbd7a2b891e1ff6699e39927ae128b Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Sat, 30 Mar 2024 09:57:27 +0800 Subject: [PATCH 10/18] refine code Signed-off-by: Zhang, Weiwei1 --- auto_round/autoround.py | 57 +++++++++++++-------- auto_round/export/export_to_itrex/export.py | 4 +- auto_round/model_info.py | 30 ++++++++++- auto_round/utils.py | 38 ++------------ 4 files changed, 72 insertions(+), 57 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index 0f2f47a2b..eb7145a8b 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -26,19 +26,18 @@ collect_minmax_scale, collect_round_v, detect_device, - get_batch_dim, get_block_names, get_module, get_scale_shape, htcore, is_hpu_available, - is_share_attention_model, logger, move_input_to_device, quant_weight, sampling_inputs, set_module, ) +from .model_info import check_share_attention_mask, check_hidden_state_dim if is_hpu_available: import habana_frameworks.torch.core as htcore # pylint: disable=E0401 @@ -592,7 +591,7 @@ def set_layerwise_config(self, weight_config): m.scale_dtype = weight_config[n]["scale_dtype"] @torch.no_grad() - def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device, batch_dim): + def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device): """Compute the output of a given block of the model for a given input. Args: @@ -612,15 +611,12 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de for i in range(0, self.n_samples, bs): end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) - share_attention_flag = is_share_attention_model(self.model) - tmp_input_ids, tmp_input_others = sampling_inputs( - input_ids, input_others, indices, self.seqlen, share_attention_flag - ) + tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, self.share_attention_mask_flag, self.input_dim) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device ) output.append(tmp_output) - output = torch.cat(output, dim=batch_dim) + output = torch.cat(output, dim=self.input_dim) torch.cuda.empty_cache() return output @@ -706,6 +702,8 @@ def cache_block_input(self, block_name, n_samples): """ self.inputs = {} self.tmp_block_name = block_name + self.share_attention_mask_flag = None + self.hidden_dim_flag = None self._replace_forward() self.calib(n_samples) self._recover_forward() @@ -724,10 +722,22 @@ def get_forward_func(self, name): """ def forward(_, hidden_states, *positional_args, **kwargs): - dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type)) - share_attention_flag = is_share_attention_model(self.model) + """ + Rewrite forward function, process and collect input data. + + Args: + hidden_states (torch.Tensor): The hidden states tensor. + *positional_args: Variable number of positional arguments. + **kwargs: Variable number of keyword arguments. + + Returns: + NotImplementedError: Getting the first layer of input and then raise NotImplementedError to saves run time. + """ + if self.share_attention_mask_flag is None: + self.input_dim = check_hidden_state_dim(self.model, positional_args) + self.share_attention_mask_flag = check_share_attention_mask(self.model, hidden_states, **kwargs) if name in self.inputs: - data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=dim) + data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=self.input_dim) self.inputs[name]["input_ids"] = data else: self.inputs[name] = {} @@ -744,7 +754,7 @@ def forward(_, hidden_states, *positional_args, **kwargs): if key not in self.inputs[name].keys(): self.inputs[name][key] = None if kwargs[key] is not None: - if (not share_attention_flag) and self.inputs[name][key] is not None: + if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None: self.inputs[name][key] = torch.cat( [self.inputs[name][key], kwargs[key].to("cpu")], dim=0 ) @@ -757,7 +767,7 @@ def forward(_, hidden_states, *positional_args, **kwargs): alibi = kwargs[key] batch = kwargs["attention_mask"].shape[0] alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2]) - if (not share_attention_flag) and self.inputs[name][key] is not None: + if (not self.share_attention_mask_flag) and self.inputs[name][key] is not None: self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0) else: self.inputs[name][key] = alibi.to("cpu") @@ -799,16 +809,13 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output) """ from torch.amp import autocast - - share_attention_flag = is_share_attention_model(self.model) - batch_dim = get_batch_dim(input_others) if not self.low_gpu_mem_usage and input_ids.device != device: input_ids = move_input_to_device(input_ids, device) input_others = move_input_to_device(input_others, device) cache_device = device if self.low_gpu_mem_usage: cache_device = "cpu" - output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device, batch_dim) + output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device) if q_input is not None: input_ids = q_input.to(cache_device) @@ -839,7 +846,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch pick_samples = self.train_bs if len(input_ids.shape) == 3: - n_samples = input_ids.shape[batch_dim] + n_samples = input_ids.shape[self.input_dim] else: n_samples = input_ids.shape[0] // self.seqlen if self.sampler != "rand": @@ -856,12 +863,17 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch total_loss = 0 for _ in range(self.gradient_accumulate_steps): current_input_ids, current_input_others = sampling_inputs( - input_ids, input_others, indices, seqlen=self.seqlen, share_attention_flag=share_attention_flag + input_ids, + input_others, + indices, + seqlen=self.seqlen, + share_attention_mask_flag=self.share_attention_mask_flag, + input_dim=self.input_dim ) if len(input_ids.shape) == 3: - if batch_dim == 0: + if self.input_dim == 0: current_output = output[indices, :, :] - elif batch_dim == 1: + elif self.input_dim == 1: current_output = output[:, indices, :] else: current_output = output[:, :, indices] @@ -920,7 +932,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch unwrapper_block(block, best_v, best_min_scale, best_max_scale) if self.use_quant_input: q_outputs = self.get_block_outputs( - block, input_ids, input_others, self.train_bs, device, cache_device, batch_dim + block, input_ids, input_others, self.train_bs, device, cache_device ) return q_outputs, output @@ -1364,3 +1376,4 @@ def __init__( optimizer, **kwargs, ) + diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py index 6f58b6517..5b2d500e1 100644 --- a/auto_round/export/export_to_itrex/export.py +++ b/auto_round/export/export_to_itrex/export.py @@ -123,7 +123,7 @@ def pack_model( m = get_module(compressed_model, k) fp_weight = m.weight.data scale, zp = v["scale"], v["zp"] - convert_dtype = torch.float32 if fp_weight.device.type == "cpu" else scale_dtype + convert_dtype = scale_dtype if not isinstance(scale, torch.Tensor): scale = torch.tensor(scale, dtype=convert_dtype) zp = torch.tensor(zp, dtype=torch.int32) @@ -133,6 +133,7 @@ def pack_model( zp = zp.clone() scale = scale.to(dtype=convert_dtype) zp = zp.to(dtype=torch.int32) + int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size, fp_weight.device) int_weight = int_weight.type(torch.int32) new_module = WeightOnlyLinear( @@ -151,3 +152,4 @@ def pack_model( set_module(compressed_model, k, new_module) return compressed_model + diff --git a/auto_round/model_info.py b/auto_round/model_info.py index fd881e814..68ae14772 100644 --- a/auto_round/model_info.py +++ b/auto_round/model_info.py @@ -1 +1,29 @@ -SHARE_ATTENTION_LIST = ["Baichuan2-13B-Chat"] +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .utils import logger, torch + + +def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs): + if attention_mask is None or not isinstance(hidden_states, torch.Tensor): + return False + is_baichuan = bool(hasattr(model, 'config') and 'baichuan' in model.config.model_type) + return bool(attention_mask.shape[0] != hidden_states.shape[0] and is_baichuan) + + +def check_hidden_state_dim(model, positional_args): + is_chatglm = hasattr(model, "config") and "chatglm" in model.config.model_type + return int(is_chatglm and positional_args is not None) + + diff --git a/auto_round/utils.py b/auto_round/utils.py index 1c2012ca6..4efd1039d 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -22,11 +22,9 @@ import psutil import torch from torch.amp import autocast - -from .model_info import SHARE_ATTENTION_LIST - logger = logging.getLogger("autoround") logger.setLevel(logging.INFO) +logger.propagate = False fh = logging.StreamHandler() fh_formatter = logging.Formatter("%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s", "%Y-%m-%d %H:%M:%S") fh.setFormatter(fh_formatter) @@ -402,33 +400,7 @@ def collect_minmax_scale(block): return min_scales, max_scales -@torch.no_grad() -def get_batch_dim(input_others): - """Gets the batch dimension based on the input positional inputs. - - Args: - input_others: A dictionary containing input data. - - Returns: - dim: The batch dimension. - """ - dim = int(len(input_others["positional_inputs"]) > 0) - return dim - - -def is_share_attention_model(model): - model_name = None - if not hasattr(model, "config") or not hasattr(model.config, "_name_or_path"): - logger.warn("Unable to get model name via config, assumed to be a normal model.") - return True - model_name = model.config._name_or_path - for key in SHARE_ATTENTION_LIST: - if key in model_name: - return True - return False - - -def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_flag=False): +def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_mask_flag=False, input_dim=0): """Samples inputs based on the given indices and sequence length. Args: @@ -442,7 +414,7 @@ def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_fl current_input_others: The sampled other input data. """ if len(input_ids.shape) == 3: - if int(len(input_others["positional_inputs"]) > 0): + if bool(input_dim): current_input_ids = input_ids[:, indices, :] else: current_input_ids = input_ids[indices, :, :] @@ -451,10 +423,9 @@ def sampling_inputs(input_ids, input_others, indices, seqlen, share_attention_fl current_input_ids = input_ids.view(n_samples, seqlen, -1) current_input_ids = current_input_ids[indices, :, :] current_input_ids = current_input_ids.reshape(-1, input.shape[-1]) - current_input_others = {"positional_inputs": input_others["positional_inputs"]} for key in input_others.keys(): - if not share_attention_flag and ("attention_mask" in key or "alibi" in key): + if not share_attention_mask_flag and ("attention_mask" in key or "alibi" in key): current_input_others[key] = None if input_others[key] is not None: current_input_others[key] = input_others[key][indices, ...] @@ -618,3 +589,4 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 + From 62ee73c47c1cdc51a9c1ec7d3b5fb005247de9ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 30 Mar 2024 01:57:58 +0000 Subject: [PATCH 11/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/autoround.py | 33 ++++++++++----------- auto_round/export/export_to_itrex/export.py | 3 +- auto_round/model_info.py | 8 ++--- auto_round/utils.py | 2 +- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index eb7145a8b..2a3a1d7ac 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -18,6 +18,7 @@ import torch +from .model_info import check_hidden_state_dim, check_share_attention_mask from .utils import ( CpuInfo, block_forward, @@ -37,7 +38,6 @@ sampling_inputs, set_module, ) -from .model_info import check_share_attention_mask, check_hidden_state_dim if is_hpu_available: import habana_frameworks.torch.core as htcore # pylint: disable=E0401 @@ -611,7 +611,9 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de for i in range(0, self.n_samples, bs): end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) - tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, self.share_attention_mask_flag, self.input_dim) + tmp_input_ids, tmp_input_others = sampling_inputs( + input_ids, input_others, indices, self.seqlen, self.share_attention_mask_flag, self.input_dim + ) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device ) @@ -722,17 +724,16 @@ def get_forward_func(self, name): """ def forward(_, hidden_states, *positional_args, **kwargs): - """ - Rewrite forward function, process and collect input data. + """Rewrite forward function, process and collect input data. - Args: - hidden_states (torch.Tensor): The hidden states tensor. - *positional_args: Variable number of positional arguments. - **kwargs: Variable number of keyword arguments. + Args: + hidden_states (torch.Tensor): The hidden states tensor. + *positional_args: Variable number of positional arguments. + **kwargs: Variable number of keyword arguments. - Returns: - NotImplementedError: Getting the first layer of input and then raise NotImplementedError to saves run time. - """ + Returns: + NotImplementedError: Getting the first layer of input and then raise NotImplementedError to saves run time. + """ if self.share_attention_mask_flag is None: self.input_dim = check_hidden_state_dim(self.model, positional_args) self.share_attention_mask_flag = check_share_attention_mask(self.model, hidden_states, **kwargs) @@ -809,6 +810,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output) """ from torch.amp import autocast + if not self.low_gpu_mem_usage and input_ids.device != device: input_ids = move_input_to_device(input_ids, device) input_others = move_input_to_device(input_others, device) @@ -863,12 +865,12 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch total_loss = 0 for _ in range(self.gradient_accumulate_steps): current_input_ids, current_input_others = sampling_inputs( - input_ids, + input_ids, input_others, indices, seqlen=self.seqlen, share_attention_mask_flag=self.share_attention_mask_flag, - input_dim=self.input_dim + input_dim=self.input_dim, ) if len(input_ids.shape) == 3: if self.input_dim == 0: @@ -931,9 +933,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch unwrapper_block(block, best_v, best_min_scale, best_max_scale) if self.use_quant_input: - q_outputs = self.get_block_outputs( - block, input_ids, input_others, self.train_bs, device, cache_device - ) + q_outputs = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device) return q_outputs, output @@ -1376,4 +1376,3 @@ def __init__( optimizer, **kwargs, ) - diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py index 5b2d500e1..578f50ec8 100644 --- a/auto_round/export/export_to_itrex/export.py +++ b/auto_round/export/export_to_itrex/export.py @@ -133,7 +133,7 @@ def pack_model( zp = zp.clone() scale = scale.to(dtype=convert_dtype) zp = zp.to(dtype=torch.int32) - + int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size, fp_weight.device) int_weight = int_weight.type(torch.int32) new_module = WeightOnlyLinear( @@ -152,4 +152,3 @@ def pack_model( set_module(compressed_model, k, new_module) return compressed_model - diff --git a/auto_round/model_info.py b/auto_round/model_info.py index 68ae14772..55c64d2e8 100644 --- a/auto_round/model_info.py +++ b/auto_round/model_info.py @@ -18,12 +18,10 @@ def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs): if attention_mask is None or not isinstance(hidden_states, torch.Tensor): return False - is_baichuan = bool(hasattr(model, 'config') and 'baichuan' in model.config.model_type) + is_baichuan = bool(hasattr(model, "config") and "baichuan" in model.config.model_type) return bool(attention_mask.shape[0] != hidden_states.shape[0] and is_baichuan) - - + + def check_hidden_state_dim(model, positional_args): is_chatglm = hasattr(model, "config") and "chatglm" in model.config.model_type return int(is_chatglm and positional_args is not None) - - diff --git a/auto_round/utils.py b/auto_round/utils.py index 4efd1039d..0b3366ac7 100644 --- a/auto_round/utils.py +++ b/auto_round/utils.py @@ -22,6 +22,7 @@ import psutil import torch from torch.amp import autocast + logger = logging.getLogger("autoround") logger.setLevel(logging.INFO) logger.propagate = False @@ -589,4 +590,3 @@ def get_number_of_sockets(self) -> int: for line in proc.stdout: return int(line.decode("utf-8", errors="ignore").strip()) return 0 - From 0bf4812ad1200cfc55e154cfa7f3bce617374120 Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Mon, 1 Apr 2024 10:33:28 +0800 Subject: [PATCH 12/18] fix scan issue Signed-off-by: Zhang, Weiwei1 --- auto_round/autoround.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index eb7145a8b..ff4f8d2cb 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -611,7 +611,14 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de for i in range(0, self.n_samples, bs): end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) - tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen, self.share_attention_mask_flag, self.input_dim) + tmp_input_ids, tmp_input_others = sampling_inputs( + input_ids, + input_others, + indices, + self.seqlen, + self.share_attention_mask_flag, + self.input_dim + ) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device ) @@ -731,7 +738,7 @@ def forward(_, hidden_states, *positional_args, **kwargs): **kwargs: Variable number of keyword arguments. Returns: - NotImplementedError: Getting the first layer of input and then raise NotImplementedError to saves run time. + NotImplementedError: Getting the first layer inputs and then raise the error to save runtime. """ if self.share_attention_mask_flag is None: self.input_dim = check_hidden_state_dim(self.model, positional_args) @@ -832,7 +839,9 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch if self.enable_minmax_tuning: optimizer = self.optimizer( - [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0 + [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], + lr=self.lr, + weight_decay=0 ) else: optimizer = self.optimizer(round_params, lr=self.lr, weight_decay=0) From c30e7154dc319205370be7708318561a4cc7b3c8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 02:37:39 +0000 Subject: [PATCH 13/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto_round/autoround.py | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index ff4f8d2cb..69fba7e3c 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -18,6 +18,7 @@ import torch +from .model_info import check_hidden_state_dim, check_share_attention_mask from .utils import ( CpuInfo, block_forward, @@ -37,7 +38,6 @@ sampling_inputs, set_module, ) -from .model_info import check_share_attention_mask, check_hidden_state_dim if is_hpu_available: import habana_frameworks.torch.core as htcore # pylint: disable=E0401 @@ -612,12 +612,7 @@ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_de end_index = min(self.n_samples, i + bs) indices = torch.arange(i, end_index).to(torch.long) tmp_input_ids, tmp_input_others = sampling_inputs( - input_ids, - input_others, - indices, - self.seqlen, - self.share_attention_mask_flag, - self.input_dim + input_ids, input_others, indices, self.seqlen, self.share_attention_mask_flag, self.input_dim ) tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to( cache_device @@ -729,17 +724,16 @@ def get_forward_func(self, name): """ def forward(_, hidden_states, *positional_args, **kwargs): - """ - Rewrite forward function, process and collect input data. + """Rewrite forward function, process and collect input data. - Args: - hidden_states (torch.Tensor): The hidden states tensor. - *positional_args: Variable number of positional arguments. - **kwargs: Variable number of keyword arguments. + Args: + hidden_states (torch.Tensor): The hidden states tensor. + *positional_args: Variable number of positional arguments. + **kwargs: Variable number of keyword arguments. - Returns: - NotImplementedError: Getting the first layer inputs and then raise the error to save runtime. - """ + Returns: + NotImplementedError: Getting the first layer inputs and then raise the error to save runtime. + """ if self.share_attention_mask_flag is None: self.input_dim = check_hidden_state_dim(self.model, positional_args) self.share_attention_mask_flag = check_share_attention_mask(self.model, hidden_states, **kwargs) @@ -816,6 +810,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output) """ from torch.amp import autocast + if not self.low_gpu_mem_usage and input_ids.device != device: input_ids = move_input_to_device(input_ids, device) input_others = move_input_to_device(input_others, device) @@ -839,9 +834,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch if self.enable_minmax_tuning: optimizer = self.optimizer( - [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], - lr=self.lr, - weight_decay=0 + [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0 ) else: optimizer = self.optimizer(round_params, lr=self.lr, weight_decay=0) @@ -872,12 +865,12 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch total_loss = 0 for _ in range(self.gradient_accumulate_steps): current_input_ids, current_input_others = sampling_inputs( - input_ids, + input_ids, input_others, indices, seqlen=self.seqlen, share_attention_mask_flag=self.share_attention_mask_flag, - input_dim=self.input_dim + input_dim=self.input_dim, ) if len(input_ids.shape) == 3: if self.input_dim == 0: @@ -940,9 +933,7 @@ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch unwrapper_block(block, best_v, best_min_scale, best_max_scale) if self.use_quant_input: - q_outputs = self.get_block_outputs( - block, input_ids, input_others, self.train_bs, device, cache_device - ) + q_outputs = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device) return q_outputs, output @@ -1385,4 +1376,3 @@ def __init__( optimizer, **kwargs, ) - From 4db74b3149119269232b6cd757c9180ba6a00027 Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Tue, 2 Apr 2024 14:38:51 +0800 Subject: [PATCH 14/18] update handler Signed-off-by: Zhang, Weiwei1 --- auto_round/model_info.py | 27 ------------ auto_round/special_model_handler.py | 68 +++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 27 deletions(-) delete mode 100644 auto_round/model_info.py create mode 100644 auto_round/special_model_handler.py diff --git a/auto_round/model_info.py b/auto_round/model_info.py deleted file mode 100644 index 55c64d2e8..000000000 --- a/auto_round/model_info.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .utils import logger, torch - - -def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs): - if attention_mask is None or not isinstance(hidden_states, torch.Tensor): - return False - is_baichuan = bool(hasattr(model, "config") and "baichuan" in model.config.model_type) - return bool(attention_mask.shape[0] != hidden_states.shape[0] and is_baichuan) - - -def check_hidden_state_dim(model, positional_args): - is_chatglm = hasattr(model, "config") and "chatglm" in model.config.model_type - return int(is_chatglm and positional_args is not None) diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py new file mode 100644 index 000000000..8d8dd6384 --- /dev/null +++ b/auto_round/special_model_handler.py @@ -0,0 +1,68 @@ +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) 2023 Intel Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .utils import logger, torch + +share_attention_mask_tuple = ('baichuan',) +special_states_dim_tuple = ('chatglm',) + +def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs): + """Performs forward pass through the wrapped linear layer with quantized weights. + + Args: + hidden_states (torch.Tensor): The hidden states of the model. + attention_mask (torch.Tensor, optional): The attention mask tensor. Defaults to None. + **kwargs: Additional keyword arguments. + + Returns: + bool: True if attention mask is shared in the model, False otherwise. + """ + if attention_mask is None or not isinstance(hidden_states, torch.Tensor): + return False + is_special = False + for key in share_attention_mask_tuple: + if hasattr(model, "config") and key in model.config.model_type: + is_special = True + break + return bool(is_special and attention_mask.shape[0] != hidden_states.shape[0]) + + +def check_hidden_state_dim(model, positional_args): + """Checks the dimensionality of the hidden states. + + Args: + positional_args: The positional arguments. + + Returns: + int: 1 if the model type is 'chatglm' and positional arguments are not None, 0 otherwise. + """ + is_special = False + for key in special_states_dim_tuple: + if hasattr(model, "config") and key in model.config.model_type: + is_special = True + break + return int(is_special and positional_args is not None) From 93b0a5936bf6d918a8a035b9c43bdbda2b600845 Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Tue, 2 Apr 2024 14:58:02 +0800 Subject: [PATCH 15/18] fixtypo Signed-off-by: Zhang, Weiwei1 --- auto_round/autoround.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index 69fba7e3c..2ab008fdf 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -18,7 +18,7 @@ import torch -from .model_info import check_hidden_state_dim, check_share_attention_mask +from .special_model_handler import check_hidden_state_dim, check_share_attention_mask from .utils import ( CpuInfo, block_forward, From b6dbcec921c33d7592e75fc93e18273ddaa2e804 Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Tue, 2 Apr 2024 17:46:52 +0800 Subject: [PATCH 16/18] fix typos Signed-off-by: Zhang, Weiwei1 --- auto_round/special_model_handler.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py index 8d8dd6384..cc9799442 100644 --- a/auto_round/special_model_handler.py +++ b/auto_round/special_model_handler.py @@ -12,26 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Copyright (c) 2023 Intel Corporation -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from .utils import logger, torch share_attention_mask_tuple = ('baichuan',) special_states_dim_tuple = ('chatglm',) def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs): - """Performs forward pass through the wrapped linear layer with quantized weights. + """Checks if the attention mask states of the hidden states are shared in the model. Args: hidden_states (torch.Tensor): The hidden states of the model. @@ -66,3 +53,5 @@ def check_hidden_state_dim(model, positional_args): is_special = True break return int(is_special and positional_args is not None) + + From b9bd1039525c8309bedf822d92ebc053cf39a24b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 10:02:44 +0000 Subject: [PATCH 17/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 2 +- auto_round/special_model_handler.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5a98d1b46..e0d2a0a7e 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ Please run the quantization code first. ##Install the latest https://github.com/intel/intel-extension-for-transformers from source first. from intel_extension_for_transformers.transformers import AutoModelForCausalLM from transformers import AutoTokenizer - + quantized_model_path = "./tmp_autoround" model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True) diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py index cc9799442..82c1aaa18 100644 --- a/auto_round/special_model_handler.py +++ b/auto_round/special_model_handler.py @@ -14,8 +14,9 @@ from .utils import logger, torch -share_attention_mask_tuple = ('baichuan',) -special_states_dim_tuple = ('chatglm',) +share_attention_mask_tuple = ("baichuan",) +special_states_dim_tuple = ("chatglm",) + def check_share_attention_mask(model, hidden_states, attention_mask=None, **kwargs): """Checks if the attention mask states of the hidden states are shared in the model. @@ -53,5 +54,3 @@ def check_hidden_state_dim(model, positional_args): is_special = True break return int(is_special and positional_args is not None) - - From 86045780ec022d32a747467ca0eaa24bd20e29ab Mon Sep 17 00:00:00 2001 From: "Zhang, Weiwei1" Date: Tue, 2 Apr 2024 22:07:51 +0800 Subject: [PATCH 18/18] fixtypo2 Signed-off-by: Zhang, Weiwei1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e0d2a0a7e..9874745f4 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ AutoRound AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference. It's tailored for a wide range of models and consistently delivers noticeable improvements, often significantly outperforming SignRound with the cost of more tuning time for quantization. -our method adopts sign gradient descent to fine-tune rounding values and minmax values of weights in just 200 steps, which competes impressively against recent methods without introducing any additional inference overhead. The below image presents an overview of AutoRound. +Our method adopts sign gradient descent to fine-tune rounding values and minmax values of weights in just 200 steps, which competes impressively against recent methods without introducing any additional inference overhead. The below image presents an overview of AutoRound.