From 7a3b0f276b3ea48a94d459a854c0b21db57be313 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Fri, 24 May 2024 22:17:22 -0300 Subject: [PATCH 01/12] v1 distributed data parallel --- norch/nn/distributed.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 norch/nn/distributed.py diff --git a/norch/nn/distributed.py b/norch/nn/distributed.py new file mode 100644 index 0000000..a9a5de5 --- /dev/null +++ b/norch/nn/distributed.py @@ -0,0 +1,21 @@ +from .module import * +import norch.distributed as dist +import os + +class DistributedDataParallel(Module): + def __init__(self, module): + super().__init__() + + self.module = module + + def forward(self, *inputs, **kwargs): + return self.module(*inputs, **kwargs) + + def backward(self): + self.module.backward() + for module, name, parameter in self.parameters(): + dist.allreduce_sum_tensor(parameter) + + + + From 5283c1bc06ea081dbc8c6814a1488f31caadfb4d Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 25 May 2024 08:52:20 -0300 Subject: [PATCH 02/12] allreduce mean --- norch/csrc/distributed.cpp | 13 +++++++++++++ norch/distributed/distributed.py | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/norch/csrc/distributed.cpp b/norch/csrc/distributed.cpp index 6b25365..cf84b85 100644 --- a/norch/csrc/distributed.cpp +++ b/norch/csrc/distributed.cpp @@ -56,6 +56,19 @@ void allreduce_sum_tensor(Tensor* tensor) { cudaStreamDestroy(stream); } +void allreduce_mean_tensor(Tensor* tensor) { + cudaStream_t stream; + cudaStreamCreate(&stream); + + // Perform NCCL AllReduce operation to calculate the mean of all tensors across all processes + NCCL_CHECK(ncclAllReduce(tensor->data, tensor->data, tensor->size, ncclFloat, ncclSum, nccl_comm, stream)); + + tensor_div_scalar(tensor, tensor->data); + + cudaStreamSynchronize(stream); + cudaStreamDestroy(stream); +} + void end_process_group() { MPI_CHECK(MPI_Finalize()); NCCL_CHECK(ncclCommDestroy(nccl_comm)); diff --git a/norch/distributed/distributed.py b/norch/distributed/distributed.py index 9c797f8..f84a7fa 100644 --- a/norch/distributed/distributed.py +++ b/norch/distributed/distributed.py @@ -22,3 +22,10 @@ def allreduce_sum_tensor(tensor): Tensor._C.allreduce_sum_tensor.restype = None Tensor._C.allreduce_sum_tensor(tensor.tensor) + +def allreduce_mean_tensor(tensor): + + Tensor._C.allreduce_mean_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.allreduce_mean_tensor.restype = None + + Tensor._C.allreduce_mean_tensor(tensor.tensor) From 1e0c408ffb4b37d2cba8911742d2ebe6832b24e8 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 25 May 2024 08:57:35 -0300 Subject: [PATCH 03/12] fix import distributed nn --- norch/csrc/distributed.cpp | 4 ++-- norch/csrc/distributed.h | 1 + norch/nn/__init__.py | 3 ++- norch/nn/distributed.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/norch/csrc/distributed.cpp b/norch/csrc/distributed.cpp index cf84b85..5239eaa 100644 --- a/norch/csrc/distributed.cpp +++ b/norch/csrc/distributed.cpp @@ -63,8 +63,8 @@ void allreduce_mean_tensor(Tensor* tensor) { // Perform NCCL AllReduce operation to calculate the mean of all tensors across all processes NCCL_CHECK(ncclAllReduce(tensor->data, tensor->data, tensor->size, ncclFloat, ncclSum, nccl_comm, stream)); - tensor_div_scalar(tensor, tensor->data); - + tensor_div_scalar_cuda(tensor, world_size, tensor->data); + cudaStreamSynchronize(stream); cudaStreamDestroy(stream); } diff --git a/norch/csrc/distributed.h b/norch/csrc/distributed.h index 1992b8f..3a94983 100644 --- a/norch/csrc/distributed.h +++ b/norch/csrc/distributed.h @@ -24,6 +24,7 @@ extern "C" { void init_process_group(int rank, int world_size); void broadcast_tensor(Tensor* tensor); void allreduce_sum_tensor(Tensor* tensor); + void allreduce_mean_tensor(Tensor* tensor); } #endif /* DISTRIBUTED_H */ diff --git a/norch/nn/__init__.py b/norch/nn/__init__.py index 8f7997b..560eef8 100644 --- a/norch/nn/__init__.py +++ b/norch/nn/__init__.py @@ -1,4 +1,5 @@ from .modules import * from .activation import * from .loss import * -from .functional import * \ No newline at end of file +from .functional import * +from .distributed import * \ No newline at end of file diff --git a/norch/nn/distributed.py b/norch/nn/distributed.py index a9a5de5..7d5060b 100644 --- a/norch/nn/distributed.py +++ b/norch/nn/distributed.py @@ -14,7 +14,7 @@ class DistributedDataParallel(Module): def backward(self): self.module.backward() for module, name, parameter in self.parameters(): - dist.allreduce_sum_tensor(parameter) + dist.allreduce_mean_tensor(parameter) From c3c8fd7199f762491089063b30b2237ebc2183ef Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 25 May 2024 09:10:34 -0300 Subject: [PATCH 04/12] fix name module parallel --- norch/nn/__init__.py | 2 +- norch/nn/{distributed.py => parallel.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename norch/nn/{distributed.py => parallel.py} (100%) diff --git a/norch/nn/__init__.py b/norch/nn/__init__.py index 560eef8..c131ea4 100644 --- a/norch/nn/__init__.py +++ b/norch/nn/__init__.py @@ -2,4 +2,4 @@ from .modules import * from .activation import * from .loss import * from .functional import * -from .distributed import * \ No newline at end of file +from .parallel import * \ No newline at end of file diff --git a/norch/nn/distributed.py b/norch/nn/parallel.py similarity index 100% rename from norch/nn/distributed.py rename to norch/nn/parallel.py From 0e9c4779a8c689f6e77c9bb9f7ee2f0c9ec86266 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 25 May 2024 09:38:14 -0300 Subject: [PATCH 05/12] add argument src to broadcast distributed --- norch/csrc/distributed.cpp | 4 ++-- norch/csrc/distributed.h | 2 +- norch/distributed/distributed.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/norch/csrc/distributed.cpp b/norch/csrc/distributed.cpp index 5239eaa..bb753b2 100644 --- a/norch/csrc/distributed.cpp +++ b/norch/csrc/distributed.cpp @@ -35,12 +35,12 @@ void init_process_group(int env_rank, int env_world_size) { } -void broadcast_tensor(Tensor* tensor) { +void broadcast_tensor(Tensor* tensor, int src) { cudaStream_t stream; cudaStreamCreate(&stream); - NCCL_CHECK(ncclBroadcast(tensor->data, tensor->data, tensor->size * sizeof(float), ncclFloat, 0, nccl_comm, stream)); + NCCL_CHECK(ncclBroadcast(tensor->data, tensor->data, tensor->size * sizeof(float), ncclFloat, src, nccl_comm, stream)); cudaStreamSynchronize(stream); cudaStreamDestroy(stream); } diff --git a/norch/csrc/distributed.h b/norch/csrc/distributed.h index 3a94983..53e318e 100644 --- a/norch/csrc/distributed.h +++ b/norch/csrc/distributed.h @@ -22,7 +22,7 @@ extern "C" { void init_process_group(int rank, int world_size); - void broadcast_tensor(Tensor* tensor); + void broadcast_tensor(Tensor* tensor, int src); void allreduce_sum_tensor(Tensor* tensor); void allreduce_mean_tensor(Tensor* tensor); } diff --git a/norch/distributed/distributed.py b/norch/distributed/distributed.py index f84a7fa..bd87bc3 100644 --- a/norch/distributed/distributed.py +++ b/norch/distributed/distributed.py @@ -9,12 +9,12 @@ def init_process_group(rank, world_size, backend='nccl'): Tensor._C.init_process_group(rank, world_size) -def broadcast_tensor(tensor): +def broadcast_tensor(tensor, src=0): - Tensor._C.broadcast_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.broadcast_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int] Tensor._C.broadcast_tensor.restype = None - Tensor._C.broadcast_tensor(tensor.tensor) + Tensor._C.broadcast_tensor(tensor.tensor, src) def allreduce_sum_tensor(tensor): From a316dfd1917f0c4869a1d026727d6acb12519d6a Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 25 May 2024 09:38:42 -0300 Subject: [PATCH 06/12] add argument src to broadcast distributed --- build/distributed.o | Bin 187592 -> 188328 bytes norch/libtensor.so | Bin 401192 -> 401240 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/build/distributed.o b/build/distributed.o index f9c8535d762138a1e98a4f1a2fd7d25cba2dbefb..11a92efd3ddd3446c173d42c05df16f04f65edb9 100644 GIT binary patch delta 9883 zcmZWu3sh9q-k*Ka+ab}gOt8AhOq%#8wYXlpm155!H8CyGOqhcTC@SixNGh1GLA0<@ z|7bp#$r8&4>L_ZcU_QpO#4@9_mCRb*Yc0Fkje9@8yZTn&?|=3_!%iV&N}aJ?D*hqL&_Zf#EzZYmCZlzUUXF-KD4xcQ0eu( zgFB8F{%iO2^T)(LCsO_C@yYi!y369+Wvw0fD|7sIXOB8}*|UxA(hKg=Q~rYZks=MM zZvLzB-NXwRb@e}%FpKm{TI}B1gf2_&{5$)eZ5Ovxd%EqPwD0`CZ0vjQA2(qCiM?HS zIPM2l)!9F_I8lWCk8SF#UFqy^w=hxDll=cINz{^)%$Zxxa^=it&P0{J)8@`1y!Wxq zn|s_lCYFOc97E-Y<6t|7LvyqhTVwbB>-4zi!~hx|CpL?})30%2n)nBK+Kb-pKfXh% z&B8FePR=+H|L4R(4#!>a9&01%3Kg>t@HoG7^4+bUv_@hmoog@RTaS$b(@5MU;yV=s za5}o&rmT)QHCRP`nXWSFZnx0;$;0$1lXgVu1m(U&t3apy3$)A8q*=jxsDb_z3!ZG^&M0mz@Z%=l;}(1v4eEeub4>c4Tj+U`4x02I z(Pq@Ca_o`xMU&Q|?)GYduhBUKPiy@Q_n=23gGeTjgF(tC-$o(n(R;DrOJWGkwCOAB@ttUd+F*=~t$IFzH@V z+H(IdIXi-G|17pGx=PB5k3_!A;zd96qo$;zh@?L5$fh`bsp{9;SHvPYCg)X=0|6{{f*>HQD3gAw}b?{RBkIhhNOK`pB z=WNlfH$AaNBv3EEs1jlNzcnJh=Bi($3tCNpN3Idqs2QW0v1>(tK^rOnZYSZU73(mH ztq|qnGul`oexcfRB9`21M67nU6+P@19jSJ+h^+~)6USSNv$S%v=uKCFNX?g0BhJ_S zyjk3SyZF)+hzs;2D$u^RBJ7kAR%)QTU0o}z#*;-9xhC)_dIc&U~Ku#ZUfcy}+HJ zL6Eh*YNOZEd_bK@m0!B$MXdT4o^p_X3f#e+wpb~)gMJw9hRqIJjg8?TEhU5b6 zO9k7Mrrk|m4p9kI`?eMJyHo2ymocGchLeV1l=31*Sv>L%WhswBX&7DRQ8QynLm4x5 zJo56Wl1I&^&Kxn8^di!HrqX0#Lf&C47LS>1^at7LU9vujEk*uf}DJwBJ;=q~R>(VzV&9w$JK=abURiyjB-WKMmLV-I4H` z+`6h5>hY4+PutQ7Z;>4-t6sFCc`s>QMJEcrqz%MJ1Rpwo&%j?7`f-HTnJVwpTG5aZ z+C3B)sdXdY2rWU>QD7`qI5a|gNZe165!wLJD4~>*+Cx@=6dAxbQtPDuS;l-u#cYq5 zXGWsjYtq&X8#zZIzWZgw*C~+2JNPot0Ha|>lgQ}-+RUhDG;3c0ny$33AhF8QRb~ml z(vr9%<&|nl6f+8rL~@Nn#ktBj3gMc@a1tgjbZn0dKYRhB7R# zJ{sPz^rnpF0y(0O(d$YJR99EkKl(=Q!^gOb!f)3hLJE3kxYTfd_2eF zqIiIUF9T1TfDO)&xOW0NdO)5i)o^$MY(8bP(S6l{n+W;DCuwxPVfU1$w80=3QR@Zlw7E~$g1Y5b26z$|uy7TqW^8J(q4U?%6wZ*lZfj(j+<9ZZLqPL=^5 zQ2C~SE>P|%2$)R96a>T$nK=Mi4v%t7h2wyvQ>M1ilT<#?+LfskoQk_@cbRL9a!!M@ z4+Um(rPGkBI~miE%T0kfTpb{tU-s~qp&SKp6e>qS3%yFxNl)XJx}pHL)N%^U!RTq$JCW1XAgFUCzGqDuO(fU6`L2Abg3Svr4b6HRbL7y4qDr8F$5FaC1&_oe^E|5ZK z<|1PsS@3+yTfi5<)|%`jT6Oi@&y5A?FiG#{BBl}%-< zOrW(mF9qlGnk?Yb7Qm27&P80>0uElRb@N~Q1be#x2ewi&A&Vis7U^|++gs=Ak;?s9UL@$xJ|-HKjLKhEIevBbQ}O*#ZxJVB8qe3j27 z*HQ#~LIx{PT`onQ^tlvd>PboEQ1|x{C}x>?dk(DNJj-|+mT?!$Uj>?Ge}MF>`>D?dWpWV(ZPZBBLMf=!B65|QL5xzqrlG{t3SwOc%c1l^xty1=j0*t5 zkHxomSC^yxfimk-6}k+xg91#al%f4pGRowg)pP*TQ>9~xa;z}dQ#w}g@m$Gi7{$%f zGy&BINz+1QS_v9w%g|N1nBu;4MVM&xrgtO-A#*g=h_u4%z7Gw5^u+t{i)Z*E-W( z4)=4?ZEa0Ca%C-`U^#LAFikD}G<2b+w>rv~3gAX=|}&mE>HH z75A>imZni~E$Y7_p=LnTRDq-6Ce+qvAsK|QFUYX%W!PFmSOpcSeXqby;l}h1Z)645 zV6b#p7a-F(NE<(GMHbfH=?kLCFI)JLVZEq_wf#G#137Ci>a5S z>ks*ui`fJ!V=1{dff^wxwR!V4;Y;fX3T{HmKFcTqg&0Y{4%a%;^$}Vexi=$W)-p0S z!}_wcj*_*6nW}>alz5q3mGXeT%X?mlWl-v;l)4gq>g0J3bTjA}Mc<}&!yQ1hBp(?8 zBzi?AiYO~nVIM_6iBaSVqSn+>GJ;TMNTqdVf;ck@Lht5Qz6*0hDY@Q-di0IGzYDd3 z)EiWo)m6(WVpFRI+*?4^Ehl3Osu)ATEoirqoF8yMTd;3qcm?o9^7!mHr`wI1+nU_#va)f%+It zmuM)8>f3=4Q4u3(jn03DY65F zW>Ayq*xBOfsM74jS8b2XoGzj8P6WZjgaIlXDN7b_2z41|P9$cZ+Ex4s;U6y(50GMa2uu=r4s+QdBv$mplsA@*A(Wj?O|{%BXfd8#`>3xuvkjIn)*C*VtYVQ3=6jrwVE%YSHdxI10hZ~25>u3foCXlNJiQ>p}giEVw zai~XVxCQ}nQjW5N>4`F*`ts^N06Igt546zz*cBP9`5=NRx0EJ{A3)$Z77F=hY+aHPZ3P@Tp38cK)64GvK-2f z@V*x~jAaWmQUx&(ON<)?e-lcp#pQkt;#!2wqhKw<_FqjAOv!nK&)i{n(^r#m7~VXY zs92pCrb-}YYLcX$SJfT?6^82w!s2(LBM5sPN=(UF$6=2`ki3SBqb*+R>@$_Rh9aQE zWOCKPn*n7Vl#_4hG+YN|0hBgM-O%0jpnP!Ex45iLWvUV`P#pgl;vazW7?k-kzIsU3 z9>YU2pCYViSVOLlVaO-rV{V2h$kQnl;VFK?cO5sU#&Jy3*HchVBRs8MPp%VSr;>33 z(~Ng0D5nvgx;K#PBytzX+ecN(!1c2m$sS!qAvw7bdm31ZGB z;}cE`B~J?}!c+YKxjx0Th>TA$?Q@udavI@j`Vn%S0sA@`XD}@|NDmS&LD z#Hjl$|K$1=M!vIXAn7;-&$8n@hikAncLTWG<76~|Yhrac%IW98RUaqUIm_oe2Vc{d ztOCcsPd9STb0}y&xxU~KO!~fnG5G`qfq;lGVOHY)5=6!cGQI?XSABet^BXScJc!B@ z20kPaW}@K)MI_<+O7dS}9UUhrh$#ik02go<$M=d0AlxU(xL^@s5Gu#l2vvTP zTwlu=_)s~Rs5?m!Nw^vNs!MXqm6)c6+D z+*1_9R4q_G!w)^p5M_%V|Mh{->M7{|A$V>-zuz delta 9322 zcmZWu33OCdwyk@OT9M%gD3eM{Gbl#I6p<7K=BBmPL;x{KN3PZa1b?(5can^ITnpnp(SH<2a;{oEawZq&C2 zE`3z*ej0~f`eRuRM_)(3n{c&}<9`(H$ebWum|7V8%5Dy!w!@79c?$2@Dvb&42T*&qy=)R6yZm=6eyO=IC zqyKtC^kPMqo6*TPMDHSTyGXma0dz{RZ*h-Ft)}pf8-yJu-s=YZM-%TG$E_Lu#l#2R zfcKzHNSHq8F&@qBH_(}io@Ua&NNZ|4>&7BQ*O;`7M|aco5$c;N(nhtLv@=dyHC;CG zPI24{`@4yEjpIGET`~n=m+pR?YjwqGs~8V8F#U*0|0+&*(a}%S6{ZVKx=)<8idjV6 z(m<~_>3iZd4ubjFMp;bPne^Z|ZACZJT&6z*of4lV9g55MCN2JHUK-D*iSIKTf65W} zBsIDxiu$B2GZGKVfW@)`>2F7OC{}WL!p*z$JaMz9qoMam?m5kA^b6 zMAKHG3v{8YX%f>LZP7<<(OYN})4Of-NgI8P!c3pD(Wh+mpH=Rk{w;U5&V9|yjXPXm zibxxptZ#$^F@Oe}iA&>v9IZCIKlMde`Y=sf1G84(5gNzzQ?_XP3XC(8u3B!3<^PSfSh;-om! zxT8u;7j$W}a8u7}ah0}LiO!AgE#h%ON!0)sY!xG^^A@zCTSW!sZAEMOHqk-H(URwq zs8_W}6(2YLdY`C~js0pw#!ccJHP?yz#Cgt)N`NcOtH;?w>OBATf~>jh4NZOm&W$}Vz#7=W=MW)g=A#2a0zj#apDQl zThPPJsI~GW29jUzfQryEYs{h-imoq;#9k5ISM2wI*TjR+$!#( zYpo(fS$}6*U#A{@TDa}e%TKH5e&VkxKL%&o^%Fl`53qliGw|nJc8lUOi2`<AOLBZ=m=v$^TkNe2a-0{H^HR<*bdXq9Nbn0FYp@`gHE2 zt{k?7CW5AN4jXO5x6?+(j{_fAZo@godRN*1Z#ZiF`?unaVyb*b4yKmT(nXbH<>0C@ z+;|R9Cby!a(JDnto=>9B-Et6xN6Q}aha{?Uac7k5A)>VTF{rY}$PrZfG^8C#l-*qp zq!x52ItHfCuqj(*a?AHLo1!I8TS|tbfUCv=#d4eRndp#@%PZxU%ZFAAx1zwR#xcZX zCbxX3ok~rNTP@s*@_3$w;U&uu9rBH1CATWM)xxbPT2;?M*@jW}$H20l<-q_U7tcit zx1!HNZ2726 znL_e~gu`!@gZ~AYPFdNqKb5-Rwf_Z-$uor_tn8_B^;Jqwj&#e+Ddf$8XHPP6;F$v@ zx|VV%t>1<019FkQ49eUDb+Zw3s+`~Hawec~Gt55$qSzENCRkxHMjTT~<^d78F!JXC$(Tw85N9uCG=h?kbN)P3 zI}+lF5NGAFcp}6l5Hk`O4GM_aQ*(YDR#c&E_%N|sj zFNZkw!R<@QO`JGirirnNb`{7W6gGnjVB?=gMggX|H$`w!Zk|bSt5$N~B)glr6+%qc zD>@2Vxp@lV);5j2g|G=1pr}wGR-liXfrr&p##nR>l}~0-K36^&W**AQ;mRk&EQ>>9 zjAl}pQP&hsVQQxUoujpkDlMaBT050dQ#%!Clh!gCC}iKMya%VE<(UTlyo&No;|!C5 zh8T5_n25YlMwNCt&4-UqxT=gUFQI$%+bnuP}V~^KS6CWsj4S&+bfzN7b2gg@)hM@2&!oo z8KA@R>S*@T^0W4(Eax7#U1?F>Ypeei*RN}<3)E@=hOtYw^F1<*_q zj8+20y(Y54Ty!N|3HJ=;UZO|75)oy}r_x3j>v1sMq|>b8x+ze? zC6#hXt5{nKbfiKnfjWPq^6$|JK+BT3Bm;b3D3$$#DQB?<0(hi%%q7D@DHeck_&kyZ zbXfs5Hh@&mBf~@tATdU+L9-gW^mpW4t+xA0o{ZJVJWORS)R|YKurd-%yVfM=-|LB4 z0~#Y=!O=Ea?No{}eO}S7wV3G#$h($L#e6c>;#53v{V3O(lOO_|YaLI`S{}hVpxD-o zR>sj7qu7fEYhPyVI@Z1n)UVJ=pw8c`1XX%4pk>=al4Z!PsSVV=koQj+-W5_7kpW6P zM3FMJA6B#ddKf*VjOOTJuSXv22c|2Tepu1dDY=gCD(hj^w20&eK?~E8z-$A}P zLFs3Xe>13*oPf}zv2H_!5!Gowm8BeM*9@K-Vf_e_rRWQ<&bsm|jT~3lrBQK%C zCMeN$KBY_s>33(s7LGw#ouK|k$>aH`RD*J?AcLuqDyqAB#$2kFcU#bG{gq)0=B$In z>nPT@1-F2aWNbnGcu)iZMFBZSDU&KneiQx!w!+`PlDu2tuc>azfBRPWYiOz709!Gn zYb!}^gZe*}k#$P9fy!A$1}G9+`K&QgyNcv?IU@NFN_?xeQ?`S1l#&6?Ia+bJIq;XU z7$`h!v$pVr@4y(wkaq{Bt^?{FP>)gS*R^^F)HpiZSiKX&ds@+N>E+o8x?0hdHu`{$ zX8M|ru0gJ6$XkP4IR+Uu$Te1(1+}^cYTRJWT(!uh>5J-%NKq}O(!H9DT11Xhk-t!T zG4ExjDpr%+1?pKv=~c|zg)gVilCcYE{c9)!g?NrG_+%<|WXlxtyrD+0lb7@jB=xEA z7gS9l(DEAM-awp>shl8RLOcWY}9le z#Z(AUZzJUd^1h9fSD{4LnWs!}zTV}1@;3I#@C_vQCn)QD?T1PA1~QnMcwOokD9l@U zFQ0(}n3##=J%Cf6Lg5ggNd{qYS$Q!u$LnywWoMb&J)Rg|w!(R`tl5#RYiF}IGqq+(T?BjwP z5=N=2I)4McTNfzru?h+`Aa#pM9Yq9%51}B>JFsI3YdOJth{h&p038J_@O#)X%-Ki<`Hm$b^m8zD42C7jFv|2) zl}~?n^}G+dLPx)!pogiuYTf(Dhn44YqM!wSK01zk9Tesge8&+nO&RJ(a_Be=`%u21tEv5)}hgXsO!0dWIK(~y@P~8Z3V??qI z>@4!Op<7W+hU!MRTe^j0I}X)sl^egr@~dc3I}X*@8ZvCuK($`6cD{Yoko*vyv&s7* zOzNTJ?mY7HuXTU<5YbJ0NPeWSkGP6GWT{Q#n9!G3H`nl2I$pA4MEgI1o{A84-$r6L@j{ zOob!aXCMk9WB>sXVM1m497IKgyq|->`!p^Th;lJZbVNvg!5&gUNVhq>>6ng;b-v5#zHNksN*4e!5xdcDemUDTi{N>-4Zvul7Ve- zr{Ye--2rz;lBv^mMd2gdJ#hEN&8|Mc4BP{74-|ESfkSW)#r>J68wMOM$|HcIMR_c6 zq9|tpr{SJ~dnWF$aL>h^gZt}9{8)s0F>V>RdQp^@;r<6j^OM<7_KQ zf6E>H4RW!|dg+z?td?QhR@Wj)%Gm9qzUpOzXMWRuzo7lmJUxv2Jf>6}7bQzt^YBcH7kE-jFyK;P6~Sy12#5WzXBDatx0-A^98K18^D zs2+id`X3te`u3vzX&2tU-AQ?9VCw|?bG*KT=yuSRlkM|(eFIUyp#^8>!NLnygsA^g zeirEK($1OJHWIZ|_OII{)B7yvF-*l}?SB&W3n84DW?PlF z_ZRg;oq2uFzbon(6vpdItd#bdy2hdd%P-6qTiL|KRl}MSZh7$z9xQ9%O5)YJLZIS1 z%xHG`aI&sE7IRtWsmpB(=i-7M%GISMZ@+d5Z_k}0cMJ0B^kqCJm5HgS`pw^t*Jrfi zT|9+|nym#ZHidBw6!rI8@%sH8jPgKL2D6Y0l~Te6GgI$Wv?_tfL{#sBhp zS)uB5pL_87o?`e_PrbwlGM9T=4AK83aPieyw$LHX!4t~$stPAK#&L!GguZbwlreG; z33HTaZ(EJmpE}MPBCM2avT$L{mzSqQ#XgA3SI zz$U@k1=l%M!6$95&<)=>PDJD>KM_wN#>bKY7?K6sHQm=JpU$)rbQoD^5>)b*D)P-45BYh$u!jGz^$C}1zLUrw<4D&hO_G+x}6hKI@g-lV-S_gAhKA#NCE2TtYo7DbH=#y_Ub67IP>~eyuEA=4ffL6 z^EP7y&p%VRE7FF_iS@(0dW)pcP>_2H!^e8ab3vZNb_=*v3oeU^8`?)cS3iu*ER#uC z>xLl~ZrFoxWcs;1;=R8V%vT$EeeX;1JW!E`boyN&QoE*)zD!87CZmrvJm8-DD z8rMIZtlM6fw|5ZjRX4=O^Y+bq@%FuxUOL@&L?crxi|28BU4KCclpBPHOONIL;fBn~ z?Re*LY~{zLrY5Tx-uZ;^d1KMHmKAS*Lu_@_tQ##N>ucd)*rr?tkunyueHJct*+}H6 z?_=HZ*_;5Qm8+W=>;Q|wPKE}GH7>k8^i!@mVyc&jsgBg5Tpqr>efOJk&xpXx(P>=o zShlOdrP|rWme<#3TQOX!J+1#o2I(rCTE_N>xYU@A#1hTa3oPZMHCLhPbh8XfeWq@+ z7{;N3@OJ_id{k`H)YQ>dQ@W{sb2QdCK*NSaTxSei4_t=B)AO?OoXh zSr@9)smb94_0vujo5J&gjTuJVmW zzFt4DgAgutJs!&^mu~zV*();4s~I~x;hL!Y)am|)wv5Nns`7xyzC8vwauEXrc@Bbw zk-NR)-P902CZ1uV1d9E6`x8pIf0&me#k?#LOR!n+*~s&f zmCoBvej62*`LbC>qoPC_I>pX_xYR&7FX~rYq^C!bT)vIS3#z#!h1(Z4i1Yb^tF7yt z!0UfzCpH|b)-?_$-OA;2jaz39W6L)#b)9G>X1)ZyII_!7Y-wLc_%#@HuCvD-n@hp2_)4QtB}|{)*#_0+w)snrprgq19f5 zmKxeG)!G9`zNc9(CD>sLtyL zTkMcs#3mz7n4vDS1~D8D#Mz?1P(wnnF#emxo=xSy&R+3K4Q%PTl^um!uBCE!l}(f< z$A&dr{gT&D7W_JhWo9CKUxRCtsIT*WS(FQ6!%`QEK~sGciqa=&IS4K4SCd2XC%gjki;zIkG?J9S+C zp;=hw^J`qBnmWdb&4Fh-ZoZnbXNyT>cbK`%&7Z3-5>{=`wen^}Yhw*CNk(!); zI}1-!=Al7gG}IzXd&LXOdT;@%C%zZ+AZ-G#SC6{Gja)$4Mec%rLh!{9UZ40zUJ)1O z`n4VJHb#)EHI)nEd@^%AdG#4l9UjAb>BMeJU7JUXxEUi(>|m~3{e{^xh1qKTc9G0J z6(@OhnT`qI{Nwn1jD5dOo)+oJhrN5nRYeTw2(j2K6pItaQMn@JUt9QkeJQ3)V^Ld0 zWUo+z{Gde``RSeP6(8nenX~-l9`U{n=ZR487kxv7gRR+nPF(8LzazFcKbhrU;#+&Y zy1_LYAi7N#>rvnZ**+nxp+g(qTb(&?k#SgjEf#0JX`@tb=0pfHul2_{- zMSZE51%pI=b9aS*BVFJb&S8om{D$)yGi#aPtTH<8&+8|fGms>4#KT-*Q0 zQZ`-edX|Z;hU)vaiQJBNqh+s_VVNt$3xm0W-7>M1{?@soTPBFZWq~+cW{CFQB6DW& z%=!M+owdmNW#VjgP`K-Fuw|3|U_FKZP@TF=#EC&aB{mz~g#f)h?~u}-E2Kujl2*Ll zUYsV?Nu4QHufTt~#N&cblH8zGlgzo*IhROLd$1aJ?h%WWQ6iFOv^bMv$|%=YVr||i zvaFgKj*2tE9C4^o7sdV}*>2~_w%+^A@(3Z=GFJ^1Q)c%#*)=IFbEOz<{qr<#-vZ(8pPKOY`-RoAv_;>Ect{G4n|ZbF*n$(@qJy!Oo@-%c&u8S==)6|dTo^8YW zhd$%VVsE2dD-i`ux2>hTezB;}!AxcKzgQf>JVlJX6CZwJUMkneVjEarzTY-HbFNqk zKNHjz6{jQisQj`OSvTDx>n7T8rHh+z&-WJ|wi8pcv3R4UhHzssz+#Haw4J~7vAnyT zZ^IdNIR|w!wgFplX4gV-_)sTSZ80B9akNZ}%RDWXkJepzci~js8sYLoB3!YWC|8%( zT;Uk8HB;OFBl4DwIQy%M__69-4a--JPJF{(Ll>RTO$J}(iWYmhGZ*EXX-< zh?9vrBLC68j~8KcSuxs%vBsn(V8lc%?Cj98r2S7B|1wvD0fiw;StC&W=!jju#8 z(n1fp`i7Om$!-~lsR)V~oUUu-c|BIIcueS;&#CSFYgN1Jk) z#NurKmwddVZ)!#)*HAq&g+Vy;ualSsU4+bVEWfONv3PSeRMaoG;+B?0$PGK;(+hF> zRlQpxhTTo5s%D2*BJbIXgZ)y$&QXXNA;hQ-JsHo%==*bZ)y+UF7p|5#;brQuQY+Wj zft=7&B*hM*zDHeNKT+(CRA2ufX8AJl@rXJ*s*2>7{1+dCyMoVRv6!1GEYy(h`aaw6 z4ywg5FoikL^&zjHA{Zr!SvJgR+ltOhV%_@>8#ZRpz;PcB>Nk8){}KJh4jS?K$T8dM zf4#{l=dW|fY~P-L_Uwo5>4B55l0PG_eZA?n6?eTtL5 zUN`A|dr4PC(EQLo;K~aum)1%h9uK@{j?iGuZnI~1%_4{hE}>LL1PRip4|ck zZFLJo-9KAC!w$LQw=5iXFFtE0S){Ol!3-^(?B zNj8eCrcjoNmdvB={5t)*_-xloBwJsCTckNX*eXx!!4-CG#nD;RiKKK zv|`t>79aSxk@GC5%UWrv<-Z!FHnRI~3yeoK((^SOj>`LfGgP3i*Kjy0KenKnG=94y z*W6|>iZ7`sy2Wekx+AAqP)!=U?rfX7t%_t6-vUuOmul4NEw9>cs6aiYQLDF{XF;`% z`(Sowxg1on)GC}aG-`F0>+h&YwN2Lc?E|WKeXaDhjsHNMEYG)~=4-T?Ebp_RdTO+? zk{^R=r5T1%8m+A4nma2}JvCZcZA;q;XNmW3idsP$wQkD8e>YU1mTJ_xDX;q7Al=-y znZ11P={zN>$$Cb z4=i)nbe^bD%U!Oy*HD3)qEX9TPTOmc+~q-g*GZdY$9)FlW(}WWjS`#X<`&dt8YMQ% z!$B2a4N$zhOrygKd6flqh(?DOa^60J^kUnceN`kWU4FdZU`*F2;oS0r7t`gM2P#r` zX_QEp(?I=A(|4CfiQnX@7SuT!C4Q4PSx}E@#^<QLBnP--2q=sI^Ys2kJVFTJajS*2#}8s3whC>*Sh$ zE|3PxoBlKy`KN%2YPxg{hrx281+`SeVemG`zg|m5@zJHy+cV+AST&Pt9yU~iNzLkfA`mKiJLX9@x%Bw7>(Hd>O-IjL*qkc%!IYr}~L-OOJ zh6+?qO=dqN*F08{+FheoJvj|j{t}Uo*;b8O_2j7*)b1L!>dBjqtuu;Eu+n*=hO>)2 z{P+ZEr+nvlxN(?9^IQ%8VRHQwh6>an8oS)&KA?(|tHM88W0#ve|HM#q4AHb1CpQOGoL;yKG}-!rJp80VdLYj_xlXDh*UvK; z>uC7Q(eSAw_sN@J9Ic_|Yizk8Kh6u6T;-al!lhCi=n|=^E-%+3|F^)Ix54+F)#7^OL;lv_U!`?>N0cx+^E0Ibh^p zq!U8hdzBWHMsXfdlvt=~Qz)+jwZ4W!zJ~uuIqzIQ+2Ndn9Q}`jaf7DSdX3#1bwd%0rv^8cl|1+T54R(Po}p|DwStK6g_1%+a*@Rqk`K zpY(3qzKc~PdGAFB`OKy5#w{9#CQaj)@~Zq~qc}J!tl~9J`&};2H%LjcJAS8OirnY2 z!RV{uvrE%*yF4FlrpXsBPmrSIw1NfFCb_U6TskN_URfs%kat|^XB1!3QKXz{&ke^R zPxqr-|7x=Ii9GIVxa1|Tz1q(hp^@U(;X})hrB37;9^&#TsgBdH(eY#+e#w zL^{{63gk4GYZiq|1Ld@$OGfc!2}SxsO`9CK`HcnAzjEG<36d-)-3*tE^0=GH(rS6% z%>|NKPP-K@U6ZHYnjkfjU*5We@6YGmHb_mk-MQ^68U3|_JADwmR<3#1fb(C}-35}Z zeE4p%^t0@E@2NCOF2C0gzbDYglq~%zuf(D<8>9?5>Ulq- zcvYe}FGJIEy1eOmvXmgd#LsBC`HOy%US9RWAl;I8yqF->*jB&9SCTyBw3o?JE%}?5 z{iGE6<;w#`@eNFcUG7I81p95vdzC3kE9JCOgE2;-rIPeSb}tW?=(ePC)Md(P@19Bya&vP(X_vgp zoGf+OcE=27Wz#50>PMbSzke@Du}(XkrH{7JtscbPP@Ix|)YNpW*_@J-R-BTf-r_6X{+c7QsU_y53ZWeH?`HaD6KI4#ccG9C6Qy=NZM&=|vLiNVc zsLnpfzDlo4nIC1m&7gUEz%J$oQ@cTC^8vOZn5K_m@{$zUvqyd?KLF!}Ae0VFX=Gg| zi259oY@`dc&LJt(q5J2oouXqN?Q=+q>iN|tsHv>RnNg^Q8tknX)M$Q7QDY2%t1)Vz zs>V-DVWGw^BR-U!sPiGojl3C20}n~{9H%JSDw3aN)4qqK$jlA@?ST?z3q;3r(fw~K zqL%`==p`SkqE9d-bCU8?a-8AxF4Uv0W z&3p@$kN@?en$O#=DcN^0P|l`%L(S&wq|zBvOLb#I;Ya9#&uy6-j^9vpDFSd^$_A*q zG*ooaUzua2OUY3NIQ~<{rfkO4GG%bfppBz2EtRvId_(8OMt945ew*I&yET$7KP&~; z)_r58Gu2|lHs5{T5-^`1^_W49-UIeB6UTo>?;e)2$dchSX`Q3*%zFeZ;{mRCVlTp1 z6FJ~+QG>azhUxEh)h|6M)gWI@=e`&+c$_CauGK)+rBm>Yo|S9} z`;XQSKIS&;IW2_;-%~WGAYnUFo}~?=ls?{cYl&n{H=dE|1zUYqNm%@UbSxWyFA&F) zqJgyNo#bx)GiDfl_Ec&}s`sQW&cuyAtw#JvVJ~{tk$BLlPNXWaA4nrENnW(?In2{4 zy{Y>-DVTU?(D-vw6j|GgPGHX=J=n96{&7z7CGoxJ)pJr!vgjj4iq(S*Gp*N`)SzMi zNKNby_kyO#{0ooMgNRw`eM7Q!D$W^q6-E}Zsa|s^-SUs*Q6&libo{S{G^{W2rlxH@^aqC-Er>AW*jay1hhlr3^&Y*=CrRW;t5K%Ld zN~DLmDatD0z~X|r>}p>&MoA>8t7a4z^y3eX?xZG}H;5KpLP9DWM62aXL1bGOI{LEY zWxWo5q2Ie>F4>9D~amMhksSR257MPS9c7Zd z^v9stahi8T@~7P|BToL!;Os}Dp0W3&^-Zulv@=T{0k*dNu~%SY>(WZ9MawTsK4d{U ztyv&>`i$#jHs>T=ot0bd8!V`4Y&YRh2cMy_>Hc#M`m_k*uN{$`>3l0<=bTouhxsTa z`xJ9SjdU)xLk}UVFk_}y`%T+0XM0ez!27N?Z6iNpA^;P)qd^)e3)=pD7N>ujg%qrVU zRPS*qQl0i3GM)Lf7h-PG;)Tg;b}ftcWB1{04#n@{u;&>6#moH;HRQ_uqhvXSC%4 zvpdqvTX4b{HW4kHu(~5Rq%f6!e_INsU*3{DJWeXm9&f~2WoXsY8kwp^{coLo{eW!6?4}7b}VN#CAu^v<&v0o6=zCS znpuwdm)n8o{rhbzOv~9F`Lr%PBDt`6C0bwYs!VP&sx8|Q7}d$tw>6D;E_u+CuEdV! znIwbnPpz2EM{~O7i!|OQJ3squO7>gkfi{TN>WGuD7g9C4<&IQ^Mm)qEZuvlJLe8~j z^RBhaiI$83^Qq#rKU&hW_mQktc3S#CYDtqGLRyapn72hOY1Sht(ZiE5KANa@I>sJ% zCDQW`;JK=a%$==Vl0a^geMsfmIFWXHh&klloQph^p!QZo?nt1Q82OJjRC>gEgRJeH zzj(hg^HfK+ni~35MgyDC1)ItF-b*f z**7p|OziR`*2kSTJ1RNyq#YK^qQiq0S0ldUXf&JSq2x^}TUxLEw zdeQmMrMhlQF@bWDZqLdc>!LKyai7_X9(yh|@;{)=f5nw0Y+x-?T~R#set}VSY604Z z0P>CHBldAT^L`Lr@C1`3{e|S-1R@2e;G%=e2F7n&uIGRODQW z_9MTa!XBdKYbiEjQ){K4_N?*XKTPSSI4H?m7h>~zZ-)8f~XS8xRPMv+GJ zrU*rJg~k4OQ+l{m3gdl;ioQ-P5#@B%_O0S5_PDzRtlX6ZYOr8#IA$3*Ok64Dp$mTcx<~-@{?o@eN z{vAmZB;v0B5%c^w^}j1s=SeFD5_!@xvv5Z)oX@hZlC;udc+%<$ibz^s(8lE6kR`3^ z^sPi{Hyoyfp8ihT{~YNj!ImRk)|QX-v?%%oApvAlTRzf_lp#|xc90Tl75AKO4XBBb z7-9_J$EiHFKT-zbtNMHmoNPtvlKU~r@L45BeApNJk&}UJO=>T#q{UTxW%Lc;6R=6>f9`nqM=Inh z_v-PN38U;tLU0Qfz7-YP>swi&iS?L5zVs_Q5)!;mX;49{)c@#XqwUkyuaca6pUQHU zglOahE4i(LoWcLmCm@6t*pb>~WrL;m#GTY{Dc(Vf;b8OsT3z-+t(AXPA|?o)WST(!A`M9mWpIHRv;YQiFUMK%J@*gPl9ZZy_>sG)*s;8k25;{8%tKNI4ep zO=3`8zQY^oK)lHge>&fRc#;~S^jTF*{Sx?Fyd4O2Ag<(U9nN7*UE14$gcGwLZ+|#Y zX;0$n()|wPZ!d}Y4;g{&_vWYAvQH}hOvyfN;wrn-U5@yw?}l)`BP|SOv$1LT8LvuC z&-;%cUC!&2?WK4-;Tp~+s|Q`CZM|x6T{l-A>>A#c1LUh(N}YBq9AAUE#&RoMP?!GZ zOyc||W4`b+gC`sL9L7`GY$`|ADc;oC1$%g>P#Wh#!iY}@dpG1kBix9a{v7h=LYAYv z*kP?I4XjSwoVP2|-k;Cb($wlmd8=x$BZ!gqE|+YTw`IX@it-hL_n>=ad#3u+OjpvF z+zFt+yOR3uHp+5x5U+hwvJ2P>aN4xZv*M|dE8>#Q23N5}IkhEt(YDn|AjuBkn#`+C z{M_RLS(^&n;BTBko0F9ybc=jh=(;HulR!7+jkbEc4e;jP$bvV>_(09W3MIzHLFi+5 zz8olT*Zk`8w`;$8(_Zc*gba$|FUx)Eynk8#t~&Pv8;=3rOguhr2#)6Mh~a&^=X|V7 zSvwoG-ODDt-A^7$yAaT3i#*NR{et1K{78Vk7}UqKqThOu@Wco>7{`^Er1Mk4a?);N ze>A=cl0uFyr|WrqvU6||dxdtCf3{Q`vBN*45LcyGP;QM@( z>_uG;;RQ{iH{D|(b;t%Edc#2KwTM&X;B1U-8yKDD*$j`YClh;cAybGQ4VmpnO=-;I zLTrK$ZR<(={4 z6sZ+g?0_Y>mk0ZV?dpuFS9N295VVjrL}aS@&DSkZ`5khCBh9!>Z!tcmLH=gymq=<< z`3NIt>f}YQ)FY!kM=Db@=}b=2kJ!kGom)?DGu;{o@ph5)*8!5d%U@sPY_}GX!3U;l zjo(Lmk!6xqM+Y-)9)`V)w~_V_BVJa{sQ5Ar=k#ly^gC8@uBw^tWff~YX<-=Sw;ks7P_$(3T9%`lw*&3Dc@GUR zs2d-iUCi^TUfk5G%@9GRCUDpBzv#3;(v^f7sGkpMP$L+fbH;7Nmaj+&q&;dhIoors zCf20w>O$=%wsd%1(u90%N1ym0+mEy3^g?eM?n^vV{;vv)(Oua|8}IjLuI}v2l;1mJ z>Rv_pz0f8@$r()f@3K|p_u8t;KSzEt^{GJ@_>$A)ksHnMBSGX(H#*l33t1OiIx3hn zBHvY^33VWSb`>uDi5K@PN| z*dhd{sV$F2C6%>+mz3Fnu8mNBSetz~%)A)|{nc`?rd-pD%`1zonw=|cyB|4uR3LFA z{j1Z=K=KjEcjaEc0CoS<>wERwuG&WK^%>UO>#=(7b$>z6_T*lFYE{we=Bkyv?!fF( z%x^nOHGhb;YW`{@5|f>QCI^vObVzNIN|re*uD2h70en%-S^qYhrUqk!Q*6bDps5uf zf&eS7>Rb;#1h35RSlIfR3hgQl!2^~76di(Rs5-PJst!{y-%a!>A+QQJzoR4SVv)S4 z<72Q-#~akahmkDuzkTzP4{{r9p|`>l1V{jrX)f2}JUgVQ9^fxF)x z368mc#Vp1BR`a07=wQ;&>e9P+^qL>?#{PHoStL?operAM>E&VoCLy{k=D=*Ym@9A? zzir0T>`k=idBYP?W14~ShSQY+m;jB`0r(8%4+Ah9mZ$^JUsLYt@*i^`!Hzy#h*RQ% zNRmtjmD9c{n7ZxCx!T*Ec_!cT_MO=jf`GKhxys}5>$+TJxlC30ID*U+iPb^L zzD2be0 ze?zB+;iK41HcB2R-@l=Mu;-CBN*X7_-_UB|FssWOZq{B0p2e+SE2H6Ts)2E~$l^hY zhAa_&GD$UUW~pl09t@wU4u%9Hpk#5QtuV7HTl8$KHBJpFA^Vms|- z%gyTh9Ky2ytt4zITWplsd||9A?D-2-SP&A6Nw(p_Y-hm*rYy|esga}y`SAreE9V6_ z>xB&;uCQm2^eHBSg<0bOgjk|^y|Ox1U;6W7$Ym#t!JjWjw?{nz~6`M#r~wLk#If#y;T#((4Hy zkiR58^s64CaWn$RLPQc<$#u-!&O0-_}Q7KERM@J;$ z*c0`LOY?rjrOhgq9x8&YhG22>iXs8ctDDg58Y0O;5*r0BDdaO%QZlsSlBObXOr|m}selE= zg_rN>)p}%<;W^8vmRY^bVgdpxh(-s6^ydi#!c3@1@(6K_Ki zwmm^V(^qSk0bpuVmKnz$2n}b=wg&qA^6?Pkn@lf$%8Euwu-~_hZjd`L-!#kAL_yWzo zB+{SszNuu7o`<0KtlV)%IQ|$rY^UY3^LEa-T(+(M>N+|n-mfIT1BQ#s|9XHbe*<*o z@^2uqnA(-lXUU|xRr2ExqO*z14W6?ml~6gE_&2}H1U$P}Nx(b!iwkJpUlq`^NENUU z;br>wIT!Hiq1KZ=MckA6BJRoIFSsY$T|<|9h&KyQmcUnBP@@c0P&$<2mdr*tnVvmU zqNBZ{-yjCi!zmNtH>DmJdKs@UQ}Rk33%TQy?3vyBjc+fbY3 zf8~*Xp~&q4%}*tbY@f1lcwo|c$+eJs>CIKHO#fmLZ&%Rt3ZmA+OD4uzNrH9zsLHg0 z;@nG<5g?{JPxu_V1iek?XhIt1(Az6q%#ABtOq-`%jHv)^{zAN3h&cmyb1~I=t71Z- z5*IUo<;}duA}_EQ+jCcI)Xf4e?RWu~*5C=3HtI5(vgovswu_CDV$|!Nsx&uf#-*hq zC`~&aDbiBcv4HylgY=(ous(IL(k<-s;fvDqoA*PF>6{sBF1{a&GMh`E!(R*g&>MEd z!w!!#VJm;*;6~51CqwLF-x0LaZ>UNWI*@qM<{>?piH+sDa+=qHL`94$XXK-%{fT&k zgdF?OK|O0+87DEEmNIq~GF!TkA1=}PJ;_MB z?AHqSw^rPMiMO}`uU^sBAA$LSSKM)r@nINT_Sf&1`sW@60lp0pzd3Us>tjqL9FH)TxzRY!BaEt4{|GC20pyRsFy3Td4_na>3 z1>H-YaoykIPqq1Jgtg8hBL`E)(HbqF%UZ{m3weg&tZq+RDLOZxF7oirBjMK@gFX3#r| z8PPAekP+v&8P<2``XLzOyxa8p5aQ`>^-pD1PigK@e3MJjxmzw{=T7GjAj9nbx~1@( z_l)!01JALziMU034~1*q+~lqa$G3AUD%|00WhqVZb#In6C+uH3oE*lt-1LW){`%EV zX_Zk3%R{GVn^DM^4F%g#f%Mx^@XNn9xR{tgOEEo8R~D0Wl!cQwEgpeyr4|jt#v$|s z1QyuMxvm(K{g_KzcT$nY_8KEcz?l8lxD@Z&mQtGKRhANT_`Q@c;&7PyjU?CXN(vPT z!yj@9Gfr>`{R^o)3KBvT@=;T%zoi7vzbi|y{!1fa$Vu92G?{I;@QNbg=lgs>em}+s zq`ZLk8v_YP3b+lSewGs29j`1Q`k*FOJM7@GIzwThFT2MjkfS_Si!XD|n|*k!?iSAz zd@Lo=qm?DiIKX3d@(%jlSL8GM!}*FCgK@a_>x{K>;9(_w+Ba0{kD8oo(x!9S$Y0`8 zFMC@`d2_h3lso&FJz@iag}3DI|ehJ;0Zr2WM&Q7ntnZ&eG9eAfb=Wt<1E~E{&JI+{*`K`B{bC z(CcDAQucF(HfQOnDe%zgGu)1S?v{EN?yoH3_)d+869>46m!}mG*9wJ*?R)u{4^Zmy zA&na!^LpYr*v(Q@n|+lG*6fY&~~j!rzKD5AuZz*ZVg^J^IKf5K#GkgR z7+uYGRTdGvRpYvzyOATksn1NZ!LA4Xs7Ok-o?PN-Y}XE+#*Q4LeP=<+f@54tef)i_ zic->cR+bX8nf^74v?d8ZavL5URhaj=AZ)0*o!ijkD6RSxxCbb73`tc>?rXMJ=00zu z#)ef}xrqEDiiorS2oV#0<06ud(C_9zgp)!)YI=mf5LVFuhi#Qbl>VX-(Pj-7aUOp} z11?HBCq&%Z%0(m`qK)Q4guOyPYU++Z>{U_3#9u3m82IxC5mxRS-_hK&lF>R_U+pZt zGnWJ#B3Y81?_QrlPEO@YJ1a*<1j^NZ!IMq)u?k=c(IJFz6W(umW zpbq3zrBP1QPW(ze8`rz*x!hiy+J*OBCHig>)KpGw$*IFc-?4(4z^SpE+F4MW3TgzW z26Jkv;7mBx;LE9=oSMClYjvFVUxeXt;>0SP7%vzv?d;-_T==NqcgBDfN^|oM$!PnP`N61j30lx&c15P|sW*!Fo z>s*<6A<*eUnRz>KHSj#J{^c@r32+k74g0r)z(}B9L7BM=a363i@cS!e<|V*#VCHrd zdsJjgVfnNiQfYa`vzo$-Dbf+xS?1w_!T`&Ny1h$I;BNHqIcD-L_o(F93pv=4x z81b;od=hw!A>O#SKPodn0)_$Op;#wi25>1b3wQ{)23QKr1vYyOI^^^PW&!6s&O`$g z_MjpcSOhErz5?p7b@zM%3xSEic;G}}25<#13wQ{)2KWk?3vB!p{eeS(_{$5r#Xvvc zAz(c42`~fbSB(C^#7q>{pfC)W3!Dxt0&WNDa7wud^aJXifdSA1m;vkr%mR)Dt^uwF z<^pqpMZgC@9a56_bMyx`1jYjg0yBVL1G9jcxhSkb;W;oD==}l=fZc&QWb(N{Kj21S zJn$kg1NaD-1?*me{=jpJ|C}_XFdB4PT=_a5^x{2lIay3TsgD z1egnSEd>MMSfI`q0)c+O_%~nxTmj4gz5-?e+rLGBU=c7ExbR=}2bKbLe)vblW#|u_ z35*A3l%qfJCt#LeCjPxL3Tsf|{0$t4{eU{D+#C-a2pq`jfpaoZNFn9sjlhk-GeBpna&rlAEYPzS z{{6Cbxw#oI7nlJYtS>ju1UlH1o7V$>1D*u>R4F$<0%idn@N#1hFckO%*bcbFw%j}l z7;9HLwsSy7;0fR;V5}oL0)GIq|MU4AcpjMO z1ciZvfo^zrvj-Rnyawz7jILI0o(S9yTnUVIE;r`_cRH76ns1}f%%$9HTNfRHp}=OY z<>qwu3>*uLt6pwi23+k{Zr%&@bcc(9bAZ-(C-oK>1oZJJH>Uu*0EYml1LpwG0oMcH z0gnL(8_LZl;2J|F1P3Fdp&|$v=?MnF4B#AKy0P566_^V=4~+0CHoA4`qMM9$U0V8`!%6 z=Kn+#rl4X8a3$~@a4%372?oG0U>R^J(7Pcl2Q~xNjsgSVSl|@kdf-Z+ePi?o#sjai zdSDswCeXVP{tsxAa&s%-YT&?36i%Qp6POx}CHOR|NV2XSKuZ z0&V~f1l~`_{GW${T_;R4VC`-gb6~5F;46xePg5)m+dG=db^eGJAJIO=mGBH)~{u(TO6Bd|8`7_b%aE^r_) zV>}cFegkBmp?x(GvkO>|i9!hqagz}o&9TJ-HUo~Cf{wsE;7nkHFVPX$A9xZtC=307 zxj?r#cy2oS0lNTu1G~>eKj1dtkHGq~(GNHP_z2i#4*Ipg{9lJcBntj>(Gl1UI0bk% z2OWV`=arie0cX!gM_?|{Azr6zx&ZxvD z#sqq-K|kQ9z^lN#@6oR%cDcaX!2bV-e!xY*VZfI^qF>8QovzMWL@g>-|Adafv%oT7 z#(H#YrPHkfwgO7Opd+vWa31j42J{2QZp2suS8qZ;;3=SY5_URU&<{8OI1qSbEBXOn z0XG6C?ZS}wp>P(3M~nb;XpMQj8^MR^GlwDVVY03(Eu+%_M&~$eMTqYG<%Uvk-1rR| zleNLB%|~VQfQR0wUxK=p^o_~dSF)qV`+(~-hQY722LltG#Na7i&fqJ$o52HmnL!$T z!(b*gJ^F`>?MdqO z*xCobY8~?!wR33?26yNr1_^XIgOzkQfRt{d1*~ZteZy+!P~#K8DH_8dg!W+2m`-Bw zBVEobcR#VNCC#9P3<|0A6c9q|JOwxFy>#ePYcHDd)Y=x`iu;t2I??$c!CRXcG@yA5 z`q0M=W>Lptz#&?nK>|%}{l2N!58 zL#KK;?qQg5ffg_v zdV#)WI0fdv1kOf(hRe{O;R^I;_#^s@=RFKJqCdmU=+AH$`oH4Ou?$b3Kf{yg&+sDp zGsLICdl=qCe}*OK&k%0+dJXrM(U{k8h7G)rN8$Oq z4!lUcN`WIT(pZLg?#XZ~`ZKh@L{~5ze2MO1_~#{Bz|cFNzGXN*pL)Fkp2+{^YrU7g z#2K4i+T#s0alT9^GfLp)Z%*mGY`VB$-M&osu&QB~X#vB{m+4!ErI)GKTZ~A<0(|k& z6C;xT79%pEfc8W!`Kf?TX6Si^u3)(S$~PnQURB1r>U70d%4iEGbSP#W%C0COVzV8T zW>=}#zl`-&8q3h9koII4Sx6@{Oev%*7g^ zhN=RvVCZ&DAw?p&dGcZx6uqy}J&Zc}8ZBVB{Tg+WVE1;=c3q>V%B_8D@=-3mM&rt@ z14z5;bO2CVLi5V4ef6!}u~SrvHbYQ!D`GtYq%O42J8NI+{Lb1Mxr&wb>_td-I_;fx z4H|)JJ$oTi@Zs6U+XFfGChhwUbARPKxaHbS>h&BCxoAy7ZqXy};Ob$w%4k0?c&>~! znSG1en4$Cvd_fUZ{cZ5@ri;uFXKTR9p%iUGQS`f`w4htfVD^LZ;6=}v!K3zFT41&g zB#ZBUL-YZp^zJu-fKKIFhcWXZckdxee29nXo6UgMCOFcEq?*2Y0{GIz9Ogr& zn`oG%50G}z<&xf)j*|4&#Q8oQk2*5uK3&1^#C^J-Vdw)ILi9dM3EIqlKy8@d+e%#k ziF-)fv***b^3(fxS^2kGCz@)-t@aFY?@f>ZOQ6l`j+)7Ev8<2V9GNZ z%W(fQ+LPf=&*@}_&M)YGpiTFhIOZwk0P^Ssy{U(@hLuno8{n%F8e)UCYhKb+_U!zM zj=;11pLm;MW|DuROBsE|Yvwt6#s&j*;x)a<>V7GuuNdAer5;uEKGItnRYmVhyx-7n zcqGUpqw3+9J_Y399P;Nxg9xqi!!jShvNd_xp z$%gVI=sn-DeCTEK2<7eg=C>X!9QYCnVx&^kw?lCteP*Yxp&y3gPNkSU4@FyD zxp@rueUS0#dl?%+@A?0sp9FoQuKb(RdSjJGC~wu3)21)1ja3{1a8ScTFZK|MhrM|C z0}plEaf&&;g2?3eB8P!~6OF4$;5iQuck%GTUhi+^DV2ZoRBxmaRrS_Z?eGApG#w8l zU83Ep>V2#hixvx6GPhb;d&w9u!ae;YmuNb)5CS{{Q|~X1=S{nH0Vc5gp!59i)TMEMO`= zm0;8HL-4+O-z*-@dhehG$e(pHP+qDpH)C!7(6)`zRLP53pwEGdU$A&xgNGGNU-poT zhaXi+5gvA`6dil>rBeLxa86JHy z&4Dge%6OsX?o9W)>OE}sqTbh;7O?ugsE=~v^}1^K&jxPPr#k9`Q10qUTUQ6qb|`l= z(xIpi8H;i&yr}b&kXJInNL^N}3=*@58MF*-;(h2&)^{(;alW*GmDwj$&HU&)lrycd zIg7>nJB3fZ%6wQ()8SoQW4zmI4_P1NBq?$j-4FDEwaU%?E6}^Yr^kc7wl-ciR-k{x z=x&OlbD+c>ybNUj`_6i&BRx=E@2=m7%3Jifo8DJ{2E`lnF^bjLX7WOi(076(?SQ5> zp4dm84W^CV;jnm=57noA-Ss}z2BN$R@A5nu-&jYQ>4C;`P`NRj{)7fLGwb6uN(4RO z&W0J~4Gri6l=Y9$<|?)G(EHjrBRJ1AqG29-AE!u^*Witr!fylp(gTCo9d%0@)A69& zOhI{06S{(xCx&4fNAn0;hVuAmdW}WXE?)nRm37!jPL8HN2I#E|$KnygqRdO5g8CbD zkU?L|CI!_Sn$oWgs9uQbYjhKFsn;~fMX(ILUgmzoe;KxLIignzIdi-@m zMP4cDi(870GeyTzG&q^WJH})8S*11YYh=UGK&R`Gte9184(dlFmz&vt{Ihnmrz4rf zIcN~wMlsOl3>tjahCX2(&!B!^s?yPheXe&dl{WT5y=Ozb{b)z~dO<_>v000D^h;Ks zf%?U1O1;y1)Nf2HH-EtdMbbFdc0K6k_Pk#)=*AAzsRsHLqyDpWE+-nB*{pQhjnzjt zLYq$X%Npp*J{N4+iLPe#S*U-~nL2r+{ut_gyU@F=p1u4$(N(F}JA2}F9vj{MMk}rKjR?8x!jNds3U4 z|F5lUkE`ms(uWJ@3aE&vJgzUmYBHE0QezDD9b=VI5tG;mih!Vih>D0vj3{cNSdD_? zB8)MSL~3Z8X{;4%f5|An*7qxzI!Y32$jdqtLl||?8av-QYwc9-%>ILY&$k|X?X~w_ z`*ALJ1HTS@S*q9@7lHnkO4AsRdjiMNbHtA&z&FgHos5?NKbj_b_-)^ZX>^tGBfz_+ zON;tr9osh@CqC@29r%~tzA(&xI9e`41gC=`H-ifL+0lch13!#2O4Ros(I8KPHZMf0 z!Fx874uaQR4c=i{bhRIx#cv3cv*-ci*MM)y7Tu3xk^6c!P3#Z+(I*iF7f?Ze@Gk+L zwt%)W&Ih(NIb!S~;Pp9b>;zkkJp_iYavcVnr;a<+Hxk~rM&L+mA;Y1Aok)e}@i;DB z*XbeKeM>vB^-LcN`rbm!G57o;2BzcTQ=FL&1i>1otM+uMXe6LDptsPUA`v9s#>eT) z8g4||L4g!G*!J&X?odpw!l*>LHpZtRkI^aPU_zN7vW_eaB8JlqwFEB z;C)mv3Tv05hIdBU$AciWbc!7|xu{mS9{%fVb+Ur@J&ed_Uo2Z&|6e@7YNrpU*xO)H z=2W}blXKng|A_lqy{yQnhPJ78L3huadIzO5S%dXzHyp=WLDo0NsC2HK=Gmy1sG?Ij z>9t^?zf(nRbM07HdJ;va+Y>#dN#cJjsGKwmk)8FtnMCF3cAD#4RYOa<9qa2kw^t+@0pc`3svFqYbC^OrR^|b3H@lnc` zgzm-nPM1GH^`OU(N|oYLmEH(C-%6UI*YD0!{4>S3UZKGGkUORq^3^GQEH+{+=Y1)C zy1|<$Z$A8xR4CG($eDbN!;UZ0BhyZuCO+<6gLDPqvgnvJ?W(4 zXrbTGJF7DkuT{KNFDqTF_!Y%#zLIh(6b~98dM*cvU9pM}QoJbC!3SHI-kiiQ$=NXP z`c!9?xmfYy5WtAK2cz(R3JeSM^`$2J*5I$x<&f z;>Tg=aQQE2L^2JW1)Sv{|D)*rMD1Ov{Db}`_({e28w0Gz`zr1eif`=Vl$&Z9hkUMt z%1&ZXsVWM@HiHX(FF|nLO>l}+{AQ`(&kAhi0_TE5Z&KS*R5Y(MMHj-6wR+Xo66G6& zjREtG(Azq{RXkbo=w4J_h){*!dvqv8O244=BYt!bdOf>zN-&O&Kx|;W?Wbt`GKApc z3&lR;;9RBO(yP2fRqsyVT+iDhg`amLoDL}e{Ik@w3=yF6zl5+>$X0Lc30UsvV3CVd zxskxxj$1Clot-;mauq-0Pl-hwdCpO35h79O6>2L&Xiqs!3Cj_x8_!V9ay+L`pcc+O zCn=^F_3fdd+jNXW2zgvi@fj*ChTMqD)Wm#`&69FXyDY_uocZd{lV=5Yz_XOd`9Zuq zSOGuXX%)k<>u4#*4BlqtFTGC9FWFmN zvDuWj+D?n_={3E+{}Yk_ycokiZ%d%M#6uAtI2kpEGl z&~K@oHsz_zc4xT;-6zHdEqy~<(JBjU>3g){qK^lC(YpYA1C_4 zKN5W@s&BaB_ca)p@h}}Yms7T!>Y>jSbc&kSq5qG*Ci2FfsPUq=q6f`b55CG1RL*(6 zAMHo(d2Ei@sjDcf=hIp*yc%8)zqdXqbmy!NHvLlRquVHD1N@r*IUU}BAb;!>#lH-{ z`roF;mk|VS*&!15Div*^<_E?eXgIMrW>52u{@jLwEQ5d+=OTRjASF{yG~LY=lh?f z=n8bs>Qu_B!1L@YRKxk2U^-P{f8dGJ!EOfKz(lEhrvj?oj7DwMB~z8!_de)sU)q;+ zW3$~a%<#vfz_dzh?^0zN~)MkyM)z>bAx1&d!eB zf=2wK_8@G$oircJ?2HS?rL+l(uUCA!Rys=YYQ@j>q?#>QkM|$YvGQ3sa$tRkbSYcW zNIk!yimf=Zm=Y{}SRFVid73y6mri+J1>a|nQR1r@VYMN`SEhWo!N>aW6NdH7S2mBD zneT3v@R{Y_L4*S4+kKl7wxOPdUJyXx z6hH6@HA2oa`#;2{V?SWwSb=Zfeud`=}Iab$wp!D&8o5FrE2v;A}@q6W!Q> z!Ts)O%KR4upHp5dim#P#m5<1`s6F;fk;n455pu4AZq&y4Tm*xi$Ptug?6m7#!8hpM zPS};E5q77_fATZ2Yxm}kc3Cf|UEfg!!GDFiUmUD*X`u6<4ta%|m_F`BkvB8`rz(FT zlswh2HK>0jzRZE#Q>XH9yORdZmh#5mrCR12bb*>`?1^PvCU>mw z3GlJL_ZJC{80@5P6~A91xOQ_ZagOK>*BR1GM!CRQ&wafw+N?}=Du1Qyvx)xI-sJkx z*A%@A#@zP`hFj7^`2??J_`fM0qkG9HHS*sTAEw7uy%qmh@e8_)GmY>aa5nNZ-q<72 z*SqX-iEeEXoG>|Q&Rj9V`;0JO@zH;kmcqbv(gxtHH2Enid>x*i5kj?`2h5_T*U>ZG zI^CBk|I&2P-{rq4emC|$-g~19N-x$(Y(~bHisR&?emC@6q13z^k#Xktl<)@1srFj? z9?y_+3h|}EH!uQ@giA|aP&<+`g?>=$W7ha96fe{c+M8PCgaLgfRTm?DG3bxj~BjpAuxfM89J56JN8M}72lr}|!S&WETVZfQci@yLS zrr?H;T3C^;Vk6cG8G8vQZ5HxtD($^e__ag!!4G#p3BGgdWG(a^KeftJABuVlPeYw- z=&1Q(-uJp0U#SgW2Ar$8ZJ(6iUCZC$h)sSAWVXKKp78%#$rlyhrQ>Uu;x|>Dzk02U z;tw2ku5JtH-dlETL=++mt399#1~b8qrEzb==I10^NK5e)Q3Pgg$Mj)hqB%UPp*>7) zP+jSIT(DG&yiFg1{DfbQNdI}2NSnE4yyBnhA+R}IE@zzkPv7mzv0R(RNS-zv_=PmpKGov%57}HHFXl0GxdhqGv8ba(KV3 z3C@?20xP5>vqp-eJ->kSlR%EL!ma92wxhCL`+b%}9$KQ(xq7Bz+V~J~wlhr+9?i<9 z$q{Ra+h^a&?3Iz9UznY>d|vk4McEkzbBnSImMt!97}(jpt~1qzx@&z-(Bv@p7?=Ok zR1xNm?S2HyM@}aHEu9Q=59wZ1E3nCTQb4$SNT7L(iI<<_!`;IJ%-hRWS{v?uGQhkl zZErXl?w;g>FZBm{+(SZkh!=`9?`m09$+dkc$K$@{i~miRhu-es?&dY8$q&({-Y7LP kO8kVkosGTS$6P)y(58MUZqh!%|A$WE(Ps?#^mjkrQ?*bl#<#?Yh0oDI!NC(%Cs@hkhu@xC~MM;J${t5cf^ow{hRYeINHj z+(o$A^#u47_jBAYaKFU;3ilh_WwxucO%^F3Im3VauZ-A?kL>RxLe|Gg*yg!Timg@<8iaA zBQObfC)~-nyW;LnGIY9LDD=Udin||fcJ&9Q;U0{8h^QL|{1o>H+#^NZXTVXSJQ_Go zls^Yf66Flwblfv>&%!+y_k7&n;$FzC&ceL}_fp*IwE|^|`#apLMEz=kYk@z?$85cu zY(nW*+}XId;ogq>cig*i|A9Lf_aWSeaUa2bG{@8KjFtZTq33qy6gw1J$=!)Q=d81> zB>gV;2sFyYE*qsn`B@!f&bC^GlC&#liEB+F{Uon*Pc+<#!NAFJr1nzdMyamCo@MI?O8)`*lEJHu5RsGRn%Upv|J{z=Ax~HhgsxwIX3ym*H zz2zmY&`@`yv{#|5trsDkkP-PMtE`-u7v0(gD@JDmS#QJ)3ntZUZxoIg<1 zk96kssmH6@7!tMWi!E$c;_}vW!fh|!z?G#8TnW5dcO9ts4%3-k zew?hk-%6(o)OqM~yTG})V25(GZO{4t?<>yFog?=N@#;A|mXqF!d8qm=(2my+Y|oo` z2o*LRg(xyYu=~G5o5hUSb59%iS%8Xki>z@4`}r4ml2dg=<_)F@#RcA6!OknZ zqK;5NTJZXGma1{VG0N4yHm_e{5%3dSdHv>7Tn-$jT-MR3H=t*;Ys(WFcI+I(vNbMs zKDG+y?fSFj0@n)Vr%p%21Zv)$*EbXOJL+-%flp=9sAEQdscN&ag8YQ7RJg{9X7{it zGXI`D%IgC~{d9ZomPH9%P9sj%T?yv=`Nh2AQ^C*Tg@yVtUxmMsE= z4Zauo=Cqsqyir)>Ikq_AQoHXY@<;j!t|w45JK@0f*E%Zq3=PX5cR5-0&N$(<;RY_Q zn`m}kOzHhrydHz7Tt<<_E{YVOj?OAJI_SxFAm(Mp2BQxL0B&=g(h=m*W zARL)}wI1=-Ukl-DJb8WJi}IAPu#6Q2oUBgc#1Kwy&h{y|o{DA>ow&&cwo>D|z{$ED z^*O)2;8)#HHZKWEE6 zE;Th-wd9TSgwLCcwsovH{|&LvQL}Eeh^z&|!RVWE8AZy-Vw)^n>ar2aQ{NEXQL!yV zrkQwlPhR@jLU&O<@!UcIbm$Gh)Z2_)&_EZXSO@mgE>He0GR)=l z5pFNyO}XaES0ep0BE?eMOVrkk*dbQ1bhb0b)l~40d!ILWz1q8}V(vB= z%j?y-o6N&N*O^B_`}cEkt>~R)PMH_W4GJ5`1*wy|H!>HCvupP9o~FT4JNaf)f3LOd zEeMyo9uMS`OPBtQ>=hN})r!41;hL=c)ag#ZS|($-z5H?1z>fy8_Z?gx3-UY!3nTX` z=grg*&l6LnV?5_qBdxudX<6)@443Mo1lhBhzn8n1uWI;7*dQ|LSFgxDn+0bK>cabZ zLF}^Bd07L)&)AlU?Pj_#&LEb&v+M;Am%0-u4&?lKO1M|f%dwazOwMzO8*LVHHuJn> zrSr0t-!==&c+t9Qr<#d0bdtRQ;!*=8U(~O%NKZvkT#sJl1+}|HiM)TIL%E!N?Ru%zOti^VL{0#B&js$8ga-m>^uE@%j>o zZS1y#f)(m zQ>d?~p9{0u-p2Yb&c9OB|1F}>*&)Ht zc9J}$RaoSz5?((=$m=GSnJ?I34X$iaU+?|0C|_+A);UWIn(CuaaI>gb#&TT;>}cDfZB?cDOYU!)cF^M0_*xX zJ1%>IFg3@E*WbIuyPyV~tJw9Yi(UVhg1<51fz8+HeE!L4F)opod%{z$TxOV9mDRSV z#A0_cPyR6`ETi~3SE;6siDGl$-i4c}rtDcf`{|C07dbV>hyRaMlPt1W^dK%rjqyw| zS5EL4&v<`=5HGTP>Q!!#I=R*)bHCfn=7MrLS*O?YdaowDUVZbJBEqXkgcqV&xfa2x z%=aV3D+Eu^89H5Gkp>Qn#Ypu8J1}7Z;v2*jw1q-;EftRaQ@jz5;?=q%zFa|P3y)Ia zq2U(ESRV`dDNtaO$`!!F%9SbNy~`K8UVWtin5o-$NEQpDVb$vzcP~shckc#f3%Mv!hpB>O^^q1j9xnOT6065Y0xs&k6F(xQWM zI!96eO3a2~qP~rrBEPAw?it?esemDSf{q1pgC^gX_r!;F){DI}RzBrg z^FLOuQAyn16=G+h`o38_uP+-XN45`(TPe;5z7^tTij{L)&#G>j)IPLo(N@c#f#HH&ZIj5A#4U5Xvn3hOF&yO!Y9Q>B2`E;*mM4lDV$#&@~=DadfWND zUY*Dpd}XSot1FJ(jtN2Ga;FY0GUnCd>`|ijU`^hM!xo96K%~s(;)NTVFXj41EWDe< z!mH+kqhiB9SG=*PE8bv{VSnQpw$b~Y@ED=kGDGze6J%Gq+%O?5<2y07s=b?>xxJR> zuK^-sZxC-li-o&?Zo&EYihg5-P_CQt+=+hTBKH1j+VD1TmD8hmGj-&>v0AXn+feL%)P*}jUf0n- z(!*lM^0#n+Wnv*>CHU%DT7h^iM4Ty?3r3062Z@)> zX`+6G9M{R8#N3cS?$j~ziw<0gI++5+sE3K=RULv~#PWle^sAl1GCmQDpz4_C|6}qz zX~!M&I*_Xg7B(D5hGMq4KIVN%Es##8%FW$n{#e0ItpTQRS^B>{6w+`~#&cV`DIj~C$uMm-yVT*F*icTc;ktcKs z8{AMF-eO0iT&oZT%(gA!Bx$Lr&%{h+^&2hTuRKJIm5GlvF)x*Ch>oKJ zm^W7USiib+%NL1v2X*Sy6*KPDF*zuuZN`7aa?qg{Z!TVVbZdoE{}cg<)j_%1cI1Q8 zLhQIy{tF^A>BXyihTzYu&CRepDRkodb#GmBu^oY2<%$+tw9|jfZ&SiDEI&PaA>>)c z_ITl|Ea9tqLR<@x$#;v{uFm7SDZC%+zv2B>vtG8y4&sxt(D!5HBqAYFM1ty&DIyZS zFiY}%1RwYEN2bciLC5!dtB41zLzpu~W zdkWoJv89+I_^)78S$K9ATOqY~W&Fj_M>kHazx_mmF2dR;!VUJmThBJMX47zjk=dM&uWeZXhKTv7ih%Cr-y%oNO9t+PW0`qxlZ=` z1?*Bgo-N<(;oo^+Q(mjS357v9^RI)L1-*pMa4fK_K1&=tjTH6&vEn^_8!mV5i4Pyd z%d4uSL=1Z^VX8X+Uy00SE%ux%gg8f`W|UB)I%sDsS7RmK4Ateng9|rHytuX3VJ%j! zMabFA-c*tHx{3Oa>ht<9#QsP1^#U==SM-%zec;#kwqRc>R^@vZD{ig#IfW06+OL){ zn5lJs$m^#HISFDejdaZU?So~l=+Zj|_nZlfR(Q(x8|^cqyYkP21Hj!q_)Bcd?RDKz z$1X`nbYX(d)|GFTtgzPU>~yK3Zlc!+ynP#VL*DaCx@VZ`Fi;nOJLsPmSmpFe9Uc{V z9X0Y?M~vu$c(T!rts>A$BhX6cFKA4_*k8*^ae|9@sXu& ziTq$=xYr|beiqRhOt?T&u5L?nB7gCcq_uL&rbJJXsuapf!O0xhMb6%2^sLWARH1Iv zP@BrvHyNe*^2<#JJjG#$!c(f@*&(0$H5|W1QU0q@Dwf?gCrSt80h^6-a<+p!eRD6V zz5F-oev^H(!=>GF3VwbgPs=vq_catzTxSfoomn(W&IZpi`8s|clO4AhJtt~73pJcY za+@vT_(hD-TZ~d8dCrz_Pg%q1(WX*nvV01hOXTt`iJsz^P0`z3!!t`Bu+`|Dfm)jsHN;#JQK^@W~ppT;-I##wye#jajbp zw7o{jRbIY#1AbGi%|4@Nw#JBJjS<=MXbb8}jS<=MYEaeVQf9m$_B~+q?4~ioxqYP}2JrD5`Q@bJ&(6w`FwH4=u&j)$sJ zOEnUP<+M5UTJjVhC@QTz;ws0gwVd*ou?p3s=~!!dngumPle<2bvqAk_)3NCqvp$!v zTTnxiD)oLYJ03B5{;6pjqOs~vxs3(2RAbej@@NZcw8pCM6nk8y?PQH)XLJDn!Ik*}XNdYUzBNUTQh7qa6SqvwwrY27u{<#L-d;Zj?9GBO7-%GkCuRd?|6z>~~(kzXEVr_sNez6(7wgw(ZtXZ_B>< ziJs!{QK4SeP=Ao8Le*jo_2Myv{`oiCtS*v z+x)Wuzsi00pES>ZG=Y1zEAK8|<#hii54e^neIoyKEnM=H|Gt*y8LrV1(5;fQkDPKn z(Ni3Iacecy@pAU{#nLLdO@UDwF83}7$2r-Yf^bK1kgD*nmDgQ2%9jgbJR4|q7e{Ke%_aT9~h;{ za5$#P0jxKvY~R+J{$%GZn1 z&l9Csx!#LJsf9fHMVh3SPrWcox8%Do zCP`j714{fQ$xWX2GEoYW|9Y7wCCc?)9q<&t`>2S^{jgH8e@^+U3`zQ4o>pq~jMiA! zJ++d_QO+(+lm^N#OAjCq|NbUix+>p&vsg076W@kQdvbnyi!Qt8WWS@5^jJg(ZTN|$ahT^?sPA3OA-rr+D%N%QX}%_^eNhb13}exI{;iiKzCpu#*!JzfTEk)}s0M}YEMAaIlXf<4)&)lqIE{Pw>#FWjNR;Kv23{^EnC)lTl z5I2>J+x}kM)*)s);oQ69==VpY26f2_E1jtx8^?uii ziMe$9F)5gSpC`qWn}YEZbdwDzWAsuO-D&7CsVOnf<&ru;jmofcB$u>Rk<{L9B-{z5 z7v|9NV^Up`J%`2_i5q$KIbG{c{G5JMV^OTquD;y{-sJ2|W_}&gdZyM_ zjx)K1Gc^k54&xp;b6SdMaAOct*c*u>!IU)rA{%#~;oP_?lll|t{7MR&{5M8nmp=9{extw}#Q_ucSIO;*q4M$IeKh4HhbHt)gd>@U+&p zPlj>Z_MVl(8{F0&*pLDL+F4MAVG* zS;r1#(TFtSN(WuQ#I+khjTfW_4t{Llkv}cS8~W2PK_;Pn>G}&&G>Pv|3ob~$L{3$7 zS``g8)5d978zSd=tJuJw=}HrX68ny zvxt`EO88Y>0(uN$BkxKN{Vk1coZMe&7huuut`c}v+pSBZ<4uw$-RnR+Xx2r^sX+vi zy+y-m|I)B!8dvzkRmngLFG{ZZ?=Yrx(*>z99rQr5p`n)~N74*FbtYcqmw|L$1qNm8 zC8;6l-IIQQNs4gI5#BtSP=cJTdt@-38%%FplA;4AA*5y{lt_j8KA|pGq!99ZZ+fjJ@zVbS zN6{wk#EX7@MRFxA(pUue*4WR?Ve_)4QV~d!Q|Z}fQWzyyC3n*2W9oHP@-iHUk;mwn zqf#L4dlf-aK3EY-#`mL(uS$U=tQS-7O~c_0Tl6?fA2LZl>FS2KD;K?C)vGe?dwvU)QUP@lY+>n zsWh%YGSaN;aL>gaT+pl(P2~hpnbwO{b|vQSbV&i24c+MA8z(o>Ncv=2S;c{k${Rv#NJ>Jt3Tx zeA<=Ul>MQ?>-MTwb>5k@Ng*7(^rjR?GLyMCYxZJZPaq-5JX|+_K=D+#lzr{2=E z+fsMZQm}?6bJmc5VeQ3P<<4B{N1RofNuS+@Pq%dEN*{Nk+iyz^qz>G^Yadm&&sJe| zr4IL`XqV2Nz%c(1`jE@wtaT^a`yNu!@Gf-TJt@g$f09Z*%g9@k=uJkx)|tBAM=*pc z&$a2K`;rGq>p_>?mlCAiwD2D0;v-h~s5>>7r1lMVc4l@|N&FY$6w|8gXcwojwA7lA zt~_hEPN_OAL{k5=J10L%=2LE7Pc~28-_IVwR?tV8v1CFrU-#48NDVS5k#2h+H4oaD zzg!5u*h=Z<(giI#%|8rY#TPLN zw9g|%9%(&*dfk^o6p^(3Lo5upIxqo= zF2~xd&GMKHd)m{BkEBM#u|0FEC-o_k8qtJD=;nYTnG9|t`k}_ za)Rk-LLtjf$ld{6Xx0<9z{S#I3>U>x@>J^K`3DOv^cT6@q)TVH9Q&T5ru9j5)Ke+e z^z)9P9twx#Z`rBv7Hk)X{_C>8^OITaR9XTFxwV#*n1 z*0f?if0WqpVjW!A+r$S?d5?I_W{5=(ZnvSnrP9X{TN4#_?Q2KTzw{tg@k@f~7>#=) z#U)fQ7C3rgN?~U2y^x*PX+CD(73Sth?vp}|0b83W22|MuoQR=E-(d4JbADZ)x!Q_bQx|g+PtDy&-{Z@)44V%*=Z;_~< zH{%UXgja8HzZLa-C-ovXThX!au$8B=)VW-8YtUVpu2q~}9;P&}vTk{{qW9lPEnQDH z22Dwh%8bH5PHjV@%8(RiMbXh^(kIeabj&-cA+0EroJjq+*b9ZJIaye17qloa_sq1deU`PBqZ5=tw#?|-zSFDs-}a;_=uX2x!2 zY&1)i?MU0UyroTZ{_b@Kdn@AH3`s1Gl!q<(}d>SaqG&AFP^zwV8N6aNQ3RNe@;pV8SIx*HM`L62JUKqtDh6l}*NDJ4{5LJE= zOMw`$t1ak7iKK=*SD$&OnlWQBOcUAQN2$kfKVxPeqb`r6+H@Qtb^Kbx4xY6nWnLdj zTFBC@lC`?EQnFU94-w!ZYx%%Nld&m%KuC+oF-mY5?kE52*cLam9NUUyKDM)((`i;D zh-^*fW7|v_BqcjHRHCcuw$nY5jjbnt8}@F*U!E{LHI{eB5X}$cOWjmGsZSoY;)B?s z3ELY-=}*_^3y(LgX&_C!*TCS+gc5`=^NFkKlMRS1`a><^CR_p+374?RhxMS+C1)B6 zmq3#0l4$0V2>QT4+WKvP6M5Uw%sI+hqPjD)E^F(dH<*41r-Q8tc8*cp+%DmK=UDlI zyGY3?RWg60IuK|v%kHD|?2Wpy4Qbk7k&-E^w5btXz0D-iX0;8eOBzS}F*agv+O=tn18GQ7>+`pa zsSWsBMn+A%Wi$*?P7VxEvMY!#b|fB>FO4%0cf%{VTO1#RIg(mZ0mbXTH~BGy4s;~p z1Sfxt+w&-<*}u%I*btiQNKSZJF&84GAb67&CfTx&8U9R~f5ya3ccXipU}koN&yD=Pczv&^W^%P zhgBi`6=|3|UpAC8x1bQqGq)UHI=~%H`?M85SPuwh9DYtZH|`4N=%K#Kl)(o9jaqUp zy$9zisawfq`=0A{H0RoA@X?-1PGetrO^KBMFa>ghTP}(lQ}`YFK8< z9c-BN0!h<0GF#_;V68b$ErzxH!AaI74;xn|0jZpUP zZ6=`=@89pvECi-EBdJy^oy~M`C*n>roGD(TgINH2Hz#%4K6k{bJHW_3VY@bK+BIEz z2!aSb87Xqj!ZyETsq&80gumE+%~XcqGJP6kre|V^x6MZw5mPU3dOehkj~K5^tAw+e z3G1=J6Fa*8yz(+OZ)bJsHh{~ziH$KhHJZ`wA4_hv{#uOpy$u+DJ|2!f=3UR)5h_`I z;Ap1p!by;opO=|#izA*^wNNuXocK$Hw23dNP0ew{)+*n@O!uRb{OCzi&Hcc9Px6f5!m)8@McYM*z3pZEE( z2YQ*x$80wn>|}4={~B%mL1_se9y>(+>yX|g%!6L7PnwWMHK=0FKRj)%D$z^4Y6B%uIHX_ zqvxKltLG*!FmlhA>YxsXXr>$X)jVIs{GyoBXtrufl9g)8*O>CA2Rz#t|9Z#=!z|T{I zmjle0gF%L;6|a8@yHu}_M!C}KW6>Yg>q9i<4{QC4*W26EXIa>Co^3=D$)^=`P$y!P zx>Ed!cYU(cg(vXs?<&luP{g7|0&j^)%PpRaKON&1FE3XuK7qh8HOHc$B=8WJZmMI) z6L>7kkn=4Pcq3j}J%L{h;t6~i{vOu!1vMo`UTAv^D@`UVt}V9}Ng_2$xwhx8xwgJFXqVQc zZri#qq3sOf%R<{VOc1Wkd$Ou68kTWwW01&9S47`zl_5St%=Bz)(w97V#WkIM#Wh9S z)6p@|^e5AljkvJTv%J8swo)>$F$2<^y?S+z_eb{^y}zLu5HUpuFc(! z_iNA#Xv-|Crfnr#YLr=iX`-sFq(s%$kU;)&#E&v&-@pf^nL5pw{am7GlNQo*QJ5Jo zZTOHyJcqK+Fg+}~HW`D$mF-GbmF2@Q?nHY;muaH4Q1&U@VEWih*EA#j;eF~q6?0$FDf51yz>)Y|FRwFFrT2l~Arrj8}YFcy3-fNvD zO{YIZ!-WV^Mc+41gubo_G7EkE;3%%|yK$<%U9gYqdyGLgrNX3Y`aXC^vs;mV>!C@YtAC3$y<7+H5B!F z!WFfC!WI3W#6y2HQ{;j8w@_3M1H=^#9iu9m@mN)~3*&7H!pf!uqbrM+ns4Z{7}AHd zc+3^~JysN1%>`FMd*auo-XpZpA)YNXIkFW;=~D7&RnrJq#k=$a5||0OBd|U3u_AFtDzs>pEV9LhOTljMiW&`)1Dbv%4o@!2P>4A=9 zFd2B8C#oZ_Xy-)Y-E)2+*g8C{rl|+UgKNqfqH4;9Rb10;qz_Y97*j22bb9h$qZchp zBxA8F3QR<%`syWJoR_Qmy_O;&)ZqcFVG^Gn^(tquB zh--u>wAi{v7poRcN>eRbR-jsRoTa8_%xJbZ;wKToEbA+e>{IBYCih45Y8TSf<{1lw zq6geDt_9pN@2+z*hCCB7b`4C|5u_H5dB}t+xvs%L)r^iXnLB1GLdA6dDR;~jSZvBS zaXn?%xSreBxSlS>T+hRQ!1Nb_)I!gBxR~p48KCM3gH2q|P?jOjJmGo{vT)mbpX<5v z57%?zAFikAQ?6&iRWPv-wa~MN4V2QKH~2~izU6d>O<;AJShwAY<+By;E$zVM_yhkytX?L?3+%)7hS& z7fkn2$T+iaYiT;cz*%Z|Qb^ZLg%#}zcfs@;lKt~Hiu4?7VRp-VirFOX4ZYb1;zJZV zjt%i^=v6&)8uP|d{GI~VfnH?&WxBW@Y|kuJq?hTr?Njb>+e=^5wW*|;_rcfPH;?h9 z8dZhYq)k4==N#{@adS6c7UrHQQ3ReP!rYFxxR+y;`lF`eXvNDB2J!sgXiLfeATBM< zK6;g#z2puzJNJd6#Z%&De^aQKZQod_K5A;xg0{&djU!$?=jOk~7nxKw-v{Z^Qi<&q z-j^>$U+mT^%?*z$XzPI(*bfT0=UtxDqXS7Za`hRl7>GbIA@Nws%D<=>Z0nZ~8v9kL zsY6JH-KS3#vjfWMjOh^gHPQeJ^86=s)gTPz>L=XXXZTi`s^-?lys=ba&gVh?{h7)j zx10A^(ICC!CVzO9d;et-mn=2m-rp~tk4IQ)Dne?o)O7j0Ch9j`qF)RlbL_e(A`CHa zcwfG~%=@zY5zQKkzRY;UHI|22YVf~OT|HWZ7#j{^~_8 z-s1s{{RHA~nz;Cs#+Ks8UaBs>-x(&}m0tav)FB@qr{>SeOuMhShWjtMhI)T18tfkl z$!8i-zloTLJ4`fmI7WHT-__;*e2RWGk+jffUsTQh&ppNLRwZ2MxC`8D>w9$52!za; zyY%J=2s5y-u*faXPE;DHPdx~UO3eIqN0GSbIhVNi9G4h*hYtJepv zdcc>KI@%mp{A~F00P--s5sFmiHH!RXcT|yLSX)f(#v?F~oS|LDBQ-_|;p2ko>hYLQ z<+r%9mcf?F`kk$=EHRITjSu~MG-+fAJE7J1Bb_!1AKRWDL!9isEmS&~`Go8K<&>hE zZ63ytMF(?laGgGNEp@g%U0r9xBV4EZ1QKTFbOa1oEGm1d)%&VI(L1(?>z#9g>m5=+ zmygF#g%;40<1tiTC#!3*K3u7Vj66jIuA4_bvpaf4F?RTUI^!D*_$Nnr0*z9tkD8o)c>*oJ%#FU{V`=m|=39#hzQ12d zp!$$KYVZ!bq^LP^R|M~2Zt?kxbl_z8AM)Tz(IH6;@TtKgZ0qbWGZR`N}eVANT(EN^8TYLPfaLnLhi9bRxFr>Dg(J zeeN8Wz0b{3_6R39}Ba^)l2SUfjywN%n& ze|04-5H{GA*bL9bEasmG8L8`ZGSu$kX+>ST>wJVA_wo^Lf0{bZKwlgb@=??DT9#^l z+*@7Eq8&_)H(fY`xY;=D)9A{bL8jPE<+|ql!*zAp!*xA4MbFHHuHQ~^zwO5_T2>vC zLIk9x-%kFWgoAs;Z&g zf2(U~v|VwUp+B@MDaexs%pt$p^*gR8(O>3?XU|Tac=C`i=0eGmV_ZpNd{b0aCEa#a zSJHYb3tkU;eJ(!VZ~qhT#uI#fN78&l+C|2$mhbp%;CZlkJ5GXA>3IZA2k)>o3g43ciK^1_}gE(@Gk4P z@QZ&b!V@oW;di%l;qCvT%`%bV9R8xCGBJ&({8nAmC!4stjoHlISq~&5K0Z!phrDU_2bmVg4#|{9XQp%sTU9Nw$_|#eBq}^ zE@F_`Y=EGiqJtKYAmejRe8h=9g1C$mZ*t;QPW(y``*GrBPCUnnR}XTfK?>313@7Gs zBKram)|&^v^4=Wb)B~JaEShc;)ZaOE3#Zx$>Lfv3 |9HCRyF3hJ+%sxW0XTb;}G z8aUPX9jD5?t&eDXk`7)3Pt4%NDV%5}h%zURZnV($B4FH2x@yyjp5YZg4#wVqU}e5YR#!;d{rKztrUMr&qh8# zP+zj@g>FwcwZ4#ie*@QB#Hn{V^`dCIMNltr>Pb$0yq8O!D5%Fd^$@4V2&-ZQbvvhS z;?!S-WGhZJZsgQ;oEj;p$LX^rh>OLXIFA!Y2;vfjn6Zcxzv0AaVN>eQnEZIB!a~Fj z0mPq-tYuv^;$;SdWF^W$A{F6PdS3fbbE=6*W}fM zCu??%TSl^KY$%i@Y3^DwcdgaVwfbwVGT8sc+=@SnPij_XUW>A+OPM(jXxp{STnOwj ztjt^r{0!*fgy&Do%+bKa;brDjV9OC@=E=aUQDx?pz+Gd@%)5bO)62}4fzKzSJ#gYQ zw8zSHayr@rU(7=Ln)n}ZoxgzqRHV;`0N|QT2mt0SgaDxb5(ogkSq1@EJSVS&0ASQA z2mlUTgZ99AYtbI~!;feW4E+i1fwMNEJ#f>nXz!xa9n401;N`7o?~;LkgS8z3P+`3T z0)W0dApqFmzYqX)-vr|3a5Avq7}{r`FzN&Z0BfCs0N`R^8PM@`nK=MEps#@)fRoRbnTG-ooi8)b z13LX(X5I{313U?Ae5K4>1pE@{fbH2qU??!)YMD71xDPlAxbB}a^8#QwFk>?ce%B!Y zxENRr{HdVK?Cg%7-#`!0lXze>a18J=>hlZHL*S^Jpkt4zzXdvQEHD-L0yr5s={9-@ z)ZIY`fs25bfiv!+J+Sa@S%%re1K+oE4+4OzfE}7akO@5n_Iyxgo(&9pSY}=iJi-vi zCAA-wnGXX)fzdFeJ1`Zv3^*D1CvYY36>vAORT1cr-yb*`I2X7QxVtC=3@8+!;xe!V zSPFE1jGp1!cw>R=TY0|#rUF+2Cj0>wFdcXq_#3bkm=E;8i$?{J{i`$AVh8|s2Tle~0ImeC0`3Ox2VMr6 zfTcjMXK0W2q6lC#@MB;qZ~<^Ka6fP*FryfS-6(iHhX7z7U@34O&;zM$6EGT>4@?C< z1WpF_c>w{yv%uZJ%m@NG%(^7+5@KpCj)l^SNdW87oo5l70$0A z05}0y3d{$3U{}+&6as)NfvLa};ACL(8wdav0Cxiyy+wQAD_|)w=pEVz;NQyvqk#j< z&>pxRI2m{qxH2FE)2JK*P!R*X44erp1?~WP1mZsc1EYbu3J3rW0!{|*0j>lV0Cxk4 z8SR17fu+E!K#w3a14aWs)|H$41Fr(713OCP=JgpUY(gOq=tRoRMZgI_XMBCEhgG>b z3b-Gb3LL60H%|rH8_LaVf!lzGfj-vd<~zWzfqEQ7>;~2a76Cf|m)ew@hXbQ+%gqab z1;A~<*>>gTjC>Rt+LxP4fw{nddOBU_8fXa20}cm9JD?$OEpRii7j;B^LxB!B zb=eIJ1zrOt1DiXQo5ulv11h2JQe}1GaK5&oJxoJ-oY}VKK0kOSw6TJp)Gp zV``O~vw*8=mz&vtBXoByH(v(M1(pFzfdM#M@^LFS#{zo-2LY!8rvuLd*8o@*8lVpUFXX&?1_YoGX@mgaK;U#>SC4Y@M&N$nNnn^~xw#m480dkoG;ZrvZjJ_S z0j2^wc$b?e1CxEq%`1TyfxCgjebH0kTOj*qp==L1gyOBmKEH@k-7KR?$+OaS`>djZz~(}6$MgNuQ)gW)NlSA8gE7=oB+jJyG? z3tS590K5kr3e50pP;Q=$!ffDrpjSim7?=et1{MOH!>~VSRBny{?gn-TMuuW+fj&(z~MpVb3P zfQ3NUrdYn4mz!gNtAPE1dBCX|C?vH&lmfQ_bAfk&g}~5g1WlAqw;5O$SOV+-Olpa- z1EvFK16Kmq0}lfa1B-!ofWfUW_E76_ggL@#h>8w4-#Vp{|+@L4=O0Q65p@BtHmrNCKTFw3HKx?g}X z!28`GA86A9^M4)+LA^2d!1z9J9dPJ}Xb7B}T5k4fsnc!z2-6do*B1?e?)_mHaMA$u z9C&FUdIo%x219|~gOQM0;Xea@jGzO4Fa$#e9P=q$4xBq29tW-&f%$(Jg`FeO5P0*m zaWcT&?g2%2#f&^oCt$~ zGH^C<&Lm84;AP-RU`)oB2oe-V0bSeRKPrBOhQJfR{=o2QXb4OLt_Kc*!R-I%?FSYC zgJz&zTMQX63fN~B+5xu%rve+zK|A0e;9+3TZ_p0-6VRa@=D+`ZG(@2nuov*mw`d5o z%Pcpq0?uB5hQR&6JHVEU&@dMN`xzJte7qR#fI(T94Zzz=&<@ymX}NhXP`3>2fIWcv zI3%zYXa}4KOpe3+FQ#aSiiRuUTHsIrK||mT;5A_T@6j+G3W0Tjk5{1~FlaT_2%zf^ zXb1chxC3}%E!qL!0n31CKcZcGY;J)afG_?J?SOUGBWT-aU_SnYhNw6L%m)tKfQG>D zfj%AZe@Zr@Au#+Gj3w~tCNu;_|BA5$uG)-tz>~ldV8j-*OMuIPF~GxH(GFMwoDH0` z3qt}t13c`(2q@eEChbP>VY+`v)tiRarpq#ardA% zO?rQ+3zZ%KTxcMJ`!tThf9YTb#dHRP*K{3&#q=P9=kz9n-c))BSU>|A$TW_@aXOd* z&e0eY&~*$<^dJL&dXvEwDm?;3(?ACKG>$<59n1iq0x-Bi*D=5se=wLuZ!-9nN=1Mf zG?2k$8pmKP9n9bpI)efJrkp`1dXRxTy~*Gel^*N;1Ia18)-VqwIN|^Hw0E=~;$UNo zN9K)uJ$4;54s6m=I+(#@I)gzcx{kqmdXRyR20qby8!p*lSusfXHV_7mbRvIR4j`S6Qv0WRKf@^WjnXTOvXH(h*4J-e zlwW49t93v>jRU}43_9Yl(m`KU(n~;h{kx3*;9&5yakaxV!o#ml22Vp29^krX@JlZg z7tw>ypfaD{WROgy=YYdBkU>ov$KZ21n877FgF!4^$6yOR$Y2J&$sm_XF8~!Z5MXun zLK*F655*<+*d*eirVF#8238b2JasmB){nwN>4h?LJ+1xyEA8(EdIgxjXJXRvP@>9Q zhKE-|reOyjFvum)?M~xf=za94@SuZP$pM|FGf=9FQWSiJtXAo(FEqM9@5MO14e2O; z!kQy!P#ni=cc6&xBV)DsC@$c&B`B_8O1%xPj#wHvLllae=s{N7OR1f~is>l+MsG5+ zP})JI5?(!iIzGM%n_srSw(VU;+h7^Uq9;S-n5TUTeS!JvOxg+5N{DWgY$?tHP#Jhcj) zdxFt>fj;Vg(`|I=V92{jTiF@B49idsqBl$R-oZOioO-Fu%(CeFd_q2ZuQwm`vzO?8 zYcwlCSw{!IQDO?2!P1@x$+O7tIu@@d9@Q~riA`=f2)Q4qw*ntOsP!f0Gq2h9x zxo;JXJt{SV9*u*WepTo_-_uJ#Prp(|w+29%Yc1>!!Mvmj^St-WQK0+b)MslI`p@s_ zy+Ge`wT#|i*&rPcukg^VCbGdYJm8N@-s%10<8e&%PZa^@-wVhGTLtJHwe)EUS;@yG zpx3=dpV%0@Y+P$2aITfnH0GcvJaoka%R`^`!bAUSWoE2_m7_4VQc61LGeM8k(nnO% zmw~rw*`ukStpiifRS*v9z?qRKrwAkLDx88y0{nzQDa=jmoE!XQw z#dX@TTpvVw7SNGE={p)&uJ<=|aKqVwQlw|WCVA2AsJ3P&d3|qCzY4vd^){5RDdiw( zAoZ=#`_n-cU@cV21|RGlKBjRMdT)Ey(^x#-xk2|O8P8c0J;G>_59mXN z3m#A>0(_`E`_b+MOdTH5KCE*6Lpp^aesOdS!{$ZwPlhv#=slp-*&=S8^%kY3-Z)I7;#>X4)S&#WFvx&h&6dsiM*B=j4*@L42UvOk)u(eu&CpKHG zSAn$YG21x!Sv};9y;9(2X-fCxo;qo=v(5LuAL6yg8}XJUVakEV?u$5{dzMQq0j?R z7X`XI4k4Fn>8ww)6&8BKU@*{J0|s#Rb9%&pUOj(KA2OWqf;w3nf`ZC$WO+>6n&iHh zeh1R=*DXE4N8Df1FRTrI_IWtMG=pD1m#)DxS^J6&r60jx_7pJO_L^2OG`*&NHU>ZJ zF5B1`{7FzL9f(I_{f2%Gw9msqrQ*~qd{+{lo8zET5&I=Q&G?+&vC#1&b>C4ZTVTg` zEYSVPly|f%!=>-&7@+-S9B3+X4$&0~Um4wL3v*YN@d#TGfTgaC2bJ|^JUEwAFFObr zRL(}rkL)RDBjrczE0hM-p@B%uqMVHK$O^iL(f3x+t3WF!a~XYSXYjQego3v{3csPC zEP6#?--!q8>U^zsqoC~QLW6MfjR$PfcS-9mYkDRTv8AVD7#8X2B8H`Ux&?@DmOWJyz4tQEC#*Wm zK%JaX-JLdeHuwix<&>FkSBdni@8_8gtK|_o8f;|d4!Xq|*2SV~ub!S}@pkl=oxg3(cqqegrQrH=at~c?5gCqa+6qh82vElD-1X}M7c7n z9DP5p*+1*+i1Y9OwAVXhH>zesE2AQVWy=Wg#M_jcz4hREhX>9>H`Fq?Sf}E-Sq-|s z7Sba-@oinR+>F6gnH;TAWI|T4cA&BUMX5hVyVk}C2h=v`gV?{>>{t8W8x^9_u-dd^ zZ9|Z?EsXohNXOJR_&J55Jjp{?w3M!?1!3DD%$jY&NBYn`V6vW!@+ZEufHhc!@=%;H zvwz~J?@hy84er*tsQcKT2D^gy4$6Z9XlGU~Y=m7`ARXh1QPqX&bl(QiMXVfv@`*Zh z3(AJ!*z?w)r(6yGh6O0v(Pyl<4MiL3=4SA>W*g*p^=T_N2xOnKzQz}dJkW=J>c=LSRAzvwGyOgAy!;V7REqo+|$VCT?B!pqH#s*G*W`>|!m)8}y7+oK9S zNudY8*BQbZbszDFiux{H=xHzXlkI+ob)i4A`q)Su?sy^$X`qB;TJGVBA*8&O@{i$3wj;1{Dl5GUm9zntrRZ0S}n_@)>P_=aDlE72Y} zujxbAu!hl5IK}>un>-!$?>-d6vl)Nw&73+N3@tw5W1R{Y|&i-eg-*-Ls+H3E<_TzWX z*#j{|lBtmKz+d3iMXK#=R}%0yQt1HWNx(16wDt7f2K*+@;_ZIW?4qlK;TzEJn?=2b zz&9X|pGAp7A$AdPak>-10T5rcruRf$K38q z*;m;EsR6u;LAQ&(Y*BA0upsm2Oo|+eqOo%R|4hog6RP-ZHD%rpKTWt@m2>Q^W|;!s zwV>OomRsVkDG+LyFZS{a_Y?RGWv*)AJ+o}QR~zubSt{6*B6&oDoRm$u7+fv~-efGu zrZWD_uR)G1qPKX^jm2kQIvNV@bbR!qe}^K3t;NSB`SrnJRxf|Tw{?b7=m^WNkGVc` za)p!4ceFihUz+ikK+5}*({DCk^vQ`sBqyIl%_VaLdp+|#o}3InUo3k2Ghak{`*Q-# zh_ga*I9wEGrTNs+$8px3J}=X^an|IZrZ0rIn`Dj#nlZ*2>1^9crDL!bNvQvyG1kFe z5Lz_R>K~O*t8BMrzK(8=ww<>D`q3A+gyNgS6Rm!}-5pJIa-vlZ6EY@Qh3@o=m|(q< zucM#io6!1>Nmg!X>Ck zI@1yXL5D&2c;Alu7$!j08+l9wmWr6;DaAu8MIVBKot7&e5c2}l_NEuKUU!L}(*a^vr1B3_JYTMM9^o(z6{iI6TebOn zBU};p3GzG%#|-<58lQw?(|b|}a`$k5DFBO8J8i)H#Qa50|5s+K7ID(LLE zf>k1iS;$TUabVB5^J&3N2blq!^(6+-Zs>D&TPySisjuUT(kniuPSA_JFQ>l`!;bat zoGWs%T3!lp*4ylLopi3^H++SEuJUhE-1}PU*A;h2G#GK~QtZEizy*6>+h> zE0*_7r;c1}hkNi1QD7gVfTsc*ATIY)f7-neJ$hnakw^R6>2FFe`A+0o6c0noiz6vG z4{}4j?zhfSy7stR9j-)izh=sZoO{4w(Q7(V6F#uLH^XQ@=+3Io=@Rndm?>hH8S&#V zbh!K{G$NS>&IHc#5$}oKPu1R~%71sK;D;37qqz5l>P?ET?_rml;xG<5uY_`!7*wQ+ z{IUJvg8%S<;O0K`dlkRDRPbLZe-?1AK;M5;$0Ag8fj@=k!;#zcwzWLv8-`sB^Tp`p ztXCDEs(5H$D#=Hvdhvu*tU&1}lzzaMI-%EHqaoTjdLUv0>%DP^;-5wcem`66GY-yD zdWT-fj%BQ!s(^Dnzj{#kc{{{ukMd_WQ_Itc0OkJ@!fGKq24R1}a`;aq8{r)_r#Rqj zM~9E#e6+%;0WaxLN4YEwJ={qhLmgCWf9iq%vX1e+>5ybHj{_*J*`x<7=F5b zRSd^k(N1?ElySW}Q$!CoEOr_Xob_y%kb5gY%usy#59GfD?dE;aJ1@{#F7haqEJ4RF zdsOt=yIg2~1nX_4@6`xq+9gNvD~g+TE>`@GUZQ8d+E=Og+g4Y6KX9|W04t4LYK?PW z)=?U#!u`-o*og9rRJznkh(5BiYvng9{jHngtO}Le2b`Vtcp*89;QSNnOf$Idfmc~? z&z|G~-C5L)ia0-gmUb0k9(U?|SEc&y!g87UJ3k`-W$1LL)lTD-RZ4HVAa+bvxtD;m z9s3f*Zu^J;vlj zHVbYVBSi6I9|?|FVy76z8;?@$aw{R@{uN#QxkKr1wuoG|${h#J_1Ksu{AQivk6zDq zj{aEqXDEM|;?K5`X9YTQ`ca{WDV>!5rsBqr^}xBD9%G~&GkDJ`|B?%Ic7;{u{COs2 zue8#lgS8VHz3)CK^7REG&s!x<*MYO0yU$YRO0;jL`pYa&Jm_#-uIqhDE{4C;4pJ#{ z_m6&}CtuV!PJ@s2xc#Vw>FGM?Oh=BwHjwqahC4@Ap&XYkUyOY(D1A^N6|I7N_yo~^ zUG2Q0^pMA?aTS8c`mZT`HSEc2qG_v9@9*=deKmqlbU%tHfqaKklWb`4P;b*P#D>sOw2Hxc$b7zK~O* zFIn})DBh{Tz>J3}z`2|i1+)wLoIMZG*){0@abJkMu_r8E^gbI*)7FBo{2-NZp6yG! zk-H;O#m+kc!dfYWiUI)Ksx6}T02=eb8qUiPTtLr*7 zu165;P=$qBk9!^!z0Ih*=Y`v%O=Yx{_jU#gPQ1Tn_*SyyN6QG5Yr*k{p5p}`_&8tI18NBtyn@v1Kb zin+6R526u@FF!;3q0d=*m@aXS#&U0j)3O7oVk4d-La34RnkMSt{EyQpycC_YDw(oN z@toXBHJq>PO@~XZ1MWy2>}Js2a$ow9owU>A329-YaNaRQ`kOgnxqScTaRUg78zdh7^kt>b=PdVrPeMLL)Y655mUV zPV>Ob&N%jgG~y$QuT?xzD;=hIwc^cvsAdz^<3qx`Rz4Hw4y+H6F1ZYi6nvRV%W!Vt z=`DO%9oT7VqB!p{M6}J|`(gxnHe-Zs3lhF!<@*|ZtS`C;?P9(a8FZHUZq5-tv)tQ@ zP{4d!u2XC|>Y3MF=qAGUnIiOmoRNBBwP&X&#ixHSxLLlx_egU2E z?wgd%`L1hJi`<p&QZS-e~sQq*_H4^T>#Y}cNbn0KI7L@;A1~{j!--3o_BQ6n!(^soHM5vb#Oit!JrB`g3`1qYlpMfKdG|{cBN{Bty1|< zePBT4Q0l1d5w%hUbh2$f3%odXhK#6v>omu#G(Fr{%hkrTVhO$N?#etME(X1E;1haaJ7 zFTt*mucZOC~&k&B~-o`O9UWP4qi!qci*;6ut$z_imk;H-4=Xv%*Xo}Lm!wVZcLqL!D@Gw;ypzF7GerHg+5 z4vOB2y^r_a=!DV>G!mPU@pr{>^08|x^#6#z?%9gSIQ@HyeFf!Id98g9&X#iUcRMw& zU<4fKFD-dO?U%`7Bz^Tq|v^s7>JF+7^B*+TPwdM~JwH*E5ReIVQ3S}1kU&6^`ATa3iZQDpH$~N< zMi09+sH4s|9WkXCF~wmNf6wLo8rU{q!ELk{;9=;@XBECM@mm zdf548JnirAiVUj5;*XQb@uIT7Yh+MDt-vOKkscV}8tHG|JmQ6A{s31@_l7!QYoI?3 zaNXbCyxwf8zc#@2n2*nNigmk223Lwv1)2xoM_$FXL#W#AYV*aPD#*hiS74BNQ)%*Q q+BXQLeiLfh#qbNuL9P#+_yNVfp(yU&I>FD;w|K;_d7|!g{p>&4>zf_` From 9d7417a531e0ebf34f47568d5c7cdcbdc41d373a Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 25 May 2024 09:48:32 -0300 Subject: [PATCH 07/12] distributed sampler --- norch/utils/data/dataloader.py | 23 +++++++++++++++-------- norch/utils/data/distributed.py | 27 +++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 norch/utils/data/distributed.py diff --git a/norch/utils/data/dataloader.py b/norch/utils/data/dataloader.py index 01bd41e..74a2800 100644 --- a/norch/utils/data/dataloader.py +++ b/norch/utils/data/dataloader.py @@ -2,19 +2,26 @@ import numpy as np from .batch import Batch -class Dataloader(object): - - def __init__(self, dataset, batch_size=32): +class Dataloader: + + def __init__(self, dataset, batch_size=32, sampler=None): self.dataset = dataset self.batch_size = batch_size + self.sampler = sampler def __iter__(self): - starts = np.arange(0, len(self.dataset), self.batch_size) - - for start in starts: + if self.sampler is not None: + indices = list(self.sampler) + else: + indices = np.arange(len(self.dataset)) + + for start in range(0, len(indices), self.batch_size): end = start + self.batch_size - batch_size = min(end, len(self.dataset)) - start - yield Batch(self.dataset[start:end], batch_size) + batch_indices = indices[start:end] + batch_size = len(batch_indices) + yield Batch([self.dataset[i] for i in batch_indices], batch_size) def __len__(self): + if self.sampler is not None: + return len(self.sampler) // self.batch_size return len(self.dataset) // self.batch_size diff --git a/norch/utils/data/distributed.py b/norch/utils/data/distributed.py new file mode 100644 index 0000000..6af54bc --- /dev/null +++ b/norch/utils/data/distributed.py @@ -0,0 +1,27 @@ +import math +import numpy as np + +class DistributedSampler: + def __init__(self, dataset, num_replicas, rank): + self.dataset = dataset + self.num_replicas = num_replicas + self.rank = rank + self.num_samples = int(math.ceil(len(self.dataset) * 1.0 / self.num_replicas)) + self.total_size = self.num_samples * self.num_replicas + + def __iter__(self): + # Create indices for the dataset + indices = np.arange(len(self.dataset)) + + # Add extra samples to make it evenly divisible + indices = np.concatenate([indices, indices[:(self.total_size - len(indices)) % len(indices)]]) + assert len(indices) == self.total_size + + # Subsample + indices = indices[self.rank:self.total_size:self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices) + + def __len__(self): + return self.num_samples \ No newline at end of file From 56fff47001a7ea728b8cb76612bfbc7d2dd295b5 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 25 May 2024 09:55:02 -0300 Subject: [PATCH 08/12] fix import data module --- norch/__init__.py | 1 + norch/utils/data/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/norch/__init__.py b/norch/__init__.py index c2a3122..80da58b 100644 --- a/norch/__init__.py +++ b/norch/__init__.py @@ -3,6 +3,7 @@ from .nn import * from .optim import * from .utils import * from .norchvision import * +from . import data __version__ = "0.0.4" __author__ = 'Lucas de Lima Nogueira' diff --git a/norch/utils/data/__init__.py b/norch/utils/data/__init__.py index 2397554..e5ea955 100644 --- a/norch/utils/data/__init__.py +++ b/norch/utils/data/__init__.py @@ -1,4 +1,5 @@ from .dataset import * from .example import * from .dataloader import * -from .batch import * \ No newline at end of file +from .batch import * +from .distributed import * \ No newline at end of file From e590f57555f731de56d79f8f7621d507905a39de Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 25 May 2024 10:53:09 -0300 Subject: [PATCH 09/12] implemented distributed sampler --- norch/__init__.py | 2 +- norch/__pycache__/__init__.cpython-38.pyc | Bin 365 -> 356 bytes norch/distributed/distributed.py | 6 ++++ norch/nn/__pycache__/__init__.cpython-38.pyc | Bin 241 -> 261 bytes norch/nn/__pycache__/parameter.cpython-38.pyc | Bin 728 -> 736 bytes norch/nn/parameter.py | 4 +-- norch/utils/__init__.py | 4 +-- norch/utils/data/dataloader.py | 18 ++++++----- norch/utils/data/dataset.py | 2 +- norch/utils/data/distributed.py | 13 ++++---- norch/utils/{utils.py => functions.py} | 0 train.py | 29 +++++++++++++++++- 12 files changed, 56 insertions(+), 22 deletions(-) rename norch/utils/{utils.py => functions.py} (100%) diff --git a/norch/__init__.py b/norch/__init__.py index 80da58b..2aae8f0 100644 --- a/norch/__init__.py +++ b/norch/__init__.py @@ -3,7 +3,7 @@ from .nn import * from .optim import * from .utils import * from .norchvision import * -from . import data +from .utils import * __version__ = "0.0.4" __author__ = 'Lucas de Lima Nogueira' diff --git a/norch/__pycache__/__init__.cpython-38.pyc b/norch/__pycache__/__init__.cpython-38.pyc index 71962d9e9e29bd7e443f82302b507406eb8b5080..9ef633144738bc5135fd57fbb835823ece228f9b 100644 GIT binary patch delta 180 zcmaFM^n{5wl$V!_0SMY32By85$SWJ_0OX`Fq%h_%gfOHq0cl1k%><>Hp)?B;pE-&( zg(Zp&h}lzEgBdj0UV_ZjWW2?w^^y_Dn%JtS#1X}lmtT~ep;wZcSDarI#gvy9#hPDG pl9_vpwX`HNr});yv!-n9K-ER;lQ|fJMFfCM4n{a+Vqs)q1OU6HCH^ns~^RjSVPO#5VD7un-@R#ly$}flMrnEQ|ou C-zsVV diff --git a/norch/distributed/distributed.py b/norch/distributed/distributed.py index bd87bc3..98bc2a0 100644 --- a/norch/distributed/distributed.py +++ b/norch/distributed/distributed.py @@ -9,6 +9,12 @@ def init_process_group(rank, world_size, backend='nccl'): Tensor._C.init_process_group(rank, world_size) +def get_rank(): + return os.getenv('OMPI_COMM_WORLD_RANK', 0) + +def get_world_size(): + return os.getenv('OMPI_COMM_WORLD_SIZE', 1) + def broadcast_tensor(tensor, src=0): Tensor._C.broadcast_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int] diff --git a/norch/nn/__pycache__/__init__.cpython-38.pyc b/norch/nn/__pycache__/__init__.cpython-38.pyc index 5b7fd1c7b09471efcfce61414f7fe60ec9c05c2f..9af1d40ac65a2aab524d9d068a81346d9cc372be 100644 GIT binary patch delta 82 zcmey!*viBk%FD~e00cdk0@KPS^2##mOw?A9Vn|`kVF_VKVG3r@WPZsAl-Fdu#i-?{ d$vQDnoimD~Ah9ShCnt4cdjby!P?CcY0s;M45hVZs delta 62 zcmZo=`pC!|%FD~e00g`~-f3|Yd1V<@CTgn)F$FVdGQVU5ifc07V$|}}WSN+$K5pF diff --git a/norch/nn/parameter.py b/norch/nn/parameter.py index 2324a1a..5a29f3f 100644 --- a/norch/nn/parameter.py +++ b/norch/nn/parameter.py @@ -1,5 +1,5 @@ from norch.tensor import Tensor -from norch.utils import utils +from norch.utils import functions import random class Parameter(Tensor): @@ -7,5 +7,5 @@ class Parameter(Tensor): A parameter is a trainable tensor. """ def __init__(self, shape): - data = utils.generate_random_list(shape=shape) + data = functions.generate_random_list(shape=shape) super().__init__(data, requires_grad=True) \ No newline at end of file diff --git a/norch/utils/__init__.py b/norch/utils/__init__.py index c6adfd1..b6b5a68 100644 --- a/norch/utils/__init__.py +++ b/norch/utils/__init__.py @@ -1,2 +1,2 @@ -from .utils import * -from .data import * \ No newline at end of file +from .data import * +from .functions import * \ No newline at end of file diff --git a/norch/utils/data/dataloader.py b/norch/utils/data/dataloader.py index 74a2800..13c02c5 100644 --- a/norch/utils/data/dataloader.py +++ b/norch/utils/data/dataloader.py @@ -11,17 +11,19 @@ class Dataloader: def __iter__(self): if self.sampler is not None: - indices = list(self.sampler) + indices = iter(self.sampler) + else: - indices = np.arange(len(self.dataset)) - - for start in range(0, len(indices), self.batch_size): - end = start + self.batch_size - batch_indices = indices[start:end] - batch_size = len(batch_indices) - yield Batch([self.dataset[i] for i in batch_indices], batch_size) + indices = range(len(self.dataset)) + + for idx in indices: + start = idx * self.batch_size + end = min(start + self.batch_size, len(self.dataset)) + yield Batch(self.dataset[start:end], end - start) + def __len__(self): if self.sampler is not None: return len(self.sampler) // self.batch_size + return len(self.dataset) // self.batch_size diff --git a/norch/utils/data/dataset.py b/norch/utils/data/dataset.py index d38c1ea..2130d52 100644 --- a/norch/utils/data/dataset.py +++ b/norch/utils/data/dataset.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod import os -from norch.utils import extract_to_dir, download_from_url +from norch.utils.functions import extract_to_dir, download_from_url from .example import Example import norch diff --git a/norch/utils/data/distributed.py b/norch/utils/data/distributed.py index 6af54bc..c83dfc4 100644 --- a/norch/utils/data/distributed.py +++ b/norch/utils/data/distributed.py @@ -1,5 +1,5 @@ -import math import numpy as np +import math class DistributedSampler: def __init__(self, dataset, num_replicas, rank): @@ -10,18 +10,17 @@ class DistributedSampler: self.total_size = self.num_samples * self.num_replicas def __iter__(self): - # Create indices for the dataset - indices = np.arange(len(self.dataset)) + indices = list(range(len(self.dataset))) # Add extra samples to make it evenly divisible - indices = np.concatenate([indices, indices[:(self.total_size - len(indices)) % len(indices)]]) + indices = indices[:self.total_size] assert len(indices) == self.total_size - # Subsample + # subsample indices = indices[self.rank:self.total_size:self.num_replicas] assert len(indices) == self.num_samples - + return iter(indices) def __len__(self): - return self.num_samples \ No newline at end of file + return self.num_samples diff --git a/norch/utils/utils.py b/norch/utils/functions.py similarity index 100% rename from norch/utils/utils.py rename to norch/utils/functions.py diff --git a/train.py b/train.py index 8f07a41..7f9a613 100644 --- a/train.py +++ b/train.py @@ -1,4 +1,4 @@ -import os +"""import os import norch import norch.distributed as dist @@ -29,3 +29,30 @@ def main(): if __name__ == "__main__": main() + +""" +import norch +import matplotlib.pyplot as plt +import numpy as np +import random + +train_data, test_data = norch.norchvision.datasets.MNIST.splits() +train_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=10, rank=2) +train_loader = norch.utils.data.Dataloader(train_data, batch_size = 1, sampler=train_sampler) +input_sample, target_sample = train_data[0] + +fig = plt.figure(figsize = (20, 10)) +columns = 4 +rows = 2 + +# Choose a random image +for image_index, batch in enumerate(train_loader): + + image, label = batch + fig.add_subplot(rows, columns, image_index+1) + plt.imshow(np.array(image).reshape(28, 28)) + plt.title(label) + plt.axis('off') + if image_index > 6: + break +plt.show() \ No newline at end of file From 10a88fe8ca6ccd660b89c416d2c61be6bf2f2949 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Sat, 25 May 2024 11:55:59 -0300 Subject: [PATCH 10/12] unit test dataloader and distributedsampler --- README.md | 10 ++-- examples/train.ipynb | 6 +- norch/norchvision/transforms.py | 2 +- norch/utils/data/dataloader.py | 2 +- tests/test_dataset.py | 93 +++++++++++++++++++++++++++++++ tests/test_distributed.py | 98 +++++++++++++++++++++++++++++++++ tests/test_nn.py | 3 +- train.py | 3 +- 8 files changed, 204 insertions(+), 13 deletions(-) create mode 100644 tests/test_dataset.py create mode 100644 tests/test_distributed.py diff --git a/README.md b/README.md index dd310b5..5ed5ebd 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ class MyModel(nn.Module): ### 3.3 - Example training ```python import norch -from norch.utils.data.dataloader import Dataloader +from norch.utils.data.dataloader import DataLoader from norch.norchvision import transforms import norch import norch.nn as nn @@ -77,21 +77,21 @@ BATCH_SIZE = 32 device = "cuda" #cpu epochs = 10 -transform = transforms.Sequential( +transform = transforms.Compose( [ transforms.ToTensor(), transforms.Reshape([-1, 784, 1]) ] ) -target_transform = transforms.Sequential( +target_transform = transforms.Compose( [ transforms.ToTensor() ] ) train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transform, target_transform=target_transform) -train_loader = Dataloader(train_data, batch_size = BATCH_SIZE) +train_loader = DataLoader(train_data, batch_size = BATCH_SIZE) class MyModel(nn.Module): def __init__(self): @@ -146,4 +146,4 @@ for epoch in range(epochs): | Loss | in progress |
  • [x] MSE
  • [X] Cross Entropy
| | Data | in progress |
  • [X] Dataset
  • [X] Batch
  • [X] Iterator
| | Convolutional Neural Network | in progress |
  • [ ] Conv2d
  • [ ] MaxPool2d
  • [ ] Dropout
| -| Distributed | in progress |
  • [ ] Distributed Data Parallel
+| Distributed | in progress |
    >
  • [ ] All reduce
  • [ ] DistributedDataParallel
    • >
  • [ ] DistributedSampler
diff --git a/examples/train.ipynb b/examples/train.ipynb index c53ea86..427933a 100644 --- a/examples/train.ipynb +++ b/examples/train.ipynb @@ -33,7 +33,7 @@ "import norch\n", "import norch.nn as nn\n", "import norch.optim as optim\n", - "from norch.utils.data.dataloader import Dataloader\n", + "from norch.utils.data.dataloader import DataLoader\n", "from norch.norchvision import transforms\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", @@ -102,7 +102,7 @@ "device = \"cuda\" #cpu\n", "epochs = 10\n", "\n", - "transform = transforms.Sequential(\n", + "transform = transforms.Compose(\n", " [\n", " transforms.ToTensor(),\n", " transforms.Reshape([-1, 784, 1])\n", @@ -116,7 +116,7 @@ ")\n", "\n", "train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transform, target_transform=target_transform)\n", - "train_loader = Dataloader(train_data, batch_size = BATCH_SIZE)\n", + "train_loader = DataLoader(train_data, batch_size = BATCH_SIZE)\n", "\n", "class MyModel(nn.Module):\n", " def __init__(self):\n", diff --git a/norch/norchvision/transforms.py b/norch/norchvision/transforms.py index 03dafad..944e426 100644 --- a/norch/norchvision/transforms.py +++ b/norch/norchvision/transforms.py @@ -11,7 +11,7 @@ class Reshape: def __call__(self, x): return x.reshape(self.shape) -class Sequential: +class Compose: def __init__(self, transforms): self.transforms = transforms diff --git a/norch/utils/data/dataloader.py b/norch/utils/data/dataloader.py index 13c02c5..8fb8142 100644 --- a/norch/utils/data/dataloader.py +++ b/norch/utils/data/dataloader.py @@ -2,7 +2,7 @@ import numpy as np from .batch import Batch -class Dataloader: +class DataLoader: def __init__(self, dataset, batch_size=32, sampler=None): self.dataset = dataset diff --git a/tests/test_dataset.py b/tests/test_dataset.py new file mode 100644 index 0000000..aceb53c --- /dev/null +++ b/tests/test_dataset.py @@ -0,0 +1,93 @@ +import unittest +import norch +from norch.utils import utils_unittests as utils +import torch +import torchvision +import numpy as np +from norch.norchvision import transforms as norch_transforms +from torchvision import transforms as torch_transforms + +class TestDataset(unittest.TestCase): + def test_dataloader_batchsize_1(self): + transforms = norch_transforms.Compose( + [ + norch_transforms.ToTensor(), + ] + ) + train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms) + train_loader = norch.utils.data.DataLoader(train_data, batch_size = 1) + + labels_norch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_norch.append(utils.to_torch(label)) + + if i > 10: + break + + transforms = torch_transforms.Compose( + [ + torch_transforms.ToTensor(), + ] + ) + + train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms) + train_loader = torch.utils.data.DataLoader(train_data, batch_size = 1) + + labels_torch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_torch.append(label) + + if i > 10: + break + + for label_norch, label_torch in zip(labels_norch, labels_torch): + self.assertTrue(utils.compare_torch(label_norch, label_torch)) + + + def test_dataloader_batchsize_32(self): + transforms = norch_transforms.Compose( + [ + norch_transforms.ToTensor(), + ] + ) + train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms) + train_loader = norch.utils.data.DataLoader(train_data, batch_size = 32) + + labels_norch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_norch.append(utils.to_torch(label)) + + if i > 10: + break + + transforms = torch_transforms.Compose( + [ + torch_transforms.ToTensor(), + ] + ) + + train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms) + train_loader = torch.utils.data.DataLoader(train_data, batch_size = 32) + + labels_torch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_torch.append(label) + + if i > 10: + break + + for label_norch, label_torch in zip(labels_norch, labels_torch): + self.assertTrue(utils.compare_torch(label_norch, label_torch)) + + + + + diff --git a/tests/test_distributed.py b/tests/test_distributed.py new file mode 100644 index 0000000..86a429d --- /dev/null +++ b/tests/test_distributed.py @@ -0,0 +1,98 @@ +import unittest +import norch +from norch.utils import utils_unittests as utils +import torch +import torchvision +import numpy as np +from norch.norchvision import transforms as norch_transforms +from torchvision import transforms as torch_transforms + +class TestDistributed(unittest.TestCase): + def test_distributed_sampler_batch_1(self): + transforms = norch_transforms.Compose( + [ + norch_transforms.ToTensor(), + ] + ) + train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms) + distributed_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2) + train_loader = norch.utils.data.DataLoader(train_data, batch_size = 1, sampler=distributed_sampler) + + labels_norch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_norch.append(utils.to_torch(label)) + + if i > 10: + break + + transforms = torch_transforms.Compose( + [ + torch_transforms.ToTensor(), + ] + ) + + train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms) + distributed_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2, shuffle=False) + train_loader = torch.utils.data.DataLoader(train_data, batch_size = 1, sampler=distributed_sampler) + + labels_torch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_torch.append(label) + + if i > 10: + break + + for label_norch, label_torch in zip(labels_norch, labels_torch): + self.assertTrue(utils.compare_torch(label_norch, label_torch)) + + + def test_distributed_sampler_batch_32(self): + transforms = norch_transforms.Compose( + [ + norch_transforms.ToTensor(), + ] + ) + train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms) + distributed_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2) + train_loader = norch.utils.data.DataLoader(train_data, batch_size = 32, sampler=distributed_sampler) + + labels_norch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_norch.append(utils.to_torch(label)) + + if i > 10: + break + + transforms = torch_transforms.Compose( + [ + torch_transforms.ToTensor(), + ] + ) + + train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms) + distributed_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2, shuffle=False) + train_loader = torch.utils.data.DataLoader(train_data, batch_size = 32, sampler=distributed_sampler) + + labels_torch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_torch.append(label) + + if i > 10: + break + + for label_norch, label_torch in zip(labels_norch, labels_torch): + print(label_norch, label_torch) + self.assertTrue(utils.compare_torch(label_norch, label_torch)) + + + + + diff --git a/tests/test_nn.py b/tests/test_nn.py index 532f5b6..50c1a79 100644 --- a/tests/test_nn.py +++ b/tests/test_nn.py @@ -200,5 +200,4 @@ class TestNNModuleActivationFn(unittest.TestCase): softmax_torch_expected = softmax_fn_torch.forward(torch_input) # Compare the results - self.assertTrue(utils.compare_torch(softmax_torch_result, softmax_torch_expected)) - + self.assertTrue(utils.compare_torch(softmax_torch_result, softmax_torch_expected)) \ No newline at end of file diff --git a/train.py b/train.py index 7f9a613..e999667 100644 --- a/train.py +++ b/train.py @@ -35,8 +35,9 @@ import norch import matplotlib.pyplot as plt import numpy as np import random +from norch.norchvision import transforms -train_data, test_data = norch.norchvision.datasets.MNIST.splits() +train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms.ToTensor()) train_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=10, rank=2) train_loader = norch.utils.data.Dataloader(train_data, batch_size = 1, sampler=train_sampler) input_sample, target_sample = train_data[0] From e3bb882d3c06b20b7b347382ad0b791002f93b86 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Mon, 27 May 2024 15:08:27 -0300 Subject: [PATCH 11/12] fix distributed dataloader --- norch/utils/data/dataloader.py | 21 +++++----- norch/utils/data/distributed.py | 2 + tests/test_distributed.py | 71 ++++++++++++++++++++++++++------- train.py | 2 +- 4 files changed, 69 insertions(+), 27 deletions(-) diff --git a/norch/utils/data/dataloader.py b/norch/utils/data/dataloader.py index 8fb8142..fa823ad 100644 --- a/norch/utils/data/dataloader.py +++ b/norch/utils/data/dataloader.py @@ -4,24 +4,23 @@ from .batch import Batch class DataLoader: - def __init__(self, dataset, batch_size=32, sampler=None): + def __init__(self, dataset, batch_size, sampler=None): self.dataset = dataset self.batch_size = batch_size self.sampler = sampler def __iter__(self): if self.sampler is not None: - indices = iter(self.sampler) - + indices = list(iter(self.sampler)) + else: - indices = range(len(self.dataset)) - - for idx in indices: - start = idx * self.batch_size - end = min(start + self.batch_size, len(self.dataset)) - yield Batch(self.dataset[start:end], end - start) - - + indices = list(range(len(self.dataset))) + + for i in range(0, len(indices), self.batch_size): + batch_indices = indices[i:i + self.batch_size] + batch_data = self.dataset[batch_indices] + yield Batch(batch_data, len(batch_data)) + def __len__(self): if self.sampler is not None: return len(self.sampler) // self.batch_size diff --git a/norch/utils/data/distributed.py b/norch/utils/data/distributed.py index c83dfc4..5d9f294 100644 --- a/norch/utils/data/distributed.py +++ b/norch/utils/data/distributed.py @@ -14,6 +14,8 @@ class DistributedSampler: # Add extra samples to make it evenly divisible indices = indices[:self.total_size] + if len(indices) < self.total_size: + indices += indices[:(self.total_size - len(indices))] assert len(indices) == self.total_size # subsample diff --git a/tests/test_distributed.py b/tests/test_distributed.py index 86a429d..342d87f 100644 --- a/tests/test_distributed.py +++ b/tests/test_distributed.py @@ -4,11 +4,12 @@ from norch.utils import utils_unittests as utils import torch import torchvision import numpy as np +import os from norch.norchvision import transforms as norch_transforms from torchvision import transforms as torch_transforms class TestDistributed(unittest.TestCase): - def test_distributed_sampler_batch_1(self): + """def test_distributed_sampler_batch_1(self): transforms = norch_transforms.Compose( [ norch_transforms.ToTensor(), @@ -16,7 +17,7 @@ class TestDistributed(unittest.TestCase): ) train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms) distributed_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2) - train_loader = norch.utils.data.DataLoader(train_data, batch_size = 1, sampler=distributed_sampler) + train_loader = norch.utils.data.DataLoader(train_data, batch_size=1, sampler=distributed_sampler) labels_norch = [] for i, batch in enumerate(train_loader): @@ -24,7 +25,7 @@ class TestDistributed(unittest.TestCase): image, label = batch labels_norch.append(utils.to_torch(label)) - if i > 10: + if i > 100: break transforms = torch_transforms.Compose( @@ -35,7 +36,7 @@ class TestDistributed(unittest.TestCase): train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms) distributed_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2, shuffle=False) - train_loader = torch.utils.data.DataLoader(train_data, batch_size = 1, sampler=distributed_sampler) + train_loader = torch.utils.data.DataLoader(train_data, batch_size=1, sampler=distributed_sampler) labels_torch = [] for i, batch in enumerate(train_loader): @@ -43,22 +44,63 @@ class TestDistributed(unittest.TestCase): image, label = batch labels_torch.append(label) - if i > 10: + if i > 100: + break + + for label_norch, label_torch in zip(labels_norch, labels_torch): + self.assertTrue(utils.compare_torch(label_norch, label_torch))""" + + + def test_distributed_sampler_batch_32_rank_2(self): + transforms = norch_transforms.Compose( + [ + norch_transforms.ToTensor(), + ] + ) + train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms) + distributed_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2) + train_loader = norch.utils.data.DataLoader(train_data, batch_size=32, sampler=distributed_sampler) + + labels_norch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_norch.append(utils.to_torch(label)) + + if i > 100: + break + + transforms = torch_transforms.Compose( + [ + torch_transforms.ToTensor(), + ] + ) + + train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms) + distributed_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2, shuffle=False) + train_loader = torch.utils.data.DataLoader(train_data, batch_size=32, sampler=distributed_sampler) + + labels_torch = [] + for i, batch in enumerate(train_loader): + + image, label = batch + labels_torch.append(label) + + if i > 100: break for label_norch, label_torch in zip(labels_norch, labels_torch): self.assertTrue(utils.compare_torch(label_norch, label_torch)) - - def test_distributed_sampler_batch_32(self): + def test_distributed_sampler_batch_32_rank_7(self): transforms = norch_transforms.Compose( [ norch_transforms.ToTensor(), ] ) train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms, target_transform=transforms) - distributed_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2) - train_loader = norch.utils.data.DataLoader(train_data, batch_size = 32, sampler=distributed_sampler) + distributed_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=7) + train_loader = norch.utils.data.DataLoader(train_data, batch_size=32, sampler=distributed_sampler) labels_norch = [] for i, batch in enumerate(train_loader): @@ -66,7 +108,7 @@ class TestDistributed(unittest.TestCase): image, label = batch labels_norch.append(utils.to_torch(label)) - if i > 10: + if i > 100: break transforms = torch_transforms.Compose( @@ -76,20 +118,19 @@ class TestDistributed(unittest.TestCase): ) train_data = torchvision.datasets.MNIST(root='./.data/', download=True, transform=transforms) - distributed_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=2, shuffle=False) - train_loader = torch.utils.data.DataLoader(train_data, batch_size = 32, sampler=distributed_sampler) - + distributed_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=8, rank=7, shuffle=False) + train_loader = torch.utils.data.DataLoader(train_data, batch_size=32, sampler=distributed_sampler) + labels_torch = [] for i, batch in enumerate(train_loader): image, label = batch labels_torch.append(label) - if i > 10: + if i > 100: break for label_norch, label_torch in zip(labels_norch, labels_torch): - print(label_norch, label_torch) self.assertTrue(utils.compare_torch(label_norch, label_torch)) diff --git a/train.py b/train.py index e999667..9ad1187 100644 --- a/train.py +++ b/train.py @@ -39,7 +39,7 @@ from norch.norchvision import transforms train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transforms.ToTensor()) train_sampler = norch.utils.data.distributed.DistributedSampler(dataset=train_data, num_replicas=10, rank=2) -train_loader = norch.utils.data.Dataloader(train_data, batch_size = 1, sampler=train_sampler) +train_loader = norch.utils.data.Dataloader(train_data, batch_size = 50, sampler=train_sampler) input_sample, target_sample = train_data[0] fig = plt.figure(figsize = (20, 10)) From 240e603c0c7084b88d50b6db7e009695e5a0a0b1 Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Mon, 27 May 2024 15:09:07 -0300 Subject: [PATCH 12/12] fix distributed dataloader --- tests/test_distributed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_distributed.py b/tests/test_distributed.py index 342d87f..1b16a20 100644 --- a/tests/test_distributed.py +++ b/tests/test_distributed.py @@ -9,7 +9,7 @@ from norch.norchvision import transforms as norch_transforms from torchvision import transforms as torch_transforms class TestDistributed(unittest.TestCase): - """def test_distributed_sampler_batch_1(self): + def test_distributed_sampler_batch_1(self): transforms = norch_transforms.Compose( [ norch_transforms.ToTensor(), @@ -48,7 +48,7 @@ class TestDistributed(unittest.TestCase): break for label_norch, label_torch in zip(labels_norch, labels_torch): - self.assertTrue(utils.compare_torch(label_norch, label_torch))""" + self.assertTrue(utils.compare_torch(label_norch, label_torch)) def test_distributed_sampler_batch_32_rank_2(self):