From 537fe5bd237c164bd42a7d917495a5cd8771b827 Mon Sep 17 00:00:00 2001 From: plaguss Date: Thu, 9 Jan 2025 13:05:03 +0100 Subject: [PATCH 01/12] Update README to include the training and eval recipes --- README.md | 2 +- recipes/README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a99b65e..688bb25a 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ To get started, we recommend the following: The initial release of Search and Learn will focus on the following techniques: * **Search against verifiers:** guide LLMs to search for solutions to "verifiable problems" (math, code) by using a stepwise or process reward model to score each step. Includes techniques like Best-of-N sampling and tree search. -* **Training process reward models (coming soon!):** train reward models to provide a sequence of scores, one for each step of the reasoning process. This ability to provide fine-grained feedback makes PRMs a natural fit for search methods with LLMs. +* **Training process reward models:** train reward models to provide a sequence of scores, one for each step of the reasoning process. This ability to provide fine-grained feedback makes PRMs a natural fit for search methods with LLMs. # Installation instructions diff --git a/recipes/README.md b/recipes/README.md index 0db7b540..26154f1c 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -181,4 +181,6 @@ To get the final numbers for the evalations, we use a [fork](https://github.com/ > [!NOTE] > We are working on switching the Qwen-Math parser for an improved one in `lighteval`. Once we have validated the results, we will be able to have a stand-alone evaluation script directly in `search-and-learn`: stay tuned! +## Training Process Reward Models +The [`training` README](training/README.md) is a guide to both train PRMs using [TRL](https://github.com/huggingface/trl) and evaluate them using the [ProcessBench](https://arxiv.org/abs/2412.06559) benchmark, with the code used to fine-tune [Qwen/Qwen2.5-Math-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-1.5B-Instruct) and [Qwen/Qwen2.5-Math-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-7B-Instruct) and evaluate the final models. From 7ba786d0917b617185b0b4574b5803389ebdb5c7 Mon Sep 17 00:00:00 2001 From: plaguss Date: Thu, 9 Jan 2025 13:05:18 +0100 Subject: [PATCH 02/12] Add training and evaluation recipes --- .../prm_qwen_math_1p5b_instruct.sh | 23 ++++ .../prm_qwen_math_1p5b_instruct.slurm | 40 ++++++ .../prm_qwen_math_7b_instruct.sh | 23 ++++ .../prm_qwen_math_7b_instruct.slurm | 40 ++++++ recipes/training/README.md | 121 ++++++++++++++++++ recipes/training/assets/wandb.png | Bin 0 -> 67356 bytes 6 files changed, 247 insertions(+) create mode 100644 recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.sh create mode 100644 recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm create mode 100644 recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.sh create mode 100644 recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm create mode 100644 recipes/training/README.md create mode 100644 recipes/training/assets/wandb.png diff --git a/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.sh b/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.sh new file mode 100644 index 00000000..ed34fc53 --- /dev/null +++ b/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.sh @@ -0,0 +1,23 @@ +accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml examples/scripts/prm.py \ + --run_name="prm/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ + --model_name_or_path="Qwen/Qwen2.5-Math-1.5B-Instruct" \ + --dataset_name="plaguss/prm800k-trl-dedup" \ + --output_dir="prm/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ + --report_to="wandb" \ + --learning_rate=1.0e-06 \ + --per_device_train_batch_size=16 \ + --per_device_eval_batch_size=8 \ + --do_eval \ + --eval_strategy="steps" \ + --eval_steps=50 \ + --gradient_accumulation_steps=4 \ + --logging_steps=25 \ + --num_train_epochs=1 \ + --max_steps=-1 \ + --warmup_steps=50 \ + --push_to_hub \ + --gradient_checkpointing \ + --max_length=2048 \ + --step_separator="\n\n" \ + --bf16 \ + --dataset_num_proc=8 diff --git a/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm b/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm new file mode 100644 index 00000000..da59050e --- /dev/null +++ b/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm @@ -0,0 +1,40 @@ +#!/bin/bash +#SBATCH --job-name=PRM-qwen-1.5 +#SBATCH --partition=hopper-prod +#SBATCH --qos=normal +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gpus-per-node=8 +#SBATCH --output=./logs/%x-%j.out +#SBATCH --err=./logs/%x-%j.err +#SBATCH --time=02-00:00:00 + +set -ex + +module load cuda/12.1 + +source .venv/bin/activate + +srun --nodes=1 --ntasks=1 --export=ALL,ACCELERATE_LOG_LEVEL=info accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml examples/scripts/prm.py \ + --run_name="prm/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ + --model_name_or_path="Qwen/Qwen2.5-Math-1.5B-Instruct" \ + --dataset_name="plaguss/prm800k-trl-dedup" \ + --output_dir="prm/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ + --report_to="wandb" \ + --learning_rate=1.0e-06 \ + --per_device_train_batch_size=16 \ + --per_device_eval_batch_size=8 \ + --do_eval \ + --eval_strategy="steps" \ + --eval_steps=50 \ + --gradient_accumulation_steps=4 \ + --logging_steps=25 \ + --num_train_epochs=1 \ + --max_steps=-1 \ + --warmup_steps=50 \ + --push_to_hub \ + --gradient_checkpointing \ + --max_length=2048 \ + --step_separator="\n\n" \ + --bf16 \ + --dataset_num_proc=8 diff --git a/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.sh b/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.sh new file mode 100644 index 00000000..25a93746 --- /dev/null +++ b/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.sh @@ -0,0 +1,23 @@ +accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml examples/scripts/prm.py \ + --run_name="prm/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ + --model_name_or_path="Qwen/Qwen2.5-Math-7B-Instruct" \ + --dataset_name="plaguss/prm800k-trl-dedup" \ + --output_dir="prm/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ + --report_to="wandb" \ + --learning_rate=1.0e-06 \ + --per_device_train_batch_size=8 \ + --per_device_eval_batch_size=4 \ + --do_eval \ + --eval_strategy="steps" \ + --eval_steps=50 \ + --gradient_accumulation_steps=4 \ + --logging_steps=25 \ + --num_train_epochs=1 \ + --max_steps=-1 \ + --warmup_steps=50 \ + --push_to_hub \ + --gradient_checkpointing \ + --max_length=2048 \ + --step_separator="\n\n" \ + --bf16 \ + --dataset_num_proc=8 diff --git a/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm b/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm new file mode 100644 index 00000000..5ee5d7d0 --- /dev/null +++ b/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm @@ -0,0 +1,40 @@ +#!/bin/bash +#SBATCH --job-name=PRM-qwen-7b +#SBATCH --partition=hopper-prod +#SBATCH --qos=normal +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gpus-per-node=8 +#SBATCH --output=./logs/%x-%j.out +#SBATCH --err=./logs/%x-%j.err +#SBATCH --time=02-00:00:00 + +set -ex + +module load cuda/12.1 + +source .venv/bin/activate + +srun --nodes=1 --ntasks=1 --export=ALL,ACCELERATE_LOG_LEVEL=info accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml examples/scripts/prm.py \ + --run_name="prm/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ + --model_name_or_path="Qwen/Qwen2.5-Math-7B-Instruct" \ + --dataset_name="plaguss/prm800k-trl-dedup" \ + --output_dir="prm/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ + --report_to="wandb" \ + --learning_rate=1.0e-06 \ + --per_device_train_batch_size=8 \ + --per_device_eval_batch_size=4 \ + --do_eval \ + --eval_strategy="steps" \ + --eval_steps=50 \ + --gradient_accumulation_steps=4 \ + --logging_steps=25 \ + --num_train_epochs=1 \ + --max_steps=-1 \ + --warmup_steps=50 \ + --push_to_hub \ + --gradient_checkpointing \ + --max_length=2048 \ + --step_separator="\n\n" \ + --bf16 \ + --dataset_num_proc=8 diff --git a/recipes/training/README.md b/recipes/training/README.md new file mode 100644 index 00000000..c0249531 --- /dev/null +++ b/recipes/training/README.md @@ -0,0 +1,121 @@ +# Training your own PRM + +In TRL v0.13.0, the [PRM trainer](https://huggingface.co/docs/trl/v0.13.0/en/prm_trainer) was introduced, simplifying the process of training your own PRM models. This section shows how to fine-tune your own models using TRL, and running the [ProcessBench](https://arxiv.org/abs/2412.06559) benchmark. + +## Fine-tuning with TRL + +Using `uv` (`pip` or any other package installer should work similar), clone the [TRL](https://github.com/huggingface/trl) repository, create a virtual environment and install the dependencies: + +```bash +git clone https://github.com/huggingface/trl.git +cd trl +uv venv .venv --python 3.12 +source .venv/bin/activate +uv pip install . +``` + +And you can navigate to the folders under `/training`. Two folders can be found containing a fine tune of [Qwen/Qwen2.5-Math-1.5B-Instruct](...) on [plaguss/prm800k-trl-dedup](...), and [Qwen/Qwen2.5-Math-7B-Instruct](...). The trainings were run in a slurm cluster with 8xH100, but they can be adapted to the number of available GPUs and resources: + +| Model | Training Script | +| :--- | :--- | +| Qwen/Qwen2.5-Math-1.5B-Instruct | [Script](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.sh) / [Slurm](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm)| +| Qwen/Qwen2.5-Math-7B-Instruct | [Script](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_7b_instruct.sh) / [Slurm](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm)| + +
+Click for a sample WandB run. + +The figure contains the weights and biases loss curves for the previous models: + +![alt text](./assets/wandb.png) + +
+ +### Models + +The following two models were fine tuned using the scripts, examples of use can be found in the corresponding repository: + +- [plaguss/Qwen2.5-Math-7B-Instruct-PRM-0.2](https://huggingface.co/plaguss/Qwen2.5-Math-7B-Instruct-PRM-0.2) + +- [plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2](https://huggingface.co/plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2) + +## Benchmarking with ProcessBench + +Using `uv` (`pip` or any other package installer should work similar), clone the [ProcessBench](https://github.com/huggingface/ProcessBench) fork that includes the script to evaluate TRL models, create a virtual environment, and install the requirements: + +```bash +git clone https://github.com/huggingface/ProcessBench.git +uv venv .venv --python 3.12 +source .venv/bin/activate +uv pip install -r requirements-trl.txt +``` + +All the experiments were run in 1xH100, the batch size should be adjusted to your capacity (for reference, a batch size of 256 for Qwen2.5-Math-1.5B-Instruct took near 0.5h, and a batch size of 128 for Qwen2.5-Math-7B-Instruct near 2h). Navigate to the `/code` folder, and run the `run_eval_prm_trl.py` script: + +```bash +cd code/ + +python run_eval_prm_trl.py \ + --model_name "plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ # Model to evaluate + --output_dir "./outputs" \ # Directory to save the results + --batch_size 256 \ # Batch size + --sep "\n\n" # Separator, MUST be the same used during training +``` + +Click the following buttons to see the example runs and results for the models: + +
+plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2 + +```bash +python run_eval_prm_trl.py \ + --config "all" \ + --model_name "plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ + --output_dir "./outputs" \ + --batch_size 256 \ + --sep "\n\n" +``` + +A report like the following will appear in the console: + +```bash +Individual Results: +---------------------------------------------------------------------- +gsm8k -> Precision: 22.71 Recall: 93.78 F1 Score: 36.56 +math -> Precision: 38.22 Recall: 70.69 F1 Score: 49.61 +olympiadbench -> Precision: 27.08 Recall: 53.98 F1 Score: 36.07 +omnimath -> Precision: 27.93 Recall: 54.77 F1 Score: 37.00 +Weighted Averages: +---------------------------------------------------------------------- +Weighted -> Precision: 30.09 Recall: 63.81 F1 Score: 40.38 +``` + +
+ + +
+plaguss/Qwen2.5-Math-7B-Instruct-PRM-0.2 + +```bash +python run_eval_prm_trl.py \ + --config "all" \ + --model_name "plaguss/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ + --output_dir "./outputs" \ + --batch_size 128 \ + --sep "\n\n" + +``` + +A report like the following will appear in the console: + +```bash +Individual Results: +---------------------------------------------------------------------- +gsm8k -> Precision: 62.32 Recall: 81.87 F1 Score: 70.77 +math -> Precision: 58.42 Recall: 67.49 F1 Score: 62.63 +olympiadbench -> Precision: 47.35 Recall: 46.90 F1 Score: 47.13 +omnimath -> Precision: 46.38 Recall: 49.79 F1 Score: 48.02 +Weighted Averages: +---------------------------------------------------------------------- +Weighted -> Precision: 52.08 Recall: 57.92 F1 Score: 54.73 +``` +
diff --git a/recipes/training/assets/wandb.png b/recipes/training/assets/wandb.png new file mode 100644 index 0000000000000000000000000000000000000000..64ee8fbc00e84eafcd27605d73cd2c7525e1b859 GIT binary patch literal 67356 zcmeEuWmuG5*C?G*Qc8ycf`HN;($d|aBHi7sA|N2r-QC?KA_5XaGcYs^Fu>454Sa*~ zyzhCw^XL3OT-RK)x%b{{@4eQ#SM9YXTuns|4~GH=2?+^LL0(z|2?=!)2??bI3j^`Q zWWb^x2?<%xT1rYyK}w2F&Bf8e+Rhvai6z$kv!o)5I?4MOYbypVwKg1b_98A#D}bzV z{Of1R=u-}UDrQ_ekYQ;w_{STGv@J~OGMoql4E7KSET9}jBKOO-ScPmPhtYojO8byM z<`&L&x*;;sMg(L1oSU1wy(op>2Le$n^(%LaS zBb)l*KZx)m~(j9JKaq|67dp3 zH0{mZUekHm+c~%jd5JRo8X<&e-#zAJp!+q%%~q5_Pg#vl%F)G~j*sIR$1?^o96CBW z5tp|XLK@Ptf2Je;6J@Y+b8`~njzxmi0p(A~j({l?MVO_YJ*ZlQnve(%%V%lbbnIk^5g7UF=M zcRifXIi7L;0~;|_|6e`-S@FMS>iuV?pun^L zp7~!r|DLJsYVIQCXpdOZP3%9E`7`;yJO4}+;k-Nfe_`S`ntwe-;8_eug!3PoiQ%*h zo2eloNgydmOKN!`?}9M>NX#!W512dlKakKxsiEdzeTdep$z}=L(iiBcb1usC@pziP z0nK4p%+OQ87Znsdc{-~iqyF^c30XJViz<5hcb^xBZv7Vz@axj-2ORj~b{(&~e~t)m zWI)T?uYO*QTwN9v7V_O^!jeEjAq+x7qeDXe+cbEzyp&r}^ZdV`{pr>~Lgwwn`qy|m zq+lH^x@-q@3XkCb1JKOK>=kC~`GIO`c(v%hpik^B#k9^7BhcqHz4x2Elbh+5sf5!zhmfR)y zeP{vpWt)ef+@RDn?>vi39A)`$?EWg(Mi|4;6KQKmgboxS_H~ty>hbGG(kWg&+ z531SzO^WkMM8{GGtl8z?z?`IzkQ*qyW=a3OGe2EK2XxWbNbujln(2drfOsOiIR9|K zpPhBGAv!#vqLozt2DU&bA+d)+@$#W3|rX0f$^geswA{V0}EuFu>L`T>C zhSKk^bm?x9&{ZFiO&KPm+!@l3M7!(olSOdl=z@(=F(=L^D&AB+hlSj) z1!-`OkNG<`-77D|njgAke!Q|IX+HlFUcc1^o%rII2;d$SlQV90qj%stiwW$Mge~3% zc?>8hD4d)LL&x_ z*mrcUnB=b-n0AG8x;v*hfA%RyYWscHjq&E;>4oFi$5=-5)Q(eNwBDc7)Yl<(;hX~7 zFHRa*mNq*MIaf#RjYl$5GC78ZM>~yPIC6NUF{!4!$r&rtbxb{ODs(vo5s(qzk}W?O zf14Xd;~g&szgE25jeP7jBO3SxQgu6wSNjQHSbI8#nqMV=M!@Q`Qr30Zlaq0_s_|;> ziR}!S*yk=HgSaqka^pua^JSin-bOc(X^mayu%!CYyKU{_(q%U~hLT{_ha-`H3=G(@ z2G(0Mc3i`{G?oHNPP(PIi*s^<@u@}Ri8<4xgM)*Q7A&)TX->CdU!=@8sv8;UMnpvw zym!1!6T7f_4&UE2+zP?vtWE3x{F#t~YyBRu-8pX9{(F>gWTxNcLsl_;>06V7M65#J z@daH|#<-Fa!FhiI5qWz0o|DNe+M-XxX(g?t1~mPPK55K7FMkWFpd|F>^7W(g!opOO zbhwDNj?S2CooP0bdSj!xps>~0E1>|9EE9ikjy+IY7)|OS%VDTtryp}UKhoZ-)rLn=n!^3kHJFY%=&3-8Q_(T3e zP(u3J4oQzuyP4rict85M#4~xmh zgUkTtaxHu&KHzC-XuOzjbjW!~;1G-KnAPcDTwVQHK*02<56tJ;F)M2sJ0YTZHuv(!BgEA^C>*)s&3Ll)C-RlVeoE2qW~ZW5>e7 zz&~mos81&oZ?=F8{_1&a8%taXFmCg4PfMr^#URMJqVbYw&rNtb$7H}+isb3sCRdv3>h!2{w9?$wb<(c8&&~SPjg6_{?W@cwQCGBn zA2IGdkl;7+WCtDh3$uW&qI-9vsCZwti`#ceagAV-9~eP?YC^^Q*RPZHMu{M) zqI_$wcb+U>`7E!YsM(cnV#l_!z}@o#t7XSmH1f!po`-^T@2v$YwUIKP)TkKf-sq^p2E9SwkV1R+#I1(x< zJI@Xgx|@*SYT$vy@)b03p!=c|lQw>X&9l+{tOcxLW|sHU*mh8Vu_nJcd9~FeVI#*~ zaEPD?73Dg7^z-M>68ty&Ox_%)!x0!w+m*FbnEmVfpPDI9jhthM%4=k&AH=aUad;E&#-M;(YV171B z2n1pc2)toj3=@kZ>pV79FZ`}v>81Qq+4pQW2Yhq%$!`O4L2eRH9qv5zjHBrGdNxl> zOY0G>ldGjo?}XeBCdfj(?`dfQ&uMAd_6;UYZs09M*UTR=U>GGb1EoGM9?G$MVTsOE zMg_T~Nj748yFwWWbw=FAIy$U!;A-uW5AWX(XgWo58WQslZPsQ9lT}nyG|WCSIN32~ zM6sLVQht0S<=_x*VJutb2JyB%Cgc=S@OM0vI{~tB4n+$*gLMS2zgP^FJ|5BPQ;|ye^;$K?cs6kM!$4E>`bW$4-A^U zBzoi6QDOFq`nPSC-GR27;COblR-UfSXgSMfxraW!<3~kO*7=BtF=) z%C7?rtN4jpp6;kfA|oH+w55IBIxXv04-tIf=a3)DJhYXmI;|$@?Vg6cz{8qQDO^XO zJv}|%HNBr(RTsk}JcACuzXibVtv`j@0i`0p(L%PTpToCnaKixE?g;GPo& zvPhzm1@a;CXAaRLy;pWzAT2j;L$uX->6&Pyt#4TLC3Tq1X2wStb-U$5gi^s=FgT{ih zft$4}JQt=zyVRxH5E)d|R_C{-F&L90Pw~SsiOGkr3^rjOh~QfdWh1$VRRQ$kkO>aj z$MSbQ zyE<>XWe6;SvdRl1fIdFZ@5j~~HzGs}!rnN!)X2DdPY9{0l^1?Kefm_%BCodgi;Ju4 z_WHmCtXcFp%vGuz3xy>wZ9k;WWni)6#lt8r;G!oggc|oj!6#XXy>qb(C00z4=Q$%? z`0usdC&#SP%(Y?!TGi1w3diX5Cp`0uL-xhcUSg~S$=>k!@At9|T6fh%fbse63nis{ z2>ooZKKho6=ytBdmmlF>-0%;+Ve|_M3PV?u&HJ=jMKWl`5D%ca=`TQ~9g#P$KgKFW zm12Zc;@*SONP-Im0?9r5ax{R+x(>(PuUGvo@TIFRVX$w(Q@cZQ<8W|>q^e&>>>I`x zTLfzuWBL4D36f;|k*zNBILgKc8CW_q6PuVt8W~oDe%q9$6V#S?xs!S z%2T3)qj=zy(!9)`X$D7WRcRSH@}a^Gr;SP7airB7gW7w=%=t`Ty3R?zA`?_)(WSl> zzrOHEeG+m8>Q~%S&8jSTF4{KWr$ThGeC>MhjTb-=Sd-aF)DUI)=$j31g5Qn&;i5-Z zGt0yJV@o&*DcOrSj6y=pWGXlDi&G{c6O)1^X~RWkl;fek#%r4M##7MsB2GgKsHH1QJc=w6mw!L8q4PQ@Wb`&hdwx3+P*U#& z2_O=HK61z$g@GGkfrU_L`zNNKS1)PeptpN#m;lGVP$CDLVejQ-17(XtwGU}($KQA_ zx`-R*uR$!tyjR$pPC~3UvG)4K@YCbjk7|b*6cP`^RDNV9O}zL9L8UtPv^AkNjs<_#oWFHPEV6fz=%lcj}B}V&+FeU)7iiMB(2SU9$ODgkn zcYKQE*7vBCjfMt$cvQ12XmX1S-b5bPE27Pd9QuBfx}VFAVk34oze+u)L5c5JJ|}rg zf^6=rUGcNGT!C@xtgo#8n2MMev=m{xK45Bu*`?@P(%g7_`Do0Rp?kF@^&Aeu)r?z5 zjrdlKMe#Vq!9S`47!A%cr-^s4m3DP?s2rXc`(~vcNyC=ZMFd_q@W`yBT}%@LHkxvj zb-gUJeJe2lRih;x9=N?M1FZJ+^D}&G68Qi$NsE;59ysegV&r%2)SDDI{x-Z~X(|@6 z(l#O_BBMBup-}Sp^<)95?`wNxLAa%r^G~yGtu=vuV$$ z3R-ldV~eh~wKTWr00UYLSQx75HmUwGR!8=>tII4ww;N)-&U=n7N%B7OlP2@2&4n7u zxP8J6x2^kj7d4lY4f!v4p8&V;lT8 zC@`psM{EK#W!P_rb2ni{ZJbFB;B;Du!UB!`5pj&dwcE5oh#nU1J;IEh419xAYn!?0 z(<9$;Fg&kB*2unSQ!Zj?tC1jpjKFZ!eDAts&^qC1nwhxplZaL6y))6Q%ABha>fC}S z#AjkmsjZ&G%5zhb#%Md2VX`)&qjC|;a3a8S$|k1n$>(S&sC(qG(c?njr|6TY6_Cu` zj@t@2j|E80-CP+PFwEWJ&<3qeqqeAAe-o_P=ct0TR>~Y`KmG`abE zVrW`&wc4slq9yy#=WR9_;x#W^%;2_g*kj@B;uoA9Mw#eTt!-bJpph@o`2!O@pe3IIu0*=|QhlfL_M@;S%pmxk}{ zoQ}N&Dko&qGY}h-mcnz`W71mEo zHOUf_?g;c}fCr?Hgw@7l(8gwKyC7{rhyYgI{*!h28Xm@Gb;D~JnVIZ6U@*MKusqEV z-dNs?KIX`2DY9wck4Rl)leRP~eFEdo4XtXb6b{n?aqGJkYICGg(@r!tB|yjdAwXS+QIsRG=%eSy7R?3}BC1pw%-2oL2OLKr+A31z zZ|uIGH(~F(<};*BBVN~grTC)hP9)_Xm!UI2En(3`X-l3rm36=v@IroDWus(0#h5l; z=7IVVx`CbpcR$%Sl+GX@+8wM&F4)KQvTTqrd9?CibXkiS+?x{7)V&G%(!^ke6vuv^1)5A z00{ni6MnL~4sc7@Qv0c1Q3hP}8R5;~%`M^L_jcIJ|B$OO+|m;FJnr1J{HKTITXw;i z@(NPkn(5on50N7iFJ!X8*h`7)MUUrQxUXjq2yrnh>7h(C&FO>$RJ**lV3WEb>8Q$6 z-s6_J#I$9I?^FH=^qF>7PRQj-gFsgW-jU&(D0@-nZt8RrlJtONZVROe2*cS(wS%!H zbWdGfo$P9Eei3kaa^f~PXyen+2k?j*%`Kg@Y- ziiHlNtSbe`vJFZ#j z(_RYzN7Pes@~8NyB)nNa)+Tk~9Ev@^eSY)}x$aa79skQ-hiQ4G4DHi0cN>SHDHgKp zjHXW0#-p-IbVNW031yEy2(bsAK&YB*Tz^1WL;r2RK+z%M4v`X?={8Gs_+S|N@pZfi z@6<lA z1)cbBnBo&-5EoDYh9Zc#@fJ!73Iw-O8R7lCH2VumqFWeo9oez=sjglv8@8^kpM^uRYtyf$GY~Qht4JRrpN3v-_g6I&mEj^77 zJO+TZG!1~ee%l0#%CC*h8cxe-;o|oC?8(xEej^KD>#0vg^&*&Pm=iL$H<3iPbGtKi@_0P5WT`>3_{a>AX(B{D=|*dR}3C9h)*+3i!yXavU^Vu_}^~K{^W2m zLySl#7_OG%xvHJ?-nP$gY&rvbQzJpD3qegFxZ)#d%_4Ku!zT6*qSoGMF z^7$v5A=diQGqSI1ZR$TChVDHUHlk$+c_DsOv^Cl3(Zmf?$?2dCEZOMVT_&=hZ`X!c zpE9aW#CR?+hb&U;cwam=NG&4E#XKwev5uxn9epjW3KiuL8(lec(XGhKqP?tZuDmKS z`4k62*pUr;o=>kiEBp>d>#t)7zLe+T=xhyM+FPdz)qbu!me0dMpsbafCC~5- zPv#+ri_;ENgQg}0s@YHKccFY_^BVQ56g5o@YBZ4eJzvvG;Dde1~X8bhTbyaZPZ*!P~3$^CFDV^eNR!L)OpXj&{inaq^VD?9zvg&lkr*u`dBI~s7v zS-<^0saOQ|;Iz5;Nwv^5NOe*hZL4IWvaq{aji~+Y*odcCd=ELeWI0J`$N8p(L6vzs zX=dq5L#?m)jnRnHy;%u*We?|&;LGXZBQ2p()V_;BS6BOKk+MAsnEn=AZ75QUQP;!V}hkT89lG8(f5_IUrqmIAT~AJkos3cX)b0 zY#JlmfW3)9SWJAokA?x{U=qY5MgzV?GOzEt2Hgqv`qNu8mb?lXKUiJtpw)aAi63n7 z{AU7z&x*nk0k(%%2i_EKz5As7QPD8|};Q@E4+2In4L z!K=9NL42^@dX5z=K)o(S{z@j!Nemxzk}Cd4yw-RkES&DpaEB?x1^-j1%)}xOz|VJ& zcvQn!rl1(qX^NI-t<`L%tC`se&(v9fvEbdnmAM`)++NI(fg?zjT@=sun)LcJ>i%SQ`61mAwT7k>;Kz70=E6f3(U4wprHzfGw?MxHXitG(Fjb1+F?BED>b$FT1 zVpDek^zc8ZvD(R0F*MC&Zo=EXFS8Pk9e=Tjcyj{3i{Fx@_`)Vrzu02BIn%Rn8Sxc? zv*G8{Nw`y*t5Dvje%wXe{-)W=Ay5si ziHgV7X_$bI?{-g{f-}V0VPM&AHT8XbBxiOmby1i%Zg@C-0A{r9=|%7`uqbl^-_>q{ zY)5qa{zEn9aKDX;C3$B{e8BiA?rVl0RYHp-O2?NG*M6(+XN4TAepGIZ$&JjD0g$rN zQWmapk0reH57;R7kAIl~gI4hX`%|c*rC=q3UC<^oHVmh}LW)J)q}%0eT$*(&uuiCy zqu!)jRkE*d&$cSpTC;Ug4r;2O(L_?TV5l?xP&PCNj&$T=my1s&>Z*ZmLK%vq<9HLP z2;~U28LxN9i*j92v3D^ z-?|A?>m9L^hSLET-OjJF-afPE%$PY2ML*Nz9lKoC?#ilYXUgQ(na-REx7fX4kYBtS zAO~_RxMbU2K3rW<3N$$WF_B9qq^r1gOWwRK+|;~`so*Ov){)hs$~nH|8W5zx**Rph zdaLS{Q4zjWL@NUKJj(f8;Z|;I8f*jzVdUkVvS1SUBO-Y2h|Q8Wzq6Dy;$nmdDm4af zxRY5Mg&7AMk6uqX7}(yYXc4UQveo4kcW@e$PM$zP3E8j~&`U%PTgWQDDkgG!+sB&n zop|3VW<<8LnETSk$`UzPJ) z80&0^w2r^^b2OD6Zl2aEv{*G(GF7ruAO7&Km*HusWM|VXW8b7=Ir^qtrh5J5@nYK^ zy>N1AmAws4X&_;@ z+o|G9UA%YehrLk&l3&a84`J|mq>;jrz3J+V`wp4kh*5}pUJehhDC5GnFhkMEw!y3T z^-G26+Wv;!sTAS*QDDgV2tfx7-+2U2tGQ#lCqD55x)c@(8QFNHaT^OF74+-3Q07e5 zK3i;~k69#D;ja*R71-II8slXUo-RVlZ`=}}KVt2@rzJao-V$MmhD?qwEkXET@r04v zdEo&oSujRd+Rn(^ofJF3{9#U2SJ4f>33D3Gv2)8=wW5Le>PUgts%L3SXz}Hs4bekB zwZlyKr0sa0o8LSqSB7n73%QMhOGZ}!H#K>~_v`3}yeIPQT_QW-VOB7?tZ-Por%l>X zgUhi|rhpYVRU@?01nXnWP5sN8nt-lU-)JVHM+TT%Aep_3%)YDv|6Xal z;?!o@$*G*LlY+R?6ZUe%U~!3d#Sw3?t(xWu5@7fIv-yON${Nh4EsBC;NpsT|vg1AJ zykW$1K|l%~B8OHv5@)y}H0TrEnMhrn5M_%<xaLDs))v4HB>5j^6(tq*W(GxB)a_A45qQei{6pe58?{IW2XZ`zdXSY1;EC9~MtBBM66^H3R)0 zpLXrDwvZB}kW6(|>G8i0Xtn$xGLqa@sM&z^@$2g9)^aUvY6ryL{~WH^nIl2;N?%p3 z(^PUBb{ot3?V0uzb7aHYeK))6;;GG%rE}xcL?z3`K<1Rbt5M|F^loQM76?8_n&_$O zVl?SeoLnd0Ur@Hr)f5mX;=w?}Q17ep6M9rBseHI57YQLHz>`+fi#3&?1T{Ait%_dc zz!p__WgdTnavWc+h#cCYh&wkkfT zBB2gE6kfhgS!w`xpFRa#XNFO%d7c?^AC{q9ckFO)xIBLgysejVsyMfKe*JL9=$L@V zm$>VV)2jh4A#k7mk>s~b)y{Y*)D_dlEX(g*XBS*-u1EjqovSpno1oib8E1qP?_e0~ z*#CrA0+4frcnBy)t+^j*S(``n4n(Yn=EDC(7xDuGmhqD8j$tM@!HRlNrhmg@>$h^-g z#XQC#-8IHQxFc(+Nc?*oEG$m;W*{B)T>U`XvkB^u@rj8iY;1~MrF}yP_e5#h+f(8pyCF)> zCl5GVG*_y$WnEF=S_?-Qd&lEf>s%DdogJs_TkpUtIkjHnm^}MNOkHwDPNi|&kToZ| zpS%;-ormbhH!7jhqagN_b5UK-P;0R)oI>xEP=S54pP*L;)H6CrKOiEcN@d3fM#@w_ zfH*qZt<2!-F&SE=;~Gd=88ya4dsN4Djrfu#snT)dPUdXwx|^elpZ(>l)X|FD8!v87 z0dh3DW#3;FyHx1n|)VmS8 zxB^$Vy*HVe3eUA9K=RRH-(-dSsQmX1;lBo4p`735+27t)b1r#n98lZLLkay>AX}!m zcd)x*y`4dIx8l3dZA1WJlAG zySU&_TtRQc28V`H?ZZg_>tMf|!Hp!(lBW>KWt-UA#l`ovXR!X_yxxFY)_(a}~x5=q}c``ek=i}iy^l_@L-vI!4^Cs6CJhJCPi(eWU8To{@ z6gNXaJ)4?YOiYsRIP=-c{fHRVd_2nO@qN&oL2Czk+>=uMFLMj;-`qTl!tXk8@$;nQ_(@F*E@p7{A$KH|2UakRj(B(6Fg@9Yn8usiTX=6l zG@E02Ooe#$srQNX^w2;d9YmW?9QLe@R*8zEroE7{pn(irI}*t}V(lElciugd--X{< zG_p@^DHV8o9#CQn`_5~)glYJ6@kdz2F@>|Su@qxUC@Krz(;%0ZQE^heXp_jul3J#w zMFW#FGnx`X=4VPw#03R-UxH6EMccL#4IPSFTuljkJYyTPD$vEn=S5L5&bAdA$L%WW z+>OdZ|3e=qRSDWLJ=%eUAT4x>OMgSEcaLPYJROtFbwk)jW>eE$(k|*?YTb!B4RZoz zkfxEyoAhaL4QV~FwW#r^hy1HI0Q2k4YWZm1^l|#=0CB6y47|ZbAAW7=Px(e zQ07myA+;fs)zoOx*f>R5f3k3C>k1aA%+w7O(}gLJz}&UoJVL+X?{l3x#dE8zU7P}S z)Ml7}+ey#_ieY<7e5Z~s8L;Eyhs2i`7k|X3npK^f(|4(3H{_dmXn8L$fh|y@Z7w}2 zo`;}WyW(^puE@6Ae*e>->|ewin@x-(aQV721i#bHuqQgR7ZCFn9^(dozK70zxz_kr zA6?Rh)*m5}%8#w6(F_cW*A9#V@h(qo<}POucS04Td8)_LF23tM(}MODuKhtVmeAy# zfKuQ7CDEJSCE-)!8vNuv>Y)sg#=g$!i~5@dpsUZ0K!s>m2VB4uIpBszi2g-8#e#fF zf7$M$r*FR%t5NhdYWFf0$~o7;0@V_w=ubTvqGWe+Y4C?TupuR0dUa&$XWE(*7tl5ERRHr<(^Z zOKtS%Fd6zP8TNk_x|6H22w@{JOdZRv_bldXZ?DX-Tf**MTF-Qaq0GaFlsv+fJtIG6 zS!_#0pWh&Y=e!30Ec8kZnY7-8SVw!!j?-xqXiECjmofYFyWl+^!_Vw@kl!W;8Gs@q zB5)dthb#J1H<#y$d)z>sp(L@>y21oxe??W0gfI3iPO2cMEOz?*5IXp-aOc%q)3CT_w%}KTKZTmh4g7n_p2~0nQZZPW3k6`Ogw=U<3ztXfm6bTbh z+R`ZIB|El&&yQMkcPp|iiJXOBY$fP|_EfWVy z)AZh}U(CYEjl5KF&aZhU+F$#JD4I}HkSgY6!xjEOTV(Wr5G;vG7xRzOD-5cjGvJ+< zCEObp1~`w9iaI-Fb*fDDM|-B@N~q8;_o95jDF#vZj?dkQ`DHn8 z|JigC+Anr$OArj*wPX8ilBOw(m5b*Rs$w3>reTy~1R6;&n*$cQ^>efUM@M6P;47Z% z@?s&ccOGJ*{TCk)bp?da((gbT5$bjI zzG-fIckfTqKiya%%2}$j=exp%%kJc))1I2VZasq1khcvg=tI}~=mR#DjXl2$*FGZ7 zx%o91)rKQTO>H?4`?sYNG8-ar0M)GG`5_|g(5=YLO`buzUC)MW1dwvN8!XSybpgOK;kej1Wj3O7HIUmb= zuyIT4r;t}8e!8|qxuZ@+r74_4JG9Mevb%m9NaNj zwmn)Z`nWL8wv#m?T=5;5haWVJIa)gXu*bDbzlQPdJorz>IFL`@VG~=QY&mNg7`%ui z>yUZ;xR1<>h=|B-#+bkcf=??Z5AA4D3&<3Mb3}xPPmOQ>^p-Vr*#7#;Rdi#$-i_1l zPvPZ(C?X}AKT;{(OR%6QCU<9tRo?-;vC$|T-|5Xy;N8GfRIlw_i=QhPv?=bL6;LrH z*0j?-``acJst=k$-9Z%kj2Ek$g`=*l^vX@tTk`|1FRJwDWMxxJYHET_0^yHaIy*Z} z6rl(!EYtr?!Ny9ULq*YUr5IOrmp@wdB?ioAZH#=nGBx=W3N)LpFh;s!ne!VDe zvdFrF`z9Ti8Wx*;`km0(SOtXD8K0j1Vqx&-d|YXQj?}A$d~f~`E5{}FbOMBwk?1Eo z@R6V> zjh)v94fy=Em(l9t$h@uZI&D{JwXfO7+shgpj z-Pm{vo$@dtyat*wFaN+wB$GPW^^0~ zCwUrjZsVI~Ci_<4C7c=gY_41PkSUpb2F5kQxc-Y}agt-!AadUbb! zQrF;bKUgi>s?43())b=vvy^3OPABlXbd|-jzL}VS{crV;#Qj#em9!+FFei2R?q$ zicJH{0Dj08fYIP4Ju0}wzSvM6@wj6Xg;dIN`&Bj8ZwV{#xZ51|G1km35!B&xY7HDz zMJrid;a3&)IU(8bN%j&5sDGk-4NB!Qn4cR!K~(rmym?NQSP_o8J=qT}2#8s#LLc|d z^bO|<@MYc$*f&2Jx5veU{4#X9A0t=sUyS@*U`5?Y;kF!rm6LJ!Nm;nfRul19kCXW9 zlst_8{8@g+$Tifj7Sv$t>2tNmZ_b%>%`w#hQz6gzLK+EcS+&|2+#q}3pr~6u&(a0V zI`?N&`eR{W&5|P|Np#d6s|lc0^(4wgqfy%TYOnAJbfYdBmn%fKMqhK15mN%R@>k}; z1S}9e`BQ}-RD^b|p{coUVSxYlw*ZW3M&BGP_r%A>uEe^Ag@r|?ZnMr&77M*{de+so zFgb@c^UBrU&DlA&?=$7X9Cj5aCuiN(LnG?mDM5bbHo8ik=!6!5HZ`%9%@;{oI%a?B zsk}7lL`5@dYik#T3Ydqid>=aNc(^_sZkXJU{+^TS2VNK0r%w<2-fHpc(*wg+Yd>p4 zNgh+&?-7ljay2x+44iaq0k*sj6R8R=BXvMKvppi1!MA|wr4CSNXyF6Dt`qoiHaXc# z^^~B`OuejVHdBZ%8ud3}x0Fyu1|3xK?YAT5XY=J-=2Fu&1Y@#%Ql1VSn$)xT%wqgn zK!kqz)U#ML)>@UCv1{{Wqm1d*wHxI>=Hz#><-)~lwRGE}sc!_|WBgmupfYTPKYvw< zXAh&R$i>?Su&!k+do^rH$|x17VbE1OCd-iGOiJ7{g zW9aB#{t=*+15TLNv%4BXmFI!@zqu9(W%1Xvipc4#u(VIx{kgKLOFq<3|LFfu^vk;) z){|27<{VPih!9OvfEi{A~C8l9ew${z?OJd&nHTTE(PsNN3%=m2=O04Fh z?b^2G{|v8xk0|Z=VzWJHrk?kqmA&(g0MqYT+3qMte6E^MUcrqA3}x=`W`7qr0`>0p zYuJfUiitSWe|Fr?<}}=0NMCF8&@ok*RK}ELu|r&d@)w*3yVN$8W?HRj4v-Iq{IpW8 z&kbMd{q_?&IhGwB?%D;~F7<5swAdWt4;>a(H1+-whZ4Rx4@y4(LC>FN>_M>%qGa~0 zDF6OIgrD|rv%LXAsF`86V-H`R_YwB{TZ~@x%PJ2Ui;)=~?t4b&?hC?B|DC!+H}>ej zO!D&(RJjZwsAbCX?9b0wcqt+hh!W~qKWVSnC47qcOJFZax-JTF_4e*hn!gRQG?ZY6 zeJB9LtduODCl`}9K7QYzev6&;$rA^U>+K0>ZgR4o{r|A{7En=kQP`-2fRYM=NGc^# z3Ifs~BGRSAkV>}#1JYojBB68)DJ2Zuttj0!)X*J6cl~Do{l3rtzw5@j>#lXzz00*& zGR(a1nRE8p`+1(d&wdZ11=`G-Q0%EWsYn5wt>3*Ds`oKp;-TE_qc5FooJec}Qe_aH z!yfuwlcT05rW1`!Rw(;0-R7Cp?~c}@PEnBAe8$&)9JuCpLlp+lx`X<6>yE3B#0$82 zltTlroLc)N<~wtxkW)qLI_4D4K)JKXi&5&UeZs@GR#w5heE`|~^c22B^Ifw$#^C3j z-zIcs1v|mw>+cI-oMOz(k&dV#&bC$w=!+M3q@SWl9Red-Yp! zNz=#@?F^nVu@Ban9+;S%BolPmcrRVPX{h2l>2(k9_cqqM^qxBW;Z(nCg&k)!IDLs; zyf_N9K58%B_>j0NO*tPHN{u4;p4A%trz)r zQ+ACl))pvu)lH|ck}&1W`04G`4#U*4$UeH$i=LuMfL&+)5k!<6_{cdelGBZBN8~GQ zkhdD^`{*~F7VE!i<(V`>=GWG^zonbMg5M*r&s~yfsK!T}MKgWQ&USFt7 zm=<1Hp_=8*uR!6n;-5yVGcy*i-T@2m=NS+vd9p3Q%Zqq%?*@nn)KC1cmj;?qtzQlG zXLjnMq1k^vH)k?sA%x;3`xAE8%Qn)pEC5sanr2<|S^Afm(jEFY^6NNj4(} z2b7oM{q6s5wj%BWj&52*@q0A8Q{xL?eV0D{HeML#i$TpsvT-%lYj(8jUAY7VxZHt&mpbkze&DdPY`6yf2V4 zcg}W9q~_wQtHWruXi3Y)oRf;Yd`97^fBW5F z0}vumw6U5w9(lInaFSVM8JFKR-q3p_w4!cz6`~wTTVjtt{`#z{)@4@h<-IZP>lsfg z;?ihs(&hoN>B`c-C9=O5bfZ19s4oRBn(8mvV+pZ_(o&!KYy+igWrfN?*H3c15lmxwg=)v;!*rrEJT4=SWz(*>D+>#?T}*hz3W8+ZaegE z-@TK?Q9SX4gD0MFBWOUsis*Nw_vbW7y>_Y)dCJ+6i>;!-rXFV*eYuBJ6*(ipGz#=* z&M~j$w{yIN;ee6G?TE;vaTt3`Iu~ha6BikIYj3sA*lxc%PW*pC;{3#yvf%d$!0A8J z3fX;X3WTnEAEM7qP+Ys4?Ckfo1J0kibvMkAkWgX^KfAX8%=mdF(D@GF z?Ml#|P4)0)JUqOFy*;4rqFqQ!F?Nvu-hLPfc!6g={K6T-uq9U zNagiV2d*2j!3zr^gE8xMK<^2Ce|mE-FA8jSIT^0#Yz7!+K_|_e&GoF4NgvGwPSb=t zqM#`+>FNXM$vBQ=){tJRVk7)_!ww zU9EoMICaXKY-~?_`DXl5hI=y8g$_xor{8H_INix+3>cCXlE{aMzJ!{!b}b8bi-N@Y zjR)?vXV9&k@_a53u7Nb%FftOWs_LTh^r?3$Q)ck8%=Ubbv|es(tQd$^4aP)9&e|Sr zkzFm*H%nr=f`Zg$0hh6EeRD}*Ao?oHsl&jY7C@~Hbu=&`U|0wlH{sLluDz9 z=hn6YsdbbM*)4EK=owF7j(EzY`rl;ab8IXfhtAHs^1?Gw`hoQZ!QQxUz+A8Pa_}G% zBV%V{h**PD20EGKx(wqKWMU(iH2w8qvx$`-@xtELX-rwPjP3p539_x$#mK_#&Ln7v z=bT4W9r>{b_mn*@uY~(dX4CpYrcTSIfj2g#81=6wXAfr)yn;``+w~-u#kO)9x}dJ| zuolO8w=Q6&3!m#CHp4G7BZF+23sJv0gzeUw5n(6dL_G~R=>vgdYIhZ+El(sQ5$y8e z$-$05Y8S8kdVMu3VB#GTeUtN|vZMJ4`x*R2{} zUIp*Gz2;@@bOQ){;%VNFu7&OSuA8wT!l$*_?RuDcS@Di{8GWd0R0*0dQB&X7&=>JL z+HDJ)>y%@$tdTp_r1tSN$57d0(gFBf%`Kx!U{mM*c z84*S?pK`9*AVlDFbMyDYe`{)AfWl-UEoql}=T?e*Nu%&5M&kI_qS5=+A6)C z8J1njb9Pld{d#63A8AdI?9!UN6Ez4whxWyxXuF9PUgM}RAk0n}^e)QOBN8{F=;oK7} zhEBYB`!kJhdJTkJ7g)RmdM3CUKO&;_bAR7~o;_Oi~e-5xx)# zJdtBX+0H(;;WUACI)=MjWnCA@C;0vT-?#yM^to*HLX8Qlv&Oqg<-z%x;j3RH{iQe* z2WrSNVAmw=&B%TSkUwL>xZZAq0hR8Y$w|F)#w+LzsJ-9kSN_a0dU{5Q^U>hmg0`l! zW74KH(Qu)QD6F?WpkbOsKx5f7ZM$G&@YxwN9L2{(g??|{+CDF-gH}Yrv2{@l$@73a zKb^BWZgtv!E_D_tm+|hg86p*I=bo&jYnn;)s`8$>{1|dH0i5%-pLyvcXK}NijFH$I zgbBtRuHT64^a|dstX^KF=eI*f5I_D(5`!3P00N|AC9W_zlz4qkud-wH1I^8hGp_Pd zOMKHj0TSrEyK(QVVT3rTxH$aJ3jAwEM@vDg1tIlbLlyfvmaD!4MC7NS<{WnUO|NSf zT+b#RpSovdKxQOkx;}iwb@b-T=+GIIajS?3mxHO(=Z*y60>$xGp|7 zjSoKya7!GZv)y+Z-~|W1CqFtf0r6@%%&X8Ex%PXfCpqq+d)WogZs1%MT*yEFyC>k; zma%kY*+t|0= zl3YFds7|(lFq=WRSE4!4H6!IsN*8k* zHV(JtI68M;r_>ey)a-6~p99h=U>mwUK4rTBx#PK~O2}(`&4(FFFYnyB(~)mK;~0VU zpX!3Ck9Ec6sM3+nnOM&oua?ePS3t7)aZHqzvj@KR3o|n2_m>Vpn#OWn!X?dgZ3IG= zhZu@@2^G6Sqn19Mlo`;sKRsW3@ZCteT8}cl(2-x4rmkOyclRM-0T1ifQy|Tcrmwiz z*edSsFvphP#Eqwon$;qEftA7VwH0lq^`cR{T-enz?FZ8QwtRXJNQSj%emSVYJ!AzK zCo{dTz(*cSEp4aU)LQa1lL~O}O)jpb6@(Z5X@CZ~G*`^fOX=IKgNyPYdyy+pBzrrm z<>$giSNI)MbT7327g`SghF2JFi;_p+0-P|$*xe9P$~Mz zYVx+t40@0}OnbCeB&PVHWWM{uh;$2OC2Qde| zgUr6?Lp0P3`xgo!BYO5v$XWgErjlO7GW z>FTXsbfr+TN!TLYs1pCsbGC}D0sXn8Fq5t;8uB4oFVYBMA!8Oo()OPumLB1npN@?d zts4$vRfpKqa^@f>%Y|LD+`{CptYlK-51!4rV5GRq-c3k_%Ud@l%k}BaZa`tq<3RZA zCLrdt&%W$SjHw8&tp})R_RXnq3l8uyRGm$#1_0EO8e7$Tk<}eal|M-!kvONR|G2!1 zKYu>xYJ~xI@`5Gd*RNk>S9;zmM&E4bjMx_%8(0uKTJ;!@E6dM+{FM&Txx}%vyW1~! z!_%{A<|tVLYqHR|zpSCd!;$#?sq#OYjp^s-_pVt8(}KdrcT=-_PxtnlP(7?udj}XA z6jrBJRw|a|@9nv)#^g^a;bq1vDxxHW38usEOr&M6ydiimaL(y26XQm96g6F})n}8B zcbnQir%aQ7*g0Kw7$hz~zb_DBs&(xex9?Ics{lFl%aV!Z)%)p(%fxo4Z?a}nR#lZ} zR#Us(WXc`Y(jv%2CFEFcHVxwvWY||cDmmN|yTpHGB;OoX(@Z1clB(sRs;X4dD#)R7 z5Pp$@_2HpP&*J^;Q$HwJ;bpA-P0_OirAr?mWi1kK&uw|>RQ2<2WeyA}@O$V_&1M;` zrt0!xU5;)U*% zP&cvgO89I(dpMt!re*4axXD=*L&Jz5+Y=FV7$VGjln>Tp{^cITLa+7=Eda6ofOeyb z;vne9j~XMc;rWW$->^Py6S}$4=2^+^vPBd|^X|#xU~Q6nY&9Hwh3npQdUX2^A7GFF zpb7nIW!~=A+EfRAq^{mVPU>czBuBxYRqw|xCdf=p4Fvg{EI-%nX_=6YkdQ0SgqjQ> zv7G$1`lm+ft#|`S7z6dmTeS)5&g#5ft7=?o>8Aicto;Sir>7bxy#WEhKAgk20Acu)ICG8cAxrxCl|E?b#_RhT|HcBsjB$`R&a=<& z5uBo`Y7X>=)*5nBVOHsaEDL~&vuNKqLv`o8LxG@E$^_{nKZ%XyOaOVk?JTTK=%u^g zqnbF1NdKK07^TGMR(7quBz@ZLxD7AADO-z_S^jHBVx0;)SU@(&LxRb)brh_usCA(| zn`Csc^hM6g702y4h9!1UzxCXG0sEMaP`~Uri?kAZZnC|H#SHaWe*wV4ODisasuX{E zuarX5L08YHe71=@x17->eplEkQ?&Aq1HPg4P2j0|NTYBP|6Qcl#XU~q|2a%{0K)}E+{xg275DuOOF;P| zB|R<<-;?0tZB~DDAfj-TStfdntY#We%lGb4Fj=G zt3(2jTRuS@8^Cqt(yZ(?444t=fj!?DFNvFanpNfqE798dM5**~>UD6K(_^6+_}k)Q zn?m&_sBRhaT{$+60=E=x>$o*aTG}-Aei&j&>_*p@%%dwb9_E$ErNC3yReK4@6n7L& z0!i_`LFjc@>|!EcK$rcXp-KBNrXF!>2^M!yLxyfqzm-idDPkVOQ10+)hkt1x?u(l1UP7#CEOhVe?1`I7*gR?2Exv**)?_1bW^M~58B=8P zD)BjuLZ_*Nvn<=0BSGK!6T@ivQm)T>>?-N+en#%16ymMTkmJZt2tY{NS_j z`8%A0NE_>gutXQ(HF~*#5z^P^qP&9qAfW+;|aht-nkGBZMAfAV+hEF z`dYvIBuC4aUR+#na(l^&Kz?Zyu4C2!yfhi`(gj3ks;am&>2vkk(kN6t7H_w|L*S_} zy~GM=YXRK%lpqZXnp*ajaFZ7t6$81&c*bZR*ZZH0qCfp;#wRF$1;bVP-$^D*G*Rk} z;2EC*i8gd9)@7fsB)TFo){V8u52Rb0_6`1M*}u);b|J{wkq^i63MBoM;3baj`4Sgd z*E)GUzmuXR@n?Fb*_c+&S^My6r}H;D_f#XUe!P4paaWsR>MaL1o1KF<5aad?Ek?bz zh}p@@`(T;2ON`rvOwr@y*Wg98Zb=FL*T;cmjS?trhnKVX=Hbshm+>R`rb`nRsq{7N8ecX`$WV5cYV%UuRI{a*K78R~40J2Br^x%9Cg5XnpC{7P$RfQUUMhEtg zUzFG1ZNu|}0L|qe)24d7Zmy!e#4PG~)(Y@9;BEK+r?Sd=s+=!Hp0+fJx2K#Gw+B=b z$T$}MXsK4wMyfkqs5{NQIh~dg4mMG}i_T{`Q;rR={}BTUF~xKslD`Qf39 z->iKHEc`I>Pa`S@E1>6Ymrho%x*jAbL|!@xUwS;44zDG?NIZO#`r;``j0Q&qgoZdI z-=UrS&YN?!@mnj_>Jq$FN-exSL*`nR%$e}%4GT>@*@s!#(j$`Qp8J<|!TKrsZ+d5WNldUeL(E(Hz4e41ye88a>nP|#lC zx}BtwSlbB_xa})riRcJ!c66Y@u5;rC!(3mE&fd<%cB)dWhr>3KxZ^ePunaglU?-Fa zRMGXlxJWEZk17aUZ7}~=^lQ_jM2k#;;%TvYp9`-R3YF{bpx4wv@ibioQzO(k?fweW zMf1%wGC|y(-&~?H(kAtJCs|#}^zKE~X-X?Z@Q;bfyIM%{4z2I&l*Yk5No;&XI z$=ly5s5FnoT(>dkq+jqo-+2;dC6_yO%ZaLqudkSj0y1mh;v*$zSM;k-y6vq^bb&4(bNWocgerXhF{?|?>tre{4)wQZ&+HG z>s_GeNjL3fOvu`6(zT?0484Q=-i5o7CsBaYt371TP!|j&j_JZ<4UM9& zH5cYIF-1I}Tm2O8w{F=haaC;O&q)+M;VfIc)zE0M4Eb8o`a`C5G^+gj4Qi=EDfG86 zllM-&kV<;_%U6(2;4KAZPnR#xQz*SQboT%ZQgv|2wf2)a^SP~KT;(uMUH5B_bDi+v z+#f$m+iVcp*PAKV6}Hy#WsQyMjHom%pC3;6%SPl?2scN+9x0z)x0O0B5SzPNrV`f0 z{QjtEe}`?q!f5Uyt8S&>4uKXa`t}%@euw*F10P;FAIR2m$vjFeQ9;7Bi3K*h^Na=y zt+(3uHk&@n1flr&%h(bRm09U9@#wwIVP3kFN|*DHaqyjK0#;yBifJ)lbA5d(|IG-_ zjMST9Sye|vM`f$ELSH?Py_1diO%aa6%zU=P%C?S0HWiEI3Q@g1+P#l?n4|d374(=g zZAKXczw-94_q5(2uiQ(9=@^Q_B$^!_mR*qU-)+4)w)Zx3Ec6W>xU;zL6};(Hwu<a;8?AOvB;6uMMHsGRdBQ6I!fjLeu->j0_ZDowxv{im?xFTh8 z8kG%iPfJTn;Apn&&B*2G+}(9H=~w&^5Kx3{1P>Rv)|A|^2F1AvdP*3Tc`x|$FWw5; ze5-(X2H)@Y>c0>P^KRZaj77ESEV#7BnV_;zox6-^OZ1-RAX%Kya0v$WX1U%QwSEgr z5-dziGP1Ifl(#n?n)iGwkdHZd+Z-(*LjNd=uYkWxL_<+VAun&Bu*D8tx8bP3<9!ZH zEr}53)KrInhk7@5H$Fvc5O4#HFsEVDGKZ>-PM)p#tAY++<-?zU8?tZXa16{%&n#8> z=!>6ZSf{u)5s+up^X&@9_$|9v7N+go=07^BD=-AT3-aW(FGPo8ww;Sox03##gq|H| zeC++55ZbBY7e2ig!Z&fYfhbA8&6xs;Veyo&~5vmZ# ztr#a(%nB?XNc&+(4Z>`6KjGIvMKlWDp_sjteq~Uk+Hs^+L@fCddP65DRgX+)#6YbS zThB#sPvIjZX6*=*dCf^Gj2iCG^}@^8;U6I|P~doysgJ zNzXp7sgc-c=Q8AxYy9kH! zH7z@xG%W`Z7=@wVFJwR=K~?Hjr5_MiSJ9i~y@|I0ZB^%*C0mZDe}TKJbsTt}0^RNq|Rj(Lcv`pcE@JBot!-DyF_>n6v@3YfLKyG6y=P~c_((fY|OlJtdPz5s3S zk46Nh)Jr8^<3ykk_Q>hGBQmDJ%YIY{zNXjDF&AIBVp)p3-z;l1mV7GpIA3r7J`|Dz z&E`3jY!E5I?H3#o5qHuLFh|=<(ggxKQ5|v)2wCq_sa$`Xo^MZJby@ia4Gj&<@3+J> zlM8hg47Gm!lFBtzT1vl7ngPijov=Uz$`f6<^nh}7W7BBH)Oz(L!NNUZMVd+63 zj?TS)29bV8LMOk=s9u+)jG};#TDCwrK4VSmszO`*yFT^>_bdn=wU7m{Qxh!3a95 zIQRkum)4w>iFU!J@3h?g6W3db$?EPp@MC?yd^bi?)*syA=c{uVP<-Jg#q@lSQoR~u ze>Lsw0=eQm;YebUaB=OX<@6_m0({oU-bq@At{+RYD<4k2*V&ITJQ8J}LEcK&Y zv_OdViwPn5F=bq}Q@2}&``J#W?KWn2#|@P$%KI+TN%q_|x~NG~-&mA)0c{mdPm4hI z7L*9NX@z|O z^#Q<3=Q>xd-k`j2&miC*m}- zG$CW00hLYJb8?gYw~(q>mioEFm?(o+RxNE6gT+M$uMZQs`Zmj#7^0m;xOS8BX@peA zNn4J$Oc;lGl`0r_d8Xe4F%8~mPsO}&;euVO07YZ$k%NzW)eCiXSJKBR{q9=}{Niit z_7_sAHVL|0gogz0wQoVUYr(^PfSWZVkMqwI5du?zAooqHC4Y`-dFT+6=LpGwjewU+&}fX z%F>7!G0b_0Z#b+IZ6WG-gqcw_u5haLue=T`+}3kM!LPLH^l4jp#*TVd3*}#5ZX=5g zo)qw1#(NMVE+th`ez;|6H)y@x*c>s^gwXe77IL%-)5`M*jE?rs1?N#C1fd@})|tZO zg8}_lP=`Y}S)!kRq;}u9ur}ti{HV-PE7!b>naaTuf1AeXKntD1my&v`0?i;xt`xT$ zq0KGJ$UnlIMhAGjMSG0-*#R|iab3$x>uQPyOA8gfV>o@0S;;6$R>z+&V7(HZ(Qcdj z?{tk5vM?V!uXvi!axiekoP`D-M)YKbLWs#x#nG5q0X*l;3TLk~I zw7~fL#Hp>p!z!cowoV)Uv~q+1a(Kfb{OdS`Knx z)`u{1>CGT9VFIr?ZKfNvcDP(yHW2PZ#$i5H6Cmef=yBZ~SyrF9D<=0n+POpZZlkxC zzW2d_u=5P;xIc8tmvhg$!b`A2xHB+3+z0~mO)4#I{Dh--69nh&$Q03akdc{#Qt~9q z(dXE5v3krT&fX$Rl0dtcR0eSx8skJagn*FShie#ZGijR}Og{?k;~%aH{470PE3<7df!rOO@J1 znY+4QIod!HN>#mZuxD*i*a^55gP8u@DM_5yHDaY3VMfj@Y zGIzyfa#@(?y;ii_%6FmZO^EMOY3bpex@dYDA;&KWJy$Jo3OjP1q>mWw+|FTgsDeRm z zw{fn;)LpjHi&-!)>fzKaZL0Ol&vzcU+qoZ zrtjGQB!QtoP)0v&oYB;EMktm;t@^^aNIP>q1VOJ8RAr!r=$RB| zK{;p?@rZa}TlhIOCK-2nDhk}VqT@>DlZBUm4};Jj`PzKu91y7FZ|kn#{J_&T=u#?Q z_9o~DDc<9{cV)umojJVd#i*_%iSr-EZByQ!%Zr^AdcaFUduN<}e^_}bb0!8n%IPt) z(vhlqrVg_yY*S8PXH>s_CKf>;pMX4ORw#{mluaxrrihCv0gZ?9b_vdxUJW8ggEBBUsk^O6T4^flL!N>ytx;I7cFEctP`DN~&11uzvkM+ry>}~gX z2=lgK(a5%_;;5dU{!F_&?7e)MmZisVInChoWe3xuk6}jZZ)$trpWAKyYM;(1z>xy` z`Rl`l)QPX|kc9HQyBdzcy~DGc6})n%-bb81^n9dnoZ_QL+*letN1qe0eljw?a<*gP0jjy%+_q_5nSV+*!9+hi;enz zZF<}T-c3_u_NYa^tWwC?h280R-dGH^pY6Qdk`;78lou+0N5La<{Bn|-UC5{PHJ5_O zD_ch!cmxaV_-k@-wNb)oI~+YV#OsuKRw(gG4 zSSkMg(Z<;VFAj0ou9R%v4nY6Ho8+8NRJ$PZg;4$Wn@xx6tL|Z`LEPNjdJQz%IF{C-t5~z@iSmXT@kh{FWzO=P!GB7kc7EDr150yMg?M@!#50a=n&GHtTEGvQ>s zyOzWm49-|aZaNK|(<$>@*rIb=DYSH!WN|w`K-l85`K$28imZQf+2xl^(p_3YuHzxp z{tvp`=PesmRaK1_)Q>9yZ>S`w*$oV24hvQvhmK#=`aI*va`UnfeE<%=vNV5q^9;^_ zmpIK*5X(4zY9fQ;wcC>OvtAD_>BqA!%)X}({Xxm+EMD<^Oqi0_BE#8p)_vN!;GXkb zlIh}gLK;C;X8C(hP$e_uyFFQ)yx*aPyB?9zZfg;ayAuJ&PdYmx4N!!x2@Vt6?X1>$ zdbJeA(3U96zi@1Bs%I#|f_1^P+&%^jF z#u}(kS;@o8W@}y$+#9ZAZ&?)B9N4LtsS zLQD({UFBh=sjc;+rHXM*27Y@Musp8lJx1K8aCK z17>bqzErfSNkPBtoQ!_d6XifU0}gVA6Ds=8kM?5B*1+$6WfXSOXlU?9J_Gdb?S;iW z@_F*;_jCy>kRDU z{fX3X*)6C@OSf|OMFTLFWEP)UU~uuj7eD76cOEbGb@DT*|J)M{!2-N)$q|0S#r`!K zdNc3Ti>aG5p+}dSR>$^MII5574vxq6=SkKFuf*1FtcE$sY1Q)nJeEAlG|nVO{~N(P zIL@&>K3;J*xsk{C4Rika$kT?c9*?YxZY!%!-p}39|8~gnHjaIkIka29h~u!&IK8h* zhb_DY|7wcP9edtI9n)1#F`Rt5p89gh8Or=a@7i}TvAg|jn`!?&ao|fp;!YT28ujH< zPk(}F&;e@#pONo~vrkfvw7!AzCMk3D4WLD=bt&0TuD6Nh`h*7t(u2CF&l|HHa=jT^ zOn{8hrG5KmVeV6<6flEUERPOx>%m*!Iy1z600i+wky{yks;Z-N^LT&6)A(?&ptI0g z2yMtAz3{>M?GaD55s!T+Ll~oDE*+oMS8CVA4+&LrAY|W9R*JPCrd{tV!MWGDay~V& zhM#~JcYaoR>RKfY)S;&32lAERKoLDhccr!FxD-7nr(*3VN-m&ZW(aTf=^M|sKkY1Y z(l49tkdCD{Ha509KHPqv>9jK@?cCt46@SrTzPmrtaiO;<ZygNm~C?=)^&I=fK zB)sJ|sK#)FgR|WYP9yGyDIn6YGd%pX1!znB#aTKTa0dB<+uocHG`G5(Ncuq)5C@;G zwF=qx>5)hA+Xe%e_sGH`@sj3| zZA0?&d&BGI&wqd6!w7)>W$ivH6=@%ZzEOhIL@IgmUYzcZqg|i^P19KKZ_!xC?`$t1 z^g$-pIx9CLgB@toKCs~&!zd%;f-L*Zw_mDUadOHl3g6fa6-$J2>5nTu`()W-?^lsh zvfr4qm_kG-aMa!XnDn+rgU9|X%6`TRSa-Cn9%0PrIIFh*B)hIvi^1R883~xU+A&GlJLM zQE<%9H@zuIgUl8z+{)-y_Sjd0bz@WnAfz6dnGO#N^U>`iCf!QrZ_3$ZLh zECX)f+?ZjRBbQcgeJGcx=y&Yvu8aAF!*fXXZ8GaAC$1<(Y0KTak3@GyYo?Zp^w-&Q zi=#9a(^5@Ql@j^1v1+wuxAa|H-Wiz`hBz+eT4kN9==C9@dg`A3?H26Dp!z&EshQ=+ zl-F0A~(`=xVQoEuY2w4TM7%Dx=vNpz!UmCImI9x29VHpjHDa8V76dpXO}zlasp+brxU@To zG-5_uhv5ru z-mq!umzXbJtbPd&o@@l!i629cWOJ(wZ#(7O_7)ch;s6M`y553<3DFY94{2z~)E;Og z_;0moRgFX+Z)#nez#KTgK459f*A}SWFcqe6Sw;;y4uGdQ9?6LAFEW%_)dxORpZmaK z@v8DFi)D9BUHR%u;MXV=l)R{n_8n=c(?cq}zq-0k_-Po8|Fmp5uy!0}pZV-l%O+zr zc~C>FHkI3yc^Rmyu>owXte=4jl8qbSY^8%mZM*~IvN5WM+`z*=M2pBdrs)iAB^E{}ewSKt3ky+&>;K%IcJkdU154RrpikSNl z@7gH_DUi$ccToD6;Wb>Y?4&dpY9s2l^L)Mc4S3+}{VPfSL{B&i3ll5N_SXHJC&B^$ z^G~nR0sL>F{r(d2ZOKzS5_unQit*%dA3O>J9-5ZH!yFvb&2QhnxnRWReQI0527I{O z#H&Pu+~h{P0mi&9PSD#5jOnMOE-hYy18j1MtzZ4OTAO&fhjqiz((u1ylb1dQrbU%7 z`-8t?dUYAdbvsc^p9<3dpny>gP%gzqjDHI7{t3G)oS^Uw4E8@42lv{!0v@;BRD)Kq z{&Rr>C6%9nCpR4-nK3*Ro0e0s$DyAbkFx8ou~z%z5x2Qkh4OR$@r)&xRZK z9TSpead^!p!JkMYaOw2bR5lWM8f2W8pqQ9`DRkkXvMwUR_VjzaVT`*{(3=jadKU$P z-W4O))30H1{uMlN!p_d_n>0X%=m&JND6ac>?I!qYcBlNZ+x;UyUcwNVfJ zqXh+Td@_1YFNpk;tw8H1F zktxTzqnzy;%~NCmz@2O`jR{#>yI6vnXG?MoptY-6whhKb9gCIBg=#0E)#K$w9^=0E zm+S`SOl1|1smt`O{XCBPO&8w}2u^%V&kP0SoV(k;$GJ_L`<+R0UlpPp1Xd@5n3x2# zhTUgqoVS;X7T3hZFri(jp_04T_5%vrJbIf^fj-eqA=g0+`GaA?L zVn?jjILtkLH3Ocy-5-hsRIAJekOl%;?#HOo8NJ>4rnafKGch3|%$g!ke|%U&76nbs zvCszTRqWR8G)+#Npl( z7`W8*BCcD(ZKC_lru;MDq1lb^2&s%-YDF|hh?Eh97EXKc8|RjrLV{r%DXu6-W@BV2 zlzS{C$Y>HY&G8y2Xt#d0I{s^I6zkY@h8~R0!Ecyq$g<_92#0bnN{^=cYRt0 zVB|^aF82x_);4=3v7Yt}&A;sb9EOYoB$+pSxWH25=c6wz>uo`_JHFrsjZ|3`x7vg^ zgO@7dZBqP!D>oG@%ceP5Ml?d-= zg?$A_EjUY8!t?^{8rpn~8z*hSPKymvT_4&gcNS|1)AzUzQ@A~>DT8I85AU{A&-$Wm zTc&&YGAYv?Ru#T*iO!>Edh1hc=Azi$l@8&jY4|_WLjvG%*C_7Wf~fq;af_l0sq1TN z2e(X2Ok}~lR)4QQFzNJqJ75uG_R2(G2A8pG?cQ?2RzZxYp@R65oX_rmCixuZ_4C~w zt;xEjoBF9~fI4r`UHI?Rox{+_T&*O(JF`26{d3G}FuLU?=#TO5g%Yp9cH8!|(G*oF zD#WSXqs}jUyzdec>KYH#MZU8cM%|MCOVrxAY?gyjK%0$Ix2F`F-s^cyvkVt;?Sati zfJ=||OCS9GeMc4Iq`;Y33?c7=br`x3=0E=Y#}%{BZOT5rKeqleC@@>tm+9k(7=POR z{fTmr@OPDXO8(z(aSqcP;61DhIj1_{zb1hTfM7l4aM0@JKX(wP$H1Hyzt8@+E>{iW zeE|iT$@PEW`^o=E9s!6LNYeC|YbRN`0Wkdew_5+X`2VHhU&exumIw<8X%7@yv#qZF z9FO61I~`rzbxe@*{1O|>1q2hDdJ|DZlK+-W4d&_8fKogw8X90#YBW}@5ETs#4Umna z0zwLRUkj*lJbhC+9wQ^;qaqt5BPS>4VqB!)0RHW~)3v};q3;tcm1t=m;8wUzGfw%G z=kz1O*c;RN{~I`-icHy(-xiW=`8nb=Ow#LMz%>|FuU>&V6q~;KizJlVY`IDPXW4=G zbp)ivaC^vp@~@c#%uU(HyDs^!mvh*}#c9fBVi^8fFg!`HE(F4SCxw0g*w%Ypu)KXD z+tdPoE&Ekq4o^pOSpRx?@gj+G9nh%j|61Q4B)|%>0u=sw`59nzf{tw>(&z$~f9)oc zcR-oE`qJv-UoV$|yXLr}1pcyS@7us=YpM;^{%hpkeSiS^BkkKh|F!krTnCeYLmTy1 z-1wOZ?z$M`>i?JV(-VM6xV_Rx_OI;#?)3l7sb70Y_DELtOI?f`gtKz86K9DD0My}^ z`??LNfdYxax4CN(rm+miJp1$D2++-_Ji@buExzy|0VQ_W%wvrl%28t}ijH`Iu89E) zG@U2ezZFsa1fj=5aWIdFUl!w39G{qAymKcGgM?w-I}iCwsr%YRHiiq+*z?ovZCKhj+J9ma?ps!U zz^_A*^l4z?`cX18g|ZVmZ(o_3vYVQ!vRv6pd`QQbXWC}5;uvqGtgB1YXkvAm0TuTp zH3N_n4U$CnH-Z8K-~!!;*XgLKZg_bq0RbcSF8li@-gU7q=rRvh&~Pc3Lf=Cw#&epo z>sh(V(Mp}`l()=wLx_eh1C0Zb)T~1L5VxiqnAtOiSHi?VbUi|IV9zf(ucl2x&%Uvlt~ zElI(pwESSa%4jav*v?M?PUjdr57yls_nSQURa!-k z!|b{U5psjg&Drxu9*2AVOLWI|Vw2J9twHGI0>8dHQ=9PN*Wlpb;>HEN?J|>61!`#X zowamd&QMNRS$%?(?<8HofdWvi4H5UA*OQd2ASjwfv512UEY+LV+e8iL5BtY0E42#& zdK74HVbQ7T4#!qzVqmJALEHyzqKFs zZT-*bcaHY$>f?PHC{1BY5HThu zDI8)^Wshi)x>lj>^7W}tJZ)cmXx+q*_D1y=k0>}b>3y5U6e35URmT+$3%$B}9&=J^ znS^}BF(Q_sx?&6rk~UTA#E!a$cRH})DKRgS<&TeCo!9*k3Lfq{O_&M}4X9hDv92+n zDESqKJ(qRuCF6s1oHuzhTRiwzFRI1sGv=AJX46X3y5|qMFIQwv#TX-vY?O^6y`1Z_ zpYp}FiSBATE)R1cj*cFd)D0Df?=6wBa$_76@E_=bW84#sJ{L~i>kSTNWo3i;;eNhB zXZx9f(yE``5@WrGpiRUuzYKYlxF!~|t4&aj=$hIfd?*jKEnN6wqP7X?c@RW4In@NJ zZuML?=Z(vfQ&LR(o@DWFn#juL4z2qL8KDBiWcyLvRqG<_GXPz^;S$G(?!239|ACz^ z^%zUQTEJt$)3z^>>h8lEbo@ z#*Af-4uc>0^0gU1kHYq_dl<5+FxwKpxG-)TnH8I%RUxY=x_MVC$1OsB&t4%e9XZIk zkDO9e#}HdzUz^-U=(4$3Ee#Zf*~@#uC0tchK3CkyHaJqsH0+wm+cc4|?NUVrBl0pz zB`Zpv&sY}?Ok{HFwr3U3TkI~>5H~baiq(h0wHN6w;Sy0=*kH=zYl^C$#$$Rij3sy8 z-6-1!xC_y|mharwI;5ew7m7eFQv=Inh0|~$|NG5%>&o^#TF12QlUmz|{+wOALi2u3 zerUNGV&N*A=TX(Z!g4|4l!*Q7vOG%sr1J9npjF-6{G}8>%;9O*=@utYG5qNqNB8F! zjA2(VY2LPtS$I_NqqNZ>{%AB&X)<)s7F6<=K6*DtoBaK2)+BaisW8H!OpzZM4ZCaC zebaS&U^f$9A!yc@mJ)kCB`VZ-Nxky9G0cA>D<*6dqfUL8(Xscz-&gk+FoS1-2(|FV=X`JnTjrW2x3T~jn|XF>=xUgwb;7d`l6|eAV)X(D|EH}In4bB^s3ORVJ~oAH??cM z5p9XpsA-(tZy{{D(8D%g%lQG6i0gphZl=(3aG^EN)UV_68^>-37rHFrQEh-ftpol9 zGRERGl+t$QPca1z!u&gzxaRVH&V<|a7Y(~5>ezK;8P7Y7>}iXP=Bgx0`#K^VzliSZ z1jcen(?IhNeh4!hX3y;{4Rtv7zBpd&D#{vise^4%F50&V>*{gq)WhpHgS-#0!nLMxu zO+OUft`o5P?mW}DR5WB&r?d+ucMoPuv4#ake{1x(=-~q=DHFWAbl!mWH5qjE6d!we z=R%uIvd$~|+@ONaYG&TR1Tq$5w>JmvjeLspNa*PgY*$io+^a%8hR(1Msuii_LZzcUOM#8- z8Y_zRA()VrT!|x>e#tW_EW2k&mR>M{gX^u6C%BT2BR;RTmg?K9KnZgd$LBe5a!XRyWnb@6L9`}~6HSJPkQ(K-$ zty{~sqj0n2Iqww!iEH>7eaLUd!OQqZ9Lt3Fe^f2<$hhG4@yw<3`ir3MInQPV)Q56t zFTO{hbfZNO+uQCxQ$OcEAH(cImL%`zWeVBX6704lg*C8cA%yGx2Ul+a*Yx-OkG}~; zL=Zti8l{x(E&~urML=R;&|`ErqN0L;bb}I-qXwfxL1J|GL~@J}Mr|YhFBtF7_y2pl z$HNEWYp;9nx#ym`=R8mGOdG)Wyq?oO6)TvIblZtRF3~o5l09@-;Oh`8|6||7me6Vg z-=oTtt`A>0AX2;cX)^;cyHckSrkeOrv8}OCHu1h_p3YXjlJY(kZ~_d!aggahZ9r>doNXuxt4Nfvz6LXLR$9{%9I=<9ea2B=g z7s?tLJL;CmVvDB!a3L556UVQWdBC?NIud1Jh_3RIn9J=8qf?u(EVrh^%>>hl>8;SZ zHL;1v6Aaro&lG&3;E3m}J6!(tMJeya#gl5FTS)(>XBk5Pcq6KDsiq#N<*@o2o$N5# zTMjgQ2xBdQ$AW9a@NxnB5b+EElhOI2fvfp1ao}o(?xurJ%?jG-hdOZIwNfTPEI~Z^ zn*Y{4bA6+sSL=wJvekf@j6Lxo5u+Vv0iYa(OMS{?-69lqZ<%2Qjl6`iiqc)a#&6AJ zj!9btDi@TU2-*1|t`@~NulEb;!S#t@lQ(NvfxsoZJP}YUfO!oj{Xy3s zta_16x;5hW`YesS%dpLm0-kY7wD<;URljhdspPQT15M@o-2xC|>z7W`nT8naV`f7i^bos1*Wdb$0iPvi#61bH=DzvdLWoi;L&R zwm2V=xqD!R?4`@Votr_>n3eM_cIri9(xE2-Z|<=E&(%`yK6TgD@|TG75bLxfZk z#I(>WEz#}ABZiTz*J+l94Z97T+^wD8X?Z%zi@gUW0Yn`Q#Ny~x>(1%Va_FK$^NSu` zvDUpZ52<|DvV0J4lAICO9xSvtE|FE(>7u=is~xi~kKF!{m%s82hB?OZrg+{!q_Opt zwJ$dEi%A%LLhvp?bpm5I4IpLh2Pg@i&U~E!UX0|(&1~*pi0G-ls;Q|0E1@HN@5Xb`EUkkiSZMF+O45=K)#prCmA3)TwwF z*tW4+KWXTT-d1q2(5~{Uvd98?;DQ@~;CS||PrBGnWc=p~Z9r()wWehAs-FUKIw}tp z_3g$4TLgn3pQ~qQP()IBuOBOZ=evg<<=%PnCd0@?|P|D0|)& zmSXhdqRcfkt$L!^bm)~-=1|8!8NF;g!(5vR-mb%~Z{}owdurD3mtNF;myULt zteWCaSMW#wyZum*+<=3Xt*vd<&77Q^a&L@PC!$cH9bfLZ#uRJ=B0zCWZ% zI#?3#6u29MXs3h;1TRO9>sUdNx7bn)YV|Je$MX;I`aA+EO7d;Dj%>Cr(ymZeBq4W> zuOv0qNd8_KQ`9q9=ZDRtTwNK|4!~|pLGK^#ro0n9SrW_kwc@KdVK%B&FrPIlHGs*- zaEH5G2#ZXWG)PZtY_0m|WHEc7yoo$|z9-PI3Da;`FM)&xZfB-{D~_MXyG5h8ZJLf} z*epz*u8$$wrBi)>Ruhyj(T);0|mW9Z%l|*Sb2aUN5h1OciR$(>B&=XQ9&NBxOfn68@IL4t-@uVC`(O7pd6V()WTS}4L-rj4o43yFdZ5 z;rk$ItM?!hRxdaaFVs%C=0LK_;adB+9?SQTgf%TN8x`%3!HX?w)53(1jT%t@Wip1L z!We<&{9I<^0LpY27jEgQRhM;c(Odu0x+M|goNz!I)bK|avXk`e+?qB%5X z)0;*~ebN?rhRRX1arJ2vo9f)u3YNM;ujTu}%Ig;LtqWQA{`_f3mo+!}y(#1Hw&9jY zk%@23!H8qph=0JZ_BO=5UznT;1f;NJHnaaoNfA`R9(8+tpf$>KEIIzFr%y z6ivQzDGF=f{l2|oy&ee(=rGONGZ zCtI^To=j(bbo|NIXqZle$5t2q$}zLtMx)WM)-4Nibo_GNV5m!K(5Q#BW5%AMs`pWN zq~`@8=#909sho-In<;TAi?j1;<1!p|o>9v^m|ol!y=Yq$$cAK#H|Yg&dV*C+lzEbY znHJvD!CGlIG~PCeyb35lZ4IfI8X?b<*re|fD}QIkoSt-CA#X|C^=9Tq)s=m%n|BjV z9FxIHCR!T%%S$&U+EV8Gt!*QLzMfu4d+%=2=QtFnf53iTs>9wzm9g$o(Mi&o4S~I0 z9~^e$N98D_7Uck)sg$5ZP4>dR!Qn$0N&a?xb-5VToi9?$B~?Sckw3~^m~>aghEp^> zw?lPgI;IZ}x=FX(F;1hN{>jSNtTyzz;rfzaxUKVO_RJicv@`5katf>CA*`zxv-i5U zwpcQ8ud3TWeKy<6cJy*W&BdYgPK=kkw~pBJsn|m9ai&X3Nv_`5liXTKv<>mWR3s}e zN4LZ_c97BquU27-fLY?)^&Hl30jYFC#;@NTO{Qq|x@RrBX5G~=vRpr*#obiio`vp0 zugq}w-W)Xf(?eCid)vw6bLX9FkF>OQjc5W9eh#g~r1!4_`X@LmA{Y2jZ6!RSnC>=L zSaURbtxK&d_1;db+ttVnXi+i6IuM<5vr1c|+_$iNIf9oL9D-Gd1GpOscpGC(LlrTH zM^nkm(Uq1H9wU<8a-WA~Yi$BC{C{1<{)tmW*S5iK+6C!XeSh3&R;JFX012O&iERLV zjp=6Bv*f-xT>}U2N#r8n@a&c+Z;qE{Q((Jr-ziC#7~6)ucv#M$x=@m}U54B~HNMh* zv&>u3^(Ys`dN`(cwOookK1Hy9L%mL~&$tO>mRQyu-(nuMWWQ`?rsqbxGLmRex4X`Y zEUc*i`_sjm7H?#ij!v^Bex=3uH&A5DYv`yU>w3!Q3rFwq8wvr>ZtAQ@=b@N}6mV`s zU4dn`SRgC%W7V!HVm_%G<(wn!n!?%JGg;ZJna#zW*z(&h#9-gD7Y3gnSfMB-deBX6jK^?WJSH2e_3x1zTEzXlSQjtDz1lG1=DulB zTP~MhK|Va=$S8x^b&VyTb(A2fuyrJUilKg_xdT$cnT*OuL_C$`0WuYF)`*7Qp`77? z-Os5qfPF>-TvXl|5LG-H`FR{K@oNBo4E`#zrtLmSH2?>(6QWT?DXpwWwTxY_G;^U-Xg8Hvy(nVZbo_{*+4fvlz@=z>~}EQy~v4?&+wVQ7h>t3d;IpOZVI}Q z-1lK)eLd>}?y$CM5AC{0pB#+wQ@HrM(ebp_Klso>y;re1J+V-OJ+O51hN^biDW3aS>Hb105qiPabIOhoDRh_%Y*O;XV^TO|_x2%v4P=rC;B4ME*D z^CY$P*FHcjGA%y9FbV)AnJsBMUl|TpFX-((z`1=1Gp9H#s;Ec{uuIrbxS^jr7GyG1 znuHYN`x&7<^K1IbfElS&z1&a7{`K_la8#J4A&LX%E)i0(Ue!{P9tp~2s;AoCYg?pU zh+{Jxzt=epA4$1>A__t^i8L~41<6faF({qbH(7T$U)FGirl4MziBvHYC zuf^rW5VHUhQ=T=UXM@a`i<6X~vdD4Io6B?p|HME)-z<5*A?{gE@$bun>YX^!PYy?zLY3o za-rZ&gG8(%^;u=H>@q*E+*v9a88Jd)^h{(CT85VE+st+@W-5zr%cJ*x4fZq@tIU#d zfI9;0!Ns;8cMsWMaSZYTBcwiG{z4vkGnPX=emWv~b0R{jqC7>VOg)mBs2Xce(0LOo zxhi9rM0ccJA?WXn8h<1m1;l&o$>@xENzNjrM2F)C3AC!VGMV&5S=OFaZ=#4qWZ1#v z&u|8bEV#9m)#ya;818rp=z{{l9Ta%O+K@iS@j%VN&`Sn)H!X$xEXDH!r^JwILVg@D zRU5iLa|r#OCs-D7Tr}gtacvFd`ZI@B0JcGP5{HWRMbayE1_~a&<4n2Nc)Z;sFGo{_ zmp#s%31JIFV@;Mw2YY21l~s5@Z0k4DVC2h=!!7_x_NGn1M7`?W zr}@)YaBd@0vt(CX_qO&9CAdGB&HF8$zTq?;UVlkf?L2>hnh3a`y~G}^@;Rtc5aDe} z_ixf7wE{5GfUDpga^g!c{@{ancWLH-?<4SZ3|%tEJO6XjTPpt3;e79ftb5<`@WF7+?U8T7h|$@ab-zHTd5 z+SEao@!0Qa^jhGSP|i@t%coDCb22{f3p_Ns`@E)hAmUD?e?0Tw6pC#3gx+%3fxhxT zJ^MNJrKPjS?~z`W+3xn6$tOpU{o35Lk8Xg!gIhTgU61!QwbXv39iMfwTg@LZcf;sp z#KL;SCnKL_QltG=h{e$ua?&z?eeyBBg~W2Cj&0Or%vH7EuHn8p^X>JER8dBO0sXFJ zM~z4nkH)eHhbe+i9`o%w)R?a!LP-^u=2{TzfV`CtnNn|Al_>wcwOr4*QVXfIM0{I5 zS?}T*|46ko&WqsVbnGmDAw^zo)+=M(u`M;bSDPL$v79=T%`c@BEh@$th`ns|cY$KB zmeF}gBD%($E&tBlFAL#UCXVNI%Ks)MWY?BIbu;pFiS9mbYO2{@_uR`L#Ef$+qLF6H z$;*lZ$IT?!Cs^ySXNH~8C-r8VH+O4Uz2;=1-J6x~PF5Qk)MJ9L7lSs_FASY-!?Vl4 zHl)(6uDmx-A07ZwPQ#XN0k=gMX&n*$E~*D)h=>}@ng|;7*C0)z z5MFz{Ei|&s4KI|^!s%3G{8v|LcD=AwcPd;>~??~bM(2uQ_Hdwr) z{a2*N{%CWAK2~}15nWgE(Euc?J{lb2C$J}{)JFQwnMP>cWe-zhrkasnzFmA3v?yd; z*HXC=Z!5NlH#xb3)U2N+ZCD&cx4s4;Zc7X}oE#74`g&xS$4H9}`bNH{+D}0)*@`XH zr5wMAKwr!k-|XB7oK1r61_q#-M0bBFbu92X#_r52&ERuPsPNYFED`;79#nSliNq?C$;=#^{ z#{duy^c5f;=GFQV*D3L2&9Cu6qHW)4*~G0%DN#!EdMx>X)B;5!u=9}UCcDpW!Y%hw zy+QFqF4GRm5pFjX9~uL-KW(3kp zP10X#rZS^9OSNOraJCfbo8|&kG7VI2j^Fko9&Jg^s=+a`(1mL^k{~jYW)>8jBh-APuU35l%wO1Wo?q~ zj=ifxC%;t>i34Fr@z8Sn0Bq6oFj@#I~2*l}Ym}(Y9qp z@zBulm~{+Y^{AtiaQj6M^jK``PRRf5 zLU4ln|LHWT?}n8sIx^lXH1(FDoqS^sbB^)yK)A-Et-no}Z!PthnWU%f7HD~Ez#&os zmBHpiIm-=*>VfF-f;fnJ9|$(AuSjWH7tSJZqfX=VqS~I{uKdQGe96s!Hl25b9-8d? zX1Jx2Kh?--{}P%)H?vsoy4ANd$;heHbYO0h+R{|_Tv7{v&xKy#8YEch?>6#774@^k zSrR41!^H4Ja!D9!ceu43h)cNn=v<(2gOf^YY?6QFst%A2O8&&%6c zIMIqH#*;g%x7t#w#xuuVO^#+?cD(2W)pk!mny{s(PmW(MBDwfsX)b$RX>?k6aML8W zFPfyJdf1fJd1*V(!(dMAGU;9-Qtso&*+O-Y>U{viDlcn(9Z7i_nMgm0e4^(2sdZ!D zrXZ`~B!{|gncFJjFsNx-VC!yYKqS1N$aD$h(JY7?{3p;Lijho7}8MYHjBn47HmX#v-Zc#yO$))Xo_0DBQB5+neL zimn5Xtjc0`!aRtyg%_EQLQ^=)w7^Ez#xSV>ynn;4n`$aH#FXUWOv%Msb*rOqJtn5; zc>3#I(`xCZEj1oo8+OJOyW{Z<$ML9DMbo9DBXuCNcS_TCM8bql5g*MjdP3EB!rHkq zgcymNj(ZtJs}3=q*w|r&IcC6hxXW?bq6|Ba{lET2Gah@$YT&4>~*)S=AiA9yRJ*@NCWe3`K;4$@q_gWs>jWUhl zvl2i~{_T?Bl|7lt;tYqB(UjrkZ|+EPTiZR=2dF2Z2%J!38Tv+w0jgNc%H$_>zWs5N zn09=ImM2~rzh$5(!7D1T`4ie;I_kGmvyj3*X=r>ha-B&NZw$rk{(6Ix5$NA&Da3os zc1`Z~msRchMWTK+_KN%?`9tv zV-40l82o+1@G@gecjA_eJ)(~TzhUaR{L=&aEt{&O-qj0O&ocSXl59cq_

7+voINOk}X`)$qq&7@16)kgr6$DM7b~RGudQ zErOu_7N|*SgqjZ3H;Om-h(r)Y|B4P5w7P7WeN^1Gt}B45OQ% zEO{wpRY4fP6p33;3`YjGj$dTvFHMnzJ6t|Pg>2MWkhB7^zft2i5Y!)h^3IpK?S-F{KFLU0GeX6C!^s)KQ^7{ z0@P;mlDfFMT27Q(Mgx^+woAWaa-U+JzcKjZy(DsthWIlVP_Aki55yFcWotK1X#K=K zzkKpc%@^w$r}tJlikO@Ik-a`f&1%f~b#^l;BhKDV`fh{azHAmt34VbO6*yN@Ram-I zSR4u=+oNSMeH;VSaF9IIkB?94D1Cg{peTNpHsZ@VhJJi;a=R!?7o32Ghy}*3=Z-wj zDBDYj?#3XpHfz=6o3K!CZSQ=rzJkGedZBuV2juQZ_Ds`Ik834kCS7un*HEnOUdA$J zMAikC$UY#ZylHRTXLz^?s)Hok61hiLeYl7(5Vp>c(;R~Ysv|O9q;rjxqeZ0iXZ{&l z_X5p8cWX5>(CgwpZ~e86`XHrv=k3ajTtr~O1X2=ni1QPBWVl!T#KR-UJDXibA*;7w z!~C+8L+lQ#L~y=r_WsIC`h@j$?P3lK@SqyWPVPq^l+55VUgF&|gNm6y^k(r#TArRP zrripVG#NEnHu1q7$-J7md47!ivK*aYJii#DZ#3$0MpX1|qa~k_AR~c_H7}AS)!>Um zOGXEmhx9>?*-_sNl}VP{!srpHGOlY%Y21ksvlx}mX+SFrTy%#`Nz6f_fjyd>av#uK zQqcfvjD>z#y-^d%de970L;OXWTbcyN^zfwx_^q(LyrEh5Ji&g}famJ^uZ9yY`i`%W znxY_h*n0D;iVffAFmx3+WvtqaM+pDa)Svti|IhVUW6Wx{=h}9(z|651_a-nB+{gYp<*pF2nadNW(lSb|O$oZ1Y)?OOJqLhKM1E+mf68 zF}nZWXuy_xgLQ*Kh56Bdb+ipJe;;hF(9!aM((v&|Z!LWzvtV!;m8@h3%ntqA5m)5% zV6c=48g&J8=z8Dy6Z7^MzT34c#&Vs<{8O*FmGQPK04x`5rCTjcE6&Zyt=>5bDeq+t ziK%7wi;^*O1u)y+aZP!2tO@r*z?js-D70Y0_95t7pCB4Dm%JlF_hoa5p zaHeI7<^hQ^p2ms_fyuBo{CBmC*jvAvhOZpv^aY_k{YTA6#aR_Coit{w&InwHbJq zVS>3MnKU|J;j0yO?Pr$*B{oqb{izh*#`K^2l|oCW+0Bi>&gN(NgP1lj`M?h|C1kk| zXg`Sch3b>lp)&7xBx8Pz6b#E%n@Nn@sG3v1|EZntUZx zQHx11mEEW&t#X0XN#2?M?BaMvFVqX`IZpqRT-t6hUsinKv>SQJ%RNqr1Xu(b)1MND}CD|SXzqLstj^bqikIs z4n`e+=Y9fE2`U>EjPgL>?L~Qf9g#b1o%|sS%ORIGi-1+>#AMUfn}pXrxilY^WRnGv zTJASs%||PR<>EDD;&aToifZ+*H`%Szu{P{Q0zH7BSLhs&c&5aY zIBHAz$BnwwDa4k?+>t1=H|&}4`kIJO_Xcs9<`G9bhj)NY6! zHt)-))kN3!%4}KXBT__BWi>$T*Kr^H{Vw81BakeR9Dx7lxf_mf`*U?d`}5KXzMy&zwoni4x=f$yMaGfHnD;?!4K9 zgFom^_=Pbt;1lW?tb7u9Fe@sm0pLSG=}t;otz;CB1%3Yy%w(V%cp%Q&`gY1V9TBh~WvmC1$M+Z`(E`^IRKmvmO9?TNpbUJ$dr7!MmG~_1T=E z?Ck6|%5n;!Ns)U)YI^Zln;JnN+z0pIPoFM(?+lPx%+yK%m4}8ki}7_mJH~*1jzP*w zD=EQQLEp6}Jj?=)dwmdYO6RqZ#&1q{UVtd}HDQcQ844TWV)c*}E)BZJKyk}DEDbfU z27oq3a@__XQxp@*2vsM+OM=bdiex-P_90<|9zMzhdU{2F`C{ujw1EV04`a1bB;UsQ zM(!+>R$0F$u<|#fAi7l>MY-5>=pXLuk2Jts07iEbLu^rVy=-BKW}2*#q~nr)jF#8s z@f&AxHZdR`=wQVY{)bSb0T_r%C-u4;b(`8#k3puUoW6T&wr9wUXA=>)&(2-vH*{2_ zVp_*cdiDa2ODo`t%a^ywbD3J=`OgxWjMJ4hFAfWAin}~KpZ<5{{qWco{+l`H)i=b% z0GZKpf=4+{33x<`CE+pl-zGt_<8Vp>G}tRc_A`_oF%}#AxI)jrl`(VVqy?`@1g<)6 z+J8Q)T2Xepe2O}n_AvHeRH6IVItl>{5S02?`z}estHmh*hPQG~zQaji!1Dovlwsu5 zx@uDlU*-S(&3PqE53o`aJ5^=~-5B?~{}(vnn=$ZTt9gnX*Qp5HPsiEM!BKO8g%W5j zb4HWXVxC&oe`|<>FOjeDhv}+XoYy(Tp3QedPLKq`ma`o6iXbnjcY(aKab-l-@jrLj z36s2ai2$b>KPPy#B~0@hTuesJPhxj9ABTO-Kj6A&P!~`EQMq z1hA~_1bxDvCTCgvq|8OxC=Fb z&JQ!1)9qBT5sel1Kc21Q_T8wPo!xJGz~iM)PC*bYdQ%yekN~zEfH=KaQ0U-hSr%|) z?7cgZu(nS`vX`KzE%Q^ey%T_I7X&(Ycor`!e%)-ovSWO8e>ds8VlwzQDe*{Ia6B2D zew4s>ncn1UF+s9`n9BbCT|-qhSEEoXc{hQo%oT;TjK%IFv|<#@rPnqHrlfC(e+ z!3en31w@+Bzf=c6pJ4|Qo(3zd9cxgzazJXi>8$UjmQkHY zB@CRv1qe1LNcrn_j{&+3<$QmcztCXr9S)6x*r*X7w5D#Q*Ci;v0o zEm~UEfe?uV;dYu2%*znV@AuH<>=z&Gf}6kC(%xwL_RSQ)b+_4<1dqDOET>ZB0A;1|a_JMRv{9hHn4{nV*Lm;B=ttT>(iMcWep+&uc{@=M|V z+%hXds-u=r|AN4$1wiIcT+RUscO%jk+vqbSvH}dJ zGI1%6jj?hY)nl6Uo!>J8?$mjP&ML^2)N^NUd`ag=Qy@~zQWgGF<_5cTij0>aWX>DI zJ)?hh6CtD&*U9bV@Y?J<0KeuKVr!k_*oZaF)k=9a6_4oZ5uQ<`*=*h+`e{!%ii~%h z=H4ZA013VeVM2;kCKpfTp5yNgBSN`xu+C9~?`yrcpV$$` zHjY;YYQa;Tespe+7-qcpe{n;9I~{4S!oK15@d?~h z$jRlI`?vNJ1>jH8zGk$#Xv|w`QEzI}A2jTjYV!&WOTJa#4&y5?92;{lF;iM?xQxCm zn1_I76yhiJ2UIM#KHd3o`}gfvE+m(4Z`0hr|0+mWj59{liW{@B;XRGZNFM_Jlq7AW z=WpQBu4WgYoeNe?MqF{`?yIR8GjnbxsQn*vF5!fUE^sOa%g@blZ*j4D zl2%u8-QU58wugO(1tf@M++H2enEmnyzebptDd+DR)y`ksK+lD`73NYt!kYhQ%&FY@ zUBEM(ge&cK54e`j=D|SFRw~!L*2<8djva^m5Wo8ms(aqtB1ji5p3~>^aiy@;nqD|o z_+0A+*N`tfz355#xpQ^0d0Mh#kTA^k;Gja{)!Tnx(f=oTnOYbS^`@8pGnq@>51L(x z+!xV;)CK81^N(NNAgtkBsr?hoa7vO8^jPmdzex!5xpdti6MW}3c$N2Z-L-Q-!WY*_ z<>_^G(lwPEE{!pyoK5j>p0dS{EVAC0XS?im9~h^SrC^LOz-|`wH(izKfHf-U+(?m}eDeN$K~e9Nb44YG z{AOB{jc2oZ-z2M?E#cE6;5)nbYX_ve?%P!e{6CA!+#!``(DkI$+b!A=w2ZiPx)05t ziRZS|r8emB1SA4wv(smmRU$P1rO4TC5s}IZ{O$}&_RLf8?{WQmHU|oRCpVjJduMl$ zQ!u=HiSYcLq97iU)gRH(|0$OT8lwXQQMoD=t)O_Yf)nBLG`9&FEH zv@}*4__85KqjC^VFk@VTH7)IPu^{cr&5_FBg(R^X1P;1=!%1H`E&nRQbeSF{^9oL2 zfah+(*!Y$&v=98n}8s8`6Sqh4GVx z*$1w3x`>{>NW~*y{Im#Wikw$HtrOBQq;&wg1YXLwWa6I7h?7I`&qrrD*&a_Z0D_}_{FT|3XW{cJ*yL8rW| z@byT5XX1Yh_&>eaZItQP3yy8R{6L5$`qyQ1a%S?-bA5SCN*o+if3%+P*kp=(3+v~y zKjYaU)+ZmAKndFq)l{5hx4|p#_^;MmXr5bb{NIACGjt2Fz!|?iSaI6W0hq}&m#&Ek z4+s>x$Y)-o>QJ{%=Y4`#;Z8|L)Lo`%4}-)R0~ySnj1Sy>$A0Y?`B z*ACgSu#l~H|GQvTf&f_CI~);T9QF8n)jrKh_OoXXLofve$*~Br)BJrPFUf*9B;=@* zTL=eJAQH_*+ds-um*X^3H8 z#uMQf6GC>ib@U^EH5uM6)GmF_xt#PL9{wnIT|Qbm(eYp9ov3!MkUu$`g-O-d)tV*6 z8G*qtsn0+KS>gKK@0H8hFCEJk-zLLYtX}2Nxf-kaez?OZHLq;l!#=GX{!c5)L z!uTEo0bQ&aO^kbJjIhy~yqOfCcKb`J(U18U?nvlY8#!*U9iK~Y%oW1naSCbvCjWJ| z217sC-!D9W0EbKoUR65G^fp9)^PxV1E_FH~M#ZR~8ubznI z81A3fmSv>=4LZm+r|?-id8jZSItALZ zb$)u*4BQGK*`l_#%5Kgd98}vBe)WNywnNH?)|a3CwAi%PHxLvF%aH!7mhbkv7&W~jJxI;2uU}Mfraf-XYlY@+z*b9q4E6NxE z5aqH%cZ>o9n_#e_h@|1MF~spvQlam6im1xPrz!74L)Y${2eJs8aC&OX>7K``c1Ewy zq$>nQRrD$LiN7iLSeIQ+LpcmI8x{tG`H&J4mD-+6_lDlJ2b18VEx+u-wgwIjULt&c z@_!(hIPDkg&4*d7fBz*MsE03FGIZgUkZ8{)d_(RN(c(t`V{FM6p+GFf_F9EbN)T?f zute<0eAW*Grz~rnst1pexQytJ$&OTzEjHpeoQc;LMk8!ODV9{`{$+YiaXy9T z&F)0*!I7S-dkq3$Yrm!c*(+wv9vS)v0%aEn+kBZ1P~AR*C=;>4S)=Dh|6(c!#}WfJ z&C}l3=T&5Fw@)9lA2j4&4m~0mU6~f7#ClD#0aG&1McZ40&lH=NNplQQW%_mpf@P_D zzu3(B{o%jR@$YCa$td`8a!-n=U!M~@6(`DNpF_#{{oLmzTKeW%*MX)`&9^ z)?9b0t17P|-0qHHB{xNuS?p4b(IFnO=eFRbZ0Zg}2IdR}0snTe@L4oYH~jP~V!JPs z(1)+&@2#DU^AE8Db%!T|rUueMx^V4`5X#Wg=Uf#MYl_$uhaKoP9c63<3(NS=rQCvg zP<$hXRGko*3FH1f5#%$_Iw4H>Z`K=^a7SevZ~^}zVWaT3XNGR@?dyO2ltB&H*xT}u zHBrKYZCunzcc4d>js(M^Llq#-?t)FnxOn5Xi=-<5zofxXUxJ#axCl%Q?yt%+l4*Tx zC0O-i6kej4Ro06(7K0I@|60286vKfl3bwAp?>h~~E)EL*)I8UN{J;*$ir!;iUkZ!% zKl6F+umn#EFlj8mkftF#LqmFMM*-$#?k;TvX!R;wqbI)SV+@vHOZ`Jke#+sR7}|0% zHo1$p?`t4CEeP&}F8>+5*5^fsaBVJ6$Imf`rn#f?!?h#=7r7iILupteN2w@cv9Cf$ z2iPD(Z{r(ZbA}TvtfW&D@fieFMqQUCcVuoJnAgHUZ@Y-Ni%d1JzLpxQX(C);jrI5+yv784Oc#B^KdOfIhYO& z!mS0#&VomG)K8yk=RV}}ekyZ^@V0+NQ1A~5z_fD6?(&=~5w1E@OuvnS%R)lC%ckCU zM&Iin^c203d*%vV4-2v;u`|D8-}_{qPe@~TN$=C5Us=ZaLHpeNR&SK5;W}l(xnLM7 z_Cp5)Umr--_BgY-*gq;-!QM35l zmY~U+-(_Tsi?}9sW=rb6J>6yc$<6UH|%^vY3 zB_>AkWE?i6x;#!f@DnSv>2q&VwL$qpE%?u%xpW^ALkqSWG9BhtI<49hzCvJ z_C&Si(z!UO)>qD;I~g(e?|+6>bb4WKP&YB;YCq~yT+EYXt5D+hc;T3AA$#Ba))5~g z*Y+WsrG>?k-h2AUNEssc7=AuBTDxUQi1J(Z+b#Y1ODxPUhyS@ zeiXWFBkbRHgDtA1@6mP8*c;cX+lz6dm~a=?0;HFm`Enp6I+Tg$yVs(^ntZN?()lN z>?DK}ahd&hf#;b^;NA&q1ZjuP|9sHfgm@JbF^$bAe~GY~&5HOae8wQZC4xb90e5kM z<*{ey{*|~zIg<}6ic~87_SXy^1Sq>9u(qs!K|`3}g%RzO18lnMhpP_f>460tmdnmw z`!svY97G>3p{iQB`H)MI5cOb>&JXVqH>ZSXm992-^9+xAYJuLdSQ7NS@{>!a4#=aj zj<=8A1uo5@eTV#FQ4``^30v;=hW_5Pd{zbZGo6Q@(Jod2ej0Cc%uyJ%f$Q69f^Nm1?Wf=AiM1n zRpT;oCM<)$T`L7_g`XC|pCMLFv?jPJve)3x6Ma_*0?Y|;HAz+@Xa|l(=Id5VKj&6s zTgx2JMNCTboZN;)UFxcWoiR?ZO00{ZFTE5qM>W_R{~6oQlh|Ud{GY&dq5I&TCZSKO zS?F(gU=81y5uu)wC`zsDJoDRL+&Xop!EEuUr%;UEVzEHjs_$^sBwSE!-zXoJc&0{w zs|5SSwj6go9$*?XUazy#fUSDgTdD=Z`s^6F!wg@ZVPZA+{GYCErJbXAr$Vvc`f%Uq z<81=*|JEu)ElV5K)(cY|TIx26PukGg%t~j{i~Le--QrE)+HCPP8~&NS=j7DY&SLni zy`tkJOhvHqAg#ArXQO&vQpW4AG7!HCw?TX}UN=EzIUvVgZ=^d`ug2s~!X=-np5JZ; zA6^>2s2t;-b^|#GXX)!R=p?{W2LHHJ&7BF&&yVTsB**@eI*ybX+}d&$;};T2ICU%z zhm=42XQ>2BctAT6n6 zq)~6CdX#Az&`8l=7|wP{kk&~nZgX{&5w*S^_x0;7%t@Y}gyzt>EAdFD&l$C$$eotP zGB{dwcQYKvTFH%Ptu(+dDXi9u#hh6*L24K5$BZoF?lJf)8V`lLjAqf3gr43X#YP2z z({%x8`iB655l_bP&Ik=v=pwYvA>l!&S?sT0+%f_6a;ngK?s`LmM>#oOrHrOeIJ@7WvGG9 zoAwm{@=^Wcn$mA^%_{HFUckhaGcL5&vL+@-M2G86TFMQ9CyLl5sU~i? z__N4tR+TZ~_)POg$%uk76(S%0a5-@LJ~8<^QsMSyz`k!zZmt0i4qD`KtH7+Qe;HU_wJfIseFigx`CAs=B7{3jz%K)B(L^bq^;Fn#rRL7}X!)vw5A%=Qv%RT6Cz{pHMNc4Jq z<_0vrn%K{Cm;VD3nDp!mOtwQ-nnr|omwn{#a|B#X&+1G!zUQsf+g<`p%t_eaen`<=aM`x+on zL#p(E^&4A?YVg?6b_#M$a>h>O_BA<~<*$9H9~?%H*Az82Q}tK`lCBRC)PIMTa-lyD zHzT*kxgQW@a=}z@(FDLZKs7ebGPJ&M}p73aQw8(2N#WPzJ za63kckGs0Y%O|CixN+GLUkIuVuvpS5^VZXxGlo7@)2-nY6STHA9aw5GN<9k)a?2hv z=e`!ZuoRJhq;X7q-L3DTlvmNp}h(jC!Ut6#M*CY>gIJ z?akN{+wuJ(z{v+=DaMcUAT>-oc&>>a^g+rL2FVCM_ko@h^^pyCF(E&zi8?g6ngJe# zMa6kc{~u!4>!phux}1RYn5`czmuv*2pl1B|qZMYUpMTDkh3RQDGgpz{H7(<@USUR? zAwFVLQDcl@($h&*ANC9@e10?Y^hSp@gFOOt61`G494wmQ{g?$Q%|ICww;I>#d`ie1 zuHOYgvU+UsHN86St2Bif6N_tU&ISG^*9fFABfMbd@4>_Hq4>(ZyOn^M=RV`c#8dX} z-?$9dnL#X399+Cmoc36(1)|UU%sj(89MEiT%WK3s)uEbDfto-QR z3H`Mhm*PDZMHkIKb$xb#Qp9y+E3HN=95LTK70}J{7wo`jJ?KP#zc_gI@877TB_=9$ zZ~ul^Ii38Fm|I&i#Q~vItkGYWuXYqD1s1+_I50#pGbdE;#%&*z{Vnn%1nPdX1UpGD zbcML z%HxM?;c(y4`UCQq-r5!}?Ib3KINdvwLhdS@IPqGSpR(Qu3HZf90j)t(k(YK6Z6%0v zRtpml##e2<+8ns;s05s;jmL+06Uh&NYF1HHw`V;a2MBiTs8$VKPCh8iW#Spyj^E(b zSi0`Y4p3WJ9cUrJBh&B^5KH)?kxHTp&I^7t!rW(+iG8cgP90TPa_n2PR=(ert7Xz% z?UALwA}%R%E5LjQ1)GUqkEwOsl@{66;ZT7L8 zq|U?#pT_PpN=e*Haq07p@afEZVb={5=vM#2mg`?H zdK!RT`W(^2bB zFieK&9S>2`$-toPL=HiF(Vok!b(=RI<=&F;^WdyNvIp>JQ}9RwlU#qQu&}lhzCyVa zMP$+^m=oij&Px1oz$dSzNdEY)1L`)^~%aznU~ihjK5+EKsmEf1||DQ>>Zy~_9O4`PqbQLbqoy_40bd*!L#Vo5AI z!sBFo8<&AoF|zg8&OyxB=Hn5I^_Q}ipt$*{g>5vK6N`?5E9^UO}?a@rxrn=M#Uz4f4wzxPe{*IR(aMR6;SZ=N?q5pk3h(|3Wg zR$=>#o9@W8=&7eUeojK7q}GNyBWhWpnS*Zru<3l1;~@3dyR%ydpsyGT3Bh^!`hH188uG^f_**!JE!tho(OD-EHpb$I!c_o{HQ>Q>rqC~^N>;_ED z_L1+^G`*23d86rHGpUd~2AtXYU~DCZHbpDWEKtWf<$v^?$(H=>JiG#WykhSUF?0Y1 z73tm*+w(holT^F4{9y}%b~q*BH`$jr(Hm&C+na+oTRQnfQ8l2(J*z5XDc+(2t-ki} zsm=hWO5wF8m@{u~;c5uOuH1Ln4D4%dr2rFnoizyP)eMkkJ_ZHtc;~eW{p25wNMBNH z=4*cE*6UqPd4*THv;gXNpnT4h6UnXH+mWtR9fFj0#oLV~l;Bkg{4MOvl%9ftJi-OK z8Ko@M`_v_)aWRSv6h#dBVhT|kThb~*wrU+rruev8Ku`G!@p*lQ+xFJe?7Y0_!6($9 z#liBsQZ9;y7e2n>k$-MuMK10fI=b&<*_ZtM)R1nlw%!;Sk|Acqd8RrWA><&Fy9*C# z%CxE;7(3FFcsxfx`^Mg8-GDp<@#yXZw)Oo{!D`|nMA}hcM{2YgsnaVPrKDx}U*Sl1m&H*~^M0O> zaFU|-LyVm%VM9~pZg?h3jwIN8<1L#3NS3N1)97thJ+V>w2;yEg2$9*JAB#t?q-Es; z5E4v`rT!{KxnM{h>CW@vXok*#W0^aJf~LBbgsQ=3qng%U_-|kC?pescjz`Ckf6fTA zLZMbkNc`mp`BkSV*4?k*S{ysK7uS(>5$o7h%lSHm-jo3;6T26V z_H-}ie@7R{Y$@-O%hp6KE7lU{BIfE7;qa-3j~C}S+*XDvDjwrw0U||<6~$lCRL_e4 z;v+V(vF;t6IR5%y8CAa+lM$a3i=f0CNJewKj zyk|Q+Ob=4^=R#}i>b|~vcS7#g3M?)pL~U+f_UexxKUSNBd5itXOk^4nCR{#3A=VAE`!nDpSIY{r|?dcQ7;$x8?Yh4{E%^!0PSIWa%BG?7`= zv+A|D?(zW_qT#UkyfM0$)7Wl*O9+x-)Kvq!cd)LPst(*3yqQo~{P33>#nudIO-XMA-6O2y-ru~_BLU{MDuBW*LN^;eL=jjAe3J^>=x!NCSZva~~ zj&KzIxwm%XE0wLSfmq-Scrw90Id%`+%Ts1u>}>U{tgcN>(Q)$dL;^8@EXid$?~Q(? z7L#8A?Oru&1+tC9^tWc@L$&Uui<+fr4PNiCRf4lR?1UE5KWVnH9P+G;u`@hCWi7V? z#Reexer;nzB2B4)w+l_W`cy#T%hD`sqj$a}{}&wjNKdH7e(a-|&7|nztFFXNhdtHn z9vrT*UAI*2L)kLWOFI9rba3)(#7@}lPxp(!X$0Tiieyi8hC1?Lp;^xbQH)=M-!#aJ z=??#$7hGWQWtLQwXMAim1yklAHYZ_`sa}tA)`@jq-VO&>-Q{ZYk6A0rdp|tNH&h7@ z+2EDPby(KkkBY>@4?)&SIl!N@M2fk(R4kZ125*{yC+sZvHELCJsG+xJ+js2)}@X2 z7(wlP?ppOK?0S;`UaFz$WclXLv=mqDAZh(@gjX(n>aG$c$XnER0~bHePB)k_^;Reg zcgxlyBXdQQQa%yhp>0SE{^!APcvd3;dD#`U?`*|N6Zr0Zwf#}F$ z8`$;GhoINqg=Y|6%dT9H%QJtp^wxw)d`?GM3&1*Om%rjXi*V`4BBi=RaVfuP;oa zb{Y& z#|HIR)$FqKcdG^n8-UU}iMOmVwufRG>)FU=v2>zWm^RvX6T@hd( zYFyRvGZSGQ;uk$pty5a4rR`&!;_){jIey1ONni=*gSX5jhW;1pvs*9GrU4s;o^Xh$Tay#q$GA{BZNs4qQ2<6s_Rmjv`s5rM!bX4wQigOiPhbga z20siiXV(}}Yj#Udt{T>8Au)lXCg-!PU@<5Bh*^55IooV=GQam6Hy!~C$qQ8uWs26| zB+La4SiyaI2ie;@VpR!A3AgjzGKrq5nH8(;LUt=XE#t(1!5jz$i<0_$8h?6gejqR@ zAhpY(sD%s@a(Iw`6y;wiTywqbR@t>g6IcF#GV+bJg4#%&pFWn ztHZv1^tBo-sti{$D0yRP{u1jO2wWkBAE*!37SP#1(YjL0V{b()%Vx%P^ z9#h#jD!1XCqk*O+-7LN3?MyK^>{!&ssWuK1y<}c0(_iNe+^wCN1npbK@CBAol|`*# z&Mb9Rl3RAoGEXSa!beJpPF$if^y4@C4fcoblcCBTOt$7~(ngf2mT5;$WKy&e4{vPE zhf1FXAYlv54euAeA3SZ)c&16;>)Wx{3`w$9;&I(_qZ`hLb)rrIqDcMpridAlR9=&A@fw`?W_wr%&8lZ*-E>jPlcr zZ5WMe8;G#G*=yrZ9=mNp4AAEXA z;k=X5wqnNtf;Db3+5ZW=Z zvb{crs4kHx(v!lS_+FyQD@+R-o2s-;@L@y&V({YK$@Gzzp}K;K{Wj+n3k?>26~;Ns z?`%c3Mj+W?+u2zC3itGdym0Y%cvvL@&2zV1(AJ5^3ln9(y;HZ#t5L30(Q0HrOc3cw zjYHHhds5zBc5}AXI*E8w_DSxjGmE%fQgxg-1}2N&p@mvY-8k3l{b}d1_lk=3 zDiN|x=^*+k~GnHETn(go& z*2Vp}#i0^- zIqO50Mb8z}^Q;D>GM4noV&0i@xVj;vgn8D0OqpLmfcofGWOmywORAf2@n^!?C1oZQ zbPS)ueX_VryQZ9rUm-kMBeflq92@S&wR$3)S0M>#=05k2g(TN$MESim{SQ$`4 zg=+-_i}kE(2ThVMEooM?ZG>1?Ko+ai?D6Wyqbq7VKDfvEpUmz@-%o-jYL0AFn!X_p zTWnk$e57en&`9RKxi}OGTU`DiPms^H`V8a-H@ey8b+1fHSc#Ja+bqC!v)!#9I#y)- zZd4$$`bqcH@f7%9=Tw6;g2h*NTHk>a6t#Msn@j&DaVVd|&x{9@w4p7D%&vXT^mN9I z7R8+ov(C<1J_L}jF~RaFK})e|Xk2!DXj7_-t;j&hMDFgm>=M>rx?wMB;hIrnBSGCEL*+8ddT<`s|TKtf>X?fuohN8{r z1!|*uHn^4@8n$bd5?qV+dlt4jfO=NkpRFdfD32Jyvqd`RY_U`NG|unpKJFHDM>UFe zsxY@haVlYLBYb;DnKlTLg+T0XNKL3YdCT&QhZUl7v70}usk|+FT4By3?dEykN<{As z%jkU~Mw>_Gy5%Ic;GFLDAP&q9)Irf3h|!5AE>gGSAZ{%R#P~CdDUE*z@f`KU42fc% zc$uZUb;>86YIwTvv&d(U;=4V5Q)6SFFIcNhe?E&h(R`C9u8cqb*<93NtTs^#<<0K1 zYx#%t!NC<)|=YCda~&FT<&4&VA5<4eaC6tTqJtAck;>qx}#Cy)@^U z5h;p#u0OMUVR@Wz$^3Wt2e{FDEG~c|3PvkBx{-KZl399bNOxv_uYe}AML2U@&`EvFYXAB_C>2oBMQi7G2s zZ_NZM91QiJBd)0IldNAuD(d|FzAEZ5FVjY;A1U+g$=oY~>EMbgO$cA_Df}moUa|c; z$GeVnHJ~sm?Vi8gs(AfDbUi^SVIyxM7+k^pa$kG>B1*XcAJHunhfT70Z-T_!nyX>4 z0PTHeAKeO=+@RePfg@!Ab_L1(-sAys_g^xx@)+<5m|a@ff8W$UBh}cebUXqUdVP@X z`R4%L=RvFAUB2_0_mw(?S@^BUArLE#yk0~6$5{DX>%)PjcBLq2YiDm z0YcD<=X!c1SC@aVHU+aTS!^S&eo`>0^h8+r%=y|jMhXE?Dr>T!GC(LQDr&W&w^|)& z2le{Izm!sq+0PuCx<>hqF_^6|;TC*+@UhKiLOI^C1&!m_th)3BAtrr4eHkgbN1-KF z8Ef9;iQVHDz_xxw*`-BeUX@4SRUnWIzsZZp57ebf^9_P8wk1{T4TsRsb5qjsIAoUIli$}RomV}@qjPYH>5+&KzpcZH}5b4HaFgtk7qBpqaP z{R)WEj&Fzv@Cfsp(}W3#R=Ktx@%-O}*Qt)XASvlcx53j+^~@HY{k~Hin6JHCfVPH3 z*%O7~mT9vHW3!>BO+&9X3kzMY!|<356zl_NXPG*k8O6|9>;a9a_N&J|Z3DGk& zgi2{8d~w}z@#!*eJ^XT^aMv8Yj)O^KF$bqSb>>QrcNzMPP0d#kn`W zd)2btp8}HA2vFs*^sJe$dm#uc_N0T;VCi+^xYuQ=;OT(nV4?KH+!452)PxuaUghCNjl%NAyX<4SNnWC6yF%a0!^i35LBWEWV-jD z)AZ|;)mMNB(05caXwYv;B>^tlxJZ(Dn%ag&ZxT<3-kIzxN}HBTm3%C5e}^mNDA}Gg ztq-)Pn8HTQDHIXz^VeNrQe2%k#;7e-k?J%3fD)K4G9S+;zbjuUYYc1rBP$d2?=iR& z?E_qiRm;3U{eb0X#Q`pEcoN$X@4{=t;Q7qVj4tEaZj(=CF4B{zZE5GeE0kbkeSQ6? zwT-p-m8GS$E$+~S-AOlp;4z+EEkO=Hcw*aUfqJVS@u)yrR$`r}cOif>hqA<;bnma3 z9z(4g7g$UUrN{{p{bsBiPkswQ=f_#+*%K16S#xq^KPv_R7Z;{oWM|* zswO;V6|jlaXVvinC}{q zrN*7~o=)ez2?hwbH3HQJg63sO{AbVJ>JY&BSnWb`qV-sjJc&Fkr0b3iDfE>7K5#+&yF z*&%6*lGw`IJ(+LcONC?*b$W5Qda2ajYD(1V#rED_dwK_dru1FRoVLry>FKkXbYj&w zD8SiCyDt{@al)0nYA5hvWWEad4AH30Ez^OK1cV0#dVRvn$QFQIM<%*AFQkU;v~(Y; zT*tv53G40ry22!tD*X|AbK>gE+^iaU^hH=wnO1C=(Jqc*JP$K`6Y?Uafv5{J4*8Kb zT^@mJ8LXxn17)H|u|yh3PjNC%7V^Z!cTf&8gQY@nz1a)S@J8qu03q? z(e_vBIop(Wlf2byGBa`V9VVE7V4j;_MC&a~6%qo=#%aAU+FxW`dCe2En`O=Nn?vF{ z3uzqj5n0F@?PT92Q@OAQ;q?1KE+D@aSLipzl{K&l26D5YC|@FtbwUY{PxlGF-8Gkm zocgL1HGwl3X8N2_b~J^BXMXqDGZvlP)L@-KI@lXS$u6(IeSInr0;(51B|~J6Z}~wo zV_)U#aE87_VCK$REXc&OKQwF_Qu%C$)ek-f9UPOxljHgpSS#44T;{JAb45Sh8 z0Rz!L%-EIIyNRupRe+qX1Vp9at7!q~;?gM))Jh2G>d#Y8Z_kx!+k{3LJHD@~l4WW&mZ zgS=|)1+dvXcB&r`D#Xgs*2G)CZ+5&wqhA1iwbs2I@Yn zynAx7(HLr_C_K=V*mRPGMP}Gu0ZI?>>}#-98?K~Q{59{eSF+0aYMEY@O0>D$B)O5p z)9OPQ+8`#e)jeOd+q9vIw`r(FjcldjV&|m!i&C&Qqv(}SD$i;;(m9}ddCH{*H4_Vh zd>^WEGo`%NVcrDMW(5#WP5-ZHp{wUPdKP=DsY^&%1wsGrZ3Z(`j%omHVR+g_ z&&37YxO;6C*fTOIP%LG41>*pGmtZ&I>FVxL==<*};ZXI(N19H5t z%UkjV?{MSVk&$^RO!FYaNw=y;sy?F7W zPSshcT;r1a4cIZUn2zMcA7_VuL0;9k}W?VliFp4~glIuNh6-1tA#W847s7_4f-JM5|DmHgz+sL zkYWCL`8m*YW1u7}d4n_4`_Q4o%<6Y;865tPYH5QiK-%R{!2PEqz;a$~68(H=DB}v1 zmi%-4x4KP3d0=d=SoC-q4WQ}j(4z6uuc5YB?0I-`bGwPN5`}ZvbQDO&^$mQqd=da zoV*~QTg_{d+hlZy_k%wLWLktG=cZYiW8HoK`M7}8^YMrTG}E7F^apOmzdl5_F5hnQ zAJy$cfbwL)pa1y>>;O$rl@(Yej1JP>e{TFK<-ov1r+@!C_$|nLkNtf3z2SeacrIm{ z>z_0Je$$2zz+xGwSm%BheQXqM|uvi0FgxN$HFx3leTlAyx-_tj>6_ Date: Thu, 9 Jan 2025 15:21:34 +0100 Subject: [PATCH 03/12] Update model and dataset paths --- recipes/training/README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/recipes/training/README.md b/recipes/training/README.md index c0249531..610794b5 100644 --- a/recipes/training/README.md +++ b/recipes/training/README.md @@ -14,7 +14,7 @@ source .venv/bin/activate uv pip install . ``` -And you can navigate to the folders under `/training`. Two folders can be found containing a fine tune of [Qwen/Qwen2.5-Math-1.5B-Instruct](...) on [plaguss/prm800k-trl-dedup](...), and [Qwen/Qwen2.5-Math-7B-Instruct](...). The trainings were run in a slurm cluster with 8xH100, but they can be adapted to the number of available GPUs and resources: +And you can navigate to the folders under `/training`. Two folders can be found containing a fine tune of [Qwen/Qwen2.5-Math-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-1.5B-Instruct) on [HuggingFaceH4/prm800k-trl-dedup](https://huggingface.co/datasets/HuggingFaceH4/prm800k-trl-dedup), and [Qwen/Qwen2.5-Math-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-7B-Instruct). The trainings were run in a slurm cluster with 8xH100, but they can be adapted to the number of available GPUs and resources: | Model | Training Script | | :--- | :--- | @@ -34,9 +34,9 @@ The figure contains the weights and biases loss curves for the previous models: The following two models were fine tuned using the scripts, examples of use can be found in the corresponding repository: -- [plaguss/Qwen2.5-Math-7B-Instruct-PRM-0.2](https://huggingface.co/plaguss/Qwen2.5-Math-7B-Instruct-PRM-0.2) +- [HuggingFaceH4/Qwen2.5-Math-7B-Instruct-PRM-0.2](https://huggingface.co/HuggingFaceH4/Qwen2.5-Math-7B-Instruct-PRM-0.2) -- [plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2](https://huggingface.co/plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2) +- [HuggingFaceH4/Qwen2.5-Math-1.5B-Instruct-PRM-0.2](https://huggingface.co/HuggingFaceH4/Qwen2.5-Math-1.5B-Instruct-PRM-0.2) ## Benchmarking with ProcessBench @@ -55,7 +55,7 @@ All the experiments were run in 1xH100, the batch size should be adjusted to you cd code/ python run_eval_prm_trl.py \ - --model_name "plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ # Model to evaluate + --model_name "HuggingFaceH4/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ # Model to evaluate --output_dir "./outputs" \ # Directory to save the results --batch_size 256 \ # Batch size --sep "\n\n" # Separator, MUST be the same used during training @@ -64,12 +64,12 @@ python run_eval_prm_trl.py \ Click the following buttons to see the example runs and results for the models:

-plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2 +HuggingFaceH4/Qwen2.5-Math-1.5B-Instruct-PRM-0.2 ```bash python run_eval_prm_trl.py \ --config "all" \ - --model_name "plaguss/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ + --model_name "HuggingFaceH4/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ --output_dir "./outputs" \ --batch_size 256 \ --sep "\n\n" @@ -93,12 +93,11 @@ Weighted -> Precision: 30.09 Recall: 63.81 F1 Score: 40.38
-plaguss/Qwen2.5-Math-7B-Instruct-PRM-0.2 +HuggingFaceH4/Qwen2.5-Math-7B-Instruct-PRM-0.2 ```bash python run_eval_prm_trl.py \ - --config "all" \ - --model_name "plaguss/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ + --model_name "HuggingFaceH4/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ --output_dir "./outputs" \ --batch_size 128 \ --sep "\n\n" From 9e61c0fb8f98f0d3e2b1d5e4bdec1c5740226b5f Mon Sep 17 00:00:00 2001 From: plaguss Date: Thu, 16 Jan 2025 14:21:56 +0100 Subject: [PATCH 04/12] Add recipes to run with trl models --- .../Llama-3.2-1B-Instruct/best_of_n_trl_prm.yaml | 13 +++++++++++++ recipes/Llama-3.2-1B-Instruct/dvts_trl.yaml | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 recipes/Llama-3.2-1B-Instruct/best_of_n_trl_prm.yaml create mode 100644 recipes/Llama-3.2-1B-Instruct/dvts_trl.yaml diff --git a/recipes/Llama-3.2-1B-Instruct/best_of_n_trl_prm.yaml b/recipes/Llama-3.2-1B-Instruct/best_of_n_trl_prm.yaml new file mode 100644 index 00000000..ac25c798 --- /dev/null +++ b/recipes/Llama-3.2-1B-Instruct/best_of_n_trl_prm.yaml @@ -0,0 +1,13 @@ +# refer to src/sal/config.py for more options + +approach: best_of_n +n: 32 +search_batch_size: 25 +sort_completed: true +filter_duplicates: true +# num_samples: 5 # REMOVE THIS LINE TO RUN ON THE WHOLE DATASET +seed: 0 +system_prompt: "Solve the following math problem efficiently and clearly:\n\n- For simple problems (2 steps or fewer):\nProvide a concise solution with minimal explanation.\n\n- For complex problems (3 steps or more):\nUse this step-by-step format:\n\n[Concise description]\n[Brief explanation and calculations]\n\n[Concise description]\n[Brief explanation and calculations]\n\n...\n\nRegardless of the approach, always conclude with:\n\nTherefore, the final answer is: $\\boxed{answer}$. I hope it is correct.\n\nWhere [answer] is just the final number or expression that solves the problem." +prm_path: "HuggingFaceH4/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" +overwrite_hub_revision: true # While testing the new PRM model +prm_batch_size: 32 \ No newline at end of file diff --git a/recipes/Llama-3.2-1B-Instruct/dvts_trl.yaml b/recipes/Llama-3.2-1B-Instruct/dvts_trl.yaml new file mode 100644 index 00000000..b17312c2 --- /dev/null +++ b/recipes/Llama-3.2-1B-Instruct/dvts_trl.yaml @@ -0,0 +1,13 @@ +# refer to src/sal/config.py for more options + +approach: dvts +n: 32 +search_batch_size: 25 +sort_completed: true +filter_duplicates: true +# num_samples: 10 # REMOVE THIS LINE TO RUN ON THE WHOLE DATASET +seed: 0 +system_prompt: "Solve the following math problem efficiently and clearly:\n\n- For simple problems (2 steps or fewer):\nProvide a concise solution with minimal explanation.\n\n- For complex problems (3 steps or more):\nUse this step-by-step format:\n\n[Concise description]\n[Brief explanation and calculations]\n\n[Concise description]\n[Brief explanation and calculations]\n\n...\n\nRegardless of the approach, always conclude with:\n\nTherefore, the final answer is: $\\boxed{answer}$. I hope it is correct.\n\nWhere [answer] is just the final number or expression that solves the problem." +prm_path: "HuggingFaceH4/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" +overwrite_hub_revision: true # While testing the new PRM model +prm_batch_size: 32 \ No newline at end of file From 08ac9436f1eea1c769234ebd02f2c4d87bbbc185 Mon Sep 17 00:00:00 2001 From: plaguss Date: Thu, 16 Jan 2025 14:22:30 +0100 Subject: [PATCH 05/12] New module with utilities for the token classification pipeline --- src/sal/models/utils.py | 89 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/sal/models/utils.py diff --git a/src/sal/models/utils.py b/src/sal/models/utils.py new file mode 100644 index 00000000..6818deb1 --- /dev/null +++ b/src/sal/models/utils.py @@ -0,0 +1,89 @@ +from dataclasses import dataclass +from functools import cached_property + +@dataclass +class Example: + problem: str + steps: list[str] + sep: str = "\n" + + @cached_property + def get_texts(self): + """Returns the lists with each problem and solution steps concatenated + with the separator. + """ + return [ + self.sep.join((self.problem, *self.steps[:i])) + self.sep + for i, step in enumerate(self.steps, start=1) + ] + + +class BatchProcessor: + """Helper class to allow passing batches to the model pipeline including different + problem and solutions steps. It allows assigning back the steps of the errors at the + end by finding the corresponding index of the problems in the batches. + """ + def __init__(self, data: list[Example], batch_size: int = 32): + self.data = data + self.batch_size = batch_size + self.current_idx = 0 + + # Create index mapping for steps + self.step_mapping = [] # [(dataset_idx, step_idx), ...] + for idx, item in enumerate(data): + for step_idx in range(len(item.steps)): + self.step_mapping.append((idx, step_idx)) + + self.total_steps = len(self.step_mapping) + + def __iter__(self): + self.current_idx = 0 + return self + + def __next__(self): + if self.current_idx >= self.total_steps: + raise StopIteration + + batch_indices = [] + batch_steps = [] + step_count = 0 + + while self.current_idx < self.total_steps and step_count < self.batch_size: + dataset_idx, step_idx = self.step_mapping[self.current_idx] + batch_indices.append((dataset_idx, step_idx)) + + # Here the steps have to be already generated + steps = self.data[dataset_idx].get_texts + batch_steps.append(steps[step_idx]) + + step_count += 1 + self.current_idx += 1 + + return batch_steps, batch_indices + + def get_total_batches(self): + """Return the total number of batches.""" + return (self.total_steps + self.batch_size - 1) // self.batch_size + + +def process_results( + results: list[dict[str, bool | str | int]], + batch_indices: list[tuple[int, int]], + processed_data: dict[int, list[dict[str, str | float | int]]] +) -> None: + """ + Assign results back to the original dataset structure. + + Args: + results: List of results from processing the batch, + the outputs from transformers.pipeline(X). + batch_indices: List of (dataset_idx, step_idx) tuples. + processed_data: Dictionary to store results, keyed by dataset index. + """ + for result, (dataset_idx, step_idx) in zip(results, batch_indices): + if dataset_idx not in processed_data: + processed_data[dataset_idx] = [] + # Ensure the list is long enough to insert at step_idx + while len(processed_data[dataset_idx]) <= step_idx: + processed_data[dataset_idx].append(None) + processed_data[dataset_idx][step_idx] = result From ae2a21294dc07d5873a60d710ec234fca4fdd285 Mon Sep 17 00:00:00 2001 From: plaguss Date: Thu, 16 Jan 2025 14:22:51 +0100 Subject: [PATCH 06/12] Add prm from trl --- src/sal/config.py | 3 ++ src/sal/models/reward_models.py | 78 ++++++++++++++++++++++++++++++++- src/sal/utils/parser.py | 17 +++++-- 3 files changed, 94 insertions(+), 4 deletions(-) diff --git a/src/sal/config.py b/src/sal/config.py index e09cc584..bcd6b082 100644 --- a/src/sal/config.py +++ b/src/sal/config.py @@ -68,6 +68,9 @@ class Config: filter_duplicates: bool = False sort_completed: bool = False + # PRM related options + separator: str = "\n\n" + def __post_init__(self): if self.approach == "dvts": if self.n % self.beam_width != 0: diff --git a/src/sal/models/reward_models.py b/src/sal/models/reward_models.py index ad11b89f..477a801b 100644 --- a/src/sal/models/reward_models.py +++ b/src/sal/models/reward_models.py @@ -13,20 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. + from itertools import accumulate +from tqdm import tqdm import torch from transformers import ( AutoModelForCausalLM, AutoTokenizer, PreTrainedModel, PreTrainedTokenizer, + pipeline, + Pipeline, ) + from sal.config import Config +from sal.models.utils import BatchProcessor, process_results, Example + CANDIDATE_TOKENS = [648, 387] STEP_TAG_ID = 12902 +LABEL_MAP = {"LABEL_0": False, "LABEL_1": True} +LABEL_FOR_TRUE = "LABEL_1" def batched_math_shepherd_inference( @@ -66,7 +75,12 @@ def batched_math_shepherd_inference( class PRM: def __init__(self, search_config: Config, **model_kwargs): self.search_config = search_config - self.model, self.tokenizer = self.load_model_and_tokenizer(**model_kwargs) + + if getattr(self, "load_pipeline", None): + # Allow loading a transformers pipeline if available (for TRL based models) + self.pipeline = self.load_pipeline(**model_kwargs) + else: + self.model, self.tokenizer = self.load_model_and_tokenizer(**model_kwargs) def load_model_and_tokenizer( self, **model_kwargs @@ -271,6 +285,64 @@ def _score_batched( return reshaped_output_scores +class TRLPRM(PRM): + def load_pipeline(self, **model_kwargs) -> Pipeline: + return pipeline( + "token-classification", + model=self.search_config.prm_path, + device_map="auto", + **model_kwargs + ) + + def score( + self, questions: list[str], outputs: list[list[str]] + ) -> list[list[float]]: + inputs_for_prm = [ + Example( + problem=question, + steps=answers, + sep=self.search_config.separator + ) + for question, answers in zip(questions, outputs) + ] + batch_processor = BatchProcessor(inputs_for_prm, self.search_config.prm_batch_size) + processed_data = {} + for batch_steps, batch_indices in tqdm( + batch_processor, + total=batch_processor.get_total_batches(), + desc="PRM Inference...", + ): + batched_outputs = self.pipeline(batch_steps) + # Assign results back to original structure + process_results(batched_outputs, batch_indices, processed_data) + # Clear GPU memory + torch.cuda.empty_cache() + + def get_scores(processed_data): + scores = [] + for _, steps in processed_data.items(): + step_scores = [] + for output in steps: + step_result = output[-1] # The last token is the one we want the score from + prob = ( + float(1 - step_result["score"]) + if step_result["entity"] == LABEL_FOR_TRUE + else float(step_result["score"]) + ) + # Add the value as a list to emulate the behaviour from the other models, which return + # logits for every step. In this case we only have the probability for the relevant + # token, so we do as if we had the value for all the steps (so we only take into account + # the last) + step_scores.append([prob]) + scores.append(step_scores) + + return scores + # Obtain the outputs scores + output_scores = get_scores(processed_data) + + return output_scores + + def load_prm(config: Config) -> PRM: if config.prm_path == "peiyi9979/math-shepherd-mistral-7b-prm": return MathShepherd(config) @@ -278,4 +350,8 @@ def load_prm(config: Config) -> PRM: if config.prm_path == "RLHFlow/Llama3.1-8B-PRM-Deepseek-Data": return RLHFFlow(config) + if config.prm_path.startswith("HuggingFaceH4"): + # Assume the models under HuggingFaceH4/ org are all trained using TRL. + return TRLPRM(config) + raise NotImplementedError(f"PRM {config.prm_path} not implemented") diff --git a/src/sal/utils/parser.py b/src/sal/utils/parser.py index fa19fa02..50f0f362 100644 --- a/src/sal/utils/parser.py +++ b/src/sal/utils/parser.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + import dataclasses import os import sys @@ -43,9 +44,19 @@ def parse_yaml_and_args( outputs = [] # strip other args list into dict of key-value pairs - other_args = { - arg.split("=")[0].strip("-"): arg.split("=")[1] for arg in other_args - } + + other_args_parsed = {} + for arg in other_args: + values = arg.split("=") + if len(values) == 1: + arg = values[0] # a boolean value, like --push-to-hub + value = "True" + else: + arg, value = values + other_args_parsed[arg.strip("-")] = value + other_args = other_args_parsed + del other_args_parsed + used_args = {} # overwrite the default/loaded value with the value provided to the command line From 076e8ac061570c582f5cbd307402188ee3a0cab3 Mon Sep 17 00:00:00 2001 From: plaguss Date: Mon, 20 Jan 2025 10:54:34 +0100 Subject: [PATCH 07/12] Remove file and command duplicates --- .../prm_qwen_math_1p5b_instruct.slurm | 27 +++---------------- ...rm_qwen_math_1p5b_instruct.sh => train.sh} | 5 +++- .../prm_qwen_math_7b_instruct.slurm | 27 +++---------------- ...{prm_qwen_math_7b_instruct.sh => train.sh} | 3 +++ 4 files changed, 13 insertions(+), 49 deletions(-) rename recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/{prm_qwen_math_1p5b_instruct.sh => train.sh} (94%) rename recipes/training/Qwen2.5-Math-7B-Instruct-PRM/{prm_qwen_math_7b_instruct.sh => train.sh} (97%) diff --git a/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm b/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm index da59050e..a689d8cb 100644 --- a/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm +++ b/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm @@ -13,28 +13,7 @@ set -ex module load cuda/12.1 -source .venv/bin/activate +conda activate sal -srun --nodes=1 --ntasks=1 --export=ALL,ACCELERATE_LOG_LEVEL=info accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml examples/scripts/prm.py \ - --run_name="prm/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ - --model_name_or_path="Qwen/Qwen2.5-Math-1.5B-Instruct" \ - --dataset_name="plaguss/prm800k-trl-dedup" \ - --output_dir="prm/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ - --report_to="wandb" \ - --learning_rate=1.0e-06 \ - --per_device_train_batch_size=16 \ - --per_device_eval_batch_size=8 \ - --do_eval \ - --eval_strategy="steps" \ - --eval_steps=50 \ - --gradient_accumulation_steps=4 \ - --logging_steps=25 \ - --num_train_epochs=1 \ - --max_steps=-1 \ - --warmup_steps=50 \ - --push_to_hub \ - --gradient_checkpointing \ - --max_length=2048 \ - --step_separator="\n\n" \ - --bf16 \ - --dataset_num_proc=8 +# Call the training script with srun +srun --nodes=1 --ntasks=1 --export=ALL,ACCELERATE_LOG_LEVEL=info ./train.sh \ No newline at end of file diff --git a/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.sh b/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/train.sh similarity index 94% rename from recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.sh rename to recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/train.sh index ed34fc53..0bd0b2d0 100644 --- a/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.sh +++ b/recipes/training/Qwen2.5-Math-1.5B-Instruct-PRM/train.sh @@ -1,3 +1,6 @@ +#!/bin/bash +set -ex + accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml examples/scripts/prm.py \ --run_name="prm/Qwen2.5-Math-1.5B-Instruct-PRM-0.2" \ --model_name_or_path="Qwen/Qwen2.5-Math-1.5B-Instruct" \ @@ -20,4 +23,4 @@ accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml --max_length=2048 \ --step_separator="\n\n" \ --bf16 \ - --dataset_num_proc=8 + --dataset_num_proc=8 \ No newline at end of file diff --git a/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm b/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm index 5ee5d7d0..9bd79955 100644 --- a/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm +++ b/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm @@ -13,28 +13,7 @@ set -ex module load cuda/12.1 -source .venv/bin/activate +conda activate sal -srun --nodes=1 --ntasks=1 --export=ALL,ACCELERATE_LOG_LEVEL=info accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml examples/scripts/prm.py \ - --run_name="prm/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ - --model_name_or_path="Qwen/Qwen2.5-Math-7B-Instruct" \ - --dataset_name="plaguss/prm800k-trl-dedup" \ - --output_dir="prm/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ - --report_to="wandb" \ - --learning_rate=1.0e-06 \ - --per_device_train_batch_size=8 \ - --per_device_eval_batch_size=4 \ - --do_eval \ - --eval_strategy="steps" \ - --eval_steps=50 \ - --gradient_accumulation_steps=4 \ - --logging_steps=25 \ - --num_train_epochs=1 \ - --max_steps=-1 \ - --warmup_steps=50 \ - --push_to_hub \ - --gradient_checkpointing \ - --max_length=2048 \ - --step_separator="\n\n" \ - --bf16 \ - --dataset_num_proc=8 +# Call the training script with srun +srun --nodes=1 --ntasks=1 --export=ALL,ACCELERATE_LOG_LEVEL=info ./train.sh \ No newline at end of file diff --git a/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.sh b/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/train.sh similarity index 97% rename from recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.sh rename to recipes/training/Qwen2.5-Math-7B-Instruct-PRM/train.sh index 25a93746..9bf8cbd3 100644 --- a/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/prm_qwen_math_7b_instruct.sh +++ b/recipes/training/Qwen2.5-Math-7B-Instruct-PRM/train.sh @@ -1,3 +1,6 @@ +#!/bin/bash +set -ex + accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml examples/scripts/prm.py \ --run_name="prm/Qwen2.5-Math-7B-Instruct-PRM-0.2" \ --model_name_or_path="Qwen/Qwen2.5-Math-7B-Instruct" \ From 37224738d11926191d9c5b6ec2303fcc9f1d6efb Mon Sep 17 00:00:00 2001 From: plaguss Date: Mon, 20 Jan 2025 10:55:09 +0100 Subject: [PATCH 08/12] Add trl as an extra to install using pip --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 40a2f1c1..7dfe736d 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ extras["quality"] = ["ruff", "isort"] extras["tests"] = ["pytest"] extras["dev"] = ["vllm==0.6.3"] + extras["quality"] + extras["tests"] - +extras["trl"] = "trl @ git+https://github.com/huggingface/trl.git" install_requires = [ "accelerate", From 692aaed7c847b4de7a3ca0fcae0968569c46e7a2 Mon Sep 17 00:00:00 2001 From: plaguss Date: Mon, 20 Jan 2025 10:57:58 +0100 Subject: [PATCH 09/12] Update README with requested changes --- recipes/training/README.md | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/recipes/training/README.md b/recipes/training/README.md index 610794b5..1d81db73 100644 --- a/recipes/training/README.md +++ b/recipes/training/README.md @@ -4,22 +4,18 @@ In TRL v0.13.0, the [PRM trainer](https://huggingface.co/docs/trl/v0.13.0/en/prm ## Fine-tuning with TRL -Using `uv` (`pip` or any other package installer should work similar), clone the [TRL](https://github.com/huggingface/trl) repository, create a virtual environment and install the dependencies: +First install [TRL](https://github.com/huggingface/trl) by cloning the repo and installing from the main branch, or simply running the following command: -```bash -git clone https://github.com/huggingface/trl.git -cd trl -uv venv .venv --python 3.12 -source .venv/bin/activate -uv pip install . +```shell +pip install -e '.[trl]' ``` And you can navigate to the folders under `/training`. Two folders can be found containing a fine tune of [Qwen/Qwen2.5-Math-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-1.5B-Instruct) on [HuggingFaceH4/prm800k-trl-dedup](https://huggingface.co/datasets/HuggingFaceH4/prm800k-trl-dedup), and [Qwen/Qwen2.5-Math-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-7B-Instruct). The trainings were run in a slurm cluster with 8xH100, but they can be adapted to the number of available GPUs and resources: | Model | Training Script | | :--- | :--- | -| Qwen/Qwen2.5-Math-1.5B-Instruct | [Script](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.sh) / [Slurm](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm)| -| Qwen/Qwen2.5-Math-7B-Instruct | [Script](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_7b_instruct.sh) / [Slurm](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm)| +| Qwen/Qwen2.5-Math-1.5B-Instruct | [Script](Qwen2.5-Math-1.5B-Instruct-PRM/train.sh) / [Slurm](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_1p5b_instruct.slurm)| +| Qwen/Qwen2.5-Math-7B-Instruct | [Script](Qwen2.5-Math-1.5B-Instruct-PRM/train.sh) / [Slurm](Qwen2.5-Math-1.5B-Instruct-PRM/prm_qwen_math_7b_instruct.slurm)|
Click for a sample WandB run. @@ -40,15 +36,31 @@ The following two models were fine tuned using the scripts, examples of use can ## Benchmarking with ProcessBench -Using `uv` (`pip` or any other package installer should work similar), clone the [ProcessBench](https://github.com/huggingface/ProcessBench) fork that includes the script to evaluate TRL models, create a virtual environment, and install the requirements: +Click on any of the instructions to install either using `uv` or `conda+pip`. You will need to clone the [ProcessBench](https://github.com/huggingface/ProcessBench) repository, create a virtual environment, and install the requirements: -```bash +
+Install with uv + +```shell git clone https://github.com/huggingface/ProcessBench.git uv venv .venv --python 3.12 source .venv/bin/activate uv pip install -r requirements-trl.txt ``` +
+ +
+Install with conda + +```shell +git clone https://github.com/huggingface/ProcessBench.git +conda create -n processbench python=3.12 && conda activate processbench +pip install -r requirements-trl.txt +``` + +
+ All the experiments were run in 1xH100, the batch size should be adjusted to your capacity (for reference, a batch size of 256 for Qwen2.5-Math-1.5B-Instruct took near 0.5h, and a batch size of 128 for Qwen2.5-Math-7B-Instruct near 2h). Navigate to the `/code` folder, and run the `run_eval_prm_trl.py` script: ```bash From b62722d926898ddfde12ac4b9069b60eb46012b5 Mon Sep 17 00:00:00 2001 From: plaguss Date: Tue, 18 Mar 2025 12:31:43 +0100 Subject: [PATCH 10/12] Add TRL PRM class --- src/sal/config.py | 3 + src/sal/models/reward_models.py | 87 ++++++++++++++++++++++++++ src/sal/models/utils.py | 104 ++++++++++++++++++++++++++++++++ src/sal/utils/parser.py | 15 +++-- 4 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 src/sal/models/utils.py diff --git a/src/sal/config.py b/src/sal/config.py index e09cc584..bcd6b082 100644 --- a/src/sal/config.py +++ b/src/sal/config.py @@ -68,6 +68,9 @@ class Config: filter_duplicates: bool = False sort_completed: bool = False + # PRM related options + separator: str = "\n\n" + def __post_init__(self): if self.approach == "dvts": if self.n % self.beam_width != 0: diff --git a/src/sal/models/reward_models.py b/src/sal/models/reward_models.py index 7b1e1e55..505541e4 100644 --- a/src/sal/models/reward_models.py +++ b/src/sal/models/reward_models.py @@ -15,9 +15,11 @@ from itertools import accumulate +import tqdm import torch from transformers import ( AutoModelForCausalLM, + AutoModelForTokenClassification, AutoTokenizer, PreTrainedModel, PreTrainedTokenizer, @@ -30,9 +32,12 @@ prepare_input, ) from sal.models.skywork_o1_prm.prm_model import SkyworkPRMModel +from sal.models.utils import BatchProcessor, process_results, Example CANDIDATE_TOKENS = [648, 387] STEP_TAG_ID = 12902 +LABEL_MAP = {"LABEL_0": False, "LABEL_1": True} +LABEL_FOR_TRUE = "LABEL_1" def batched_math_shepherd_inference( @@ -340,6 +345,85 @@ def load_model_and_tokenizer( return SkyworkO1._load_model_and_tokenizer(prm_model_path, **model_kwargs) +class TRLPRM(PRM): + def load_model_and_tokenizer( + self, **model_kwargs + ) -> tuple[PreTrainedModel, PreTrainedTokenizer]: + tokenizer = AutoTokenizer.from_pretrained( + self.search_config.prm_path + ) + tokenizer.padding_side = "left" # To extract the predicted token as the last token of the right + model = AutoModelForTokenClassification.from_pretrained( + self.search_config.prm_path, + device_map="auto", + torch_dtype=torch.float16, + **model_kwargs, + ).eval() + + return model, tokenizer + + def score( + self, questions: list[str], outputs: list[list[str]] + ) -> list[list[float]]: + + inputs_for_prm = [] + for question, output in zip(questions, outputs): + # Split using \n\n here as that's what we asked the model to use to split the steps + for answer in output: + inputs_for_prm.append( + Example( + problem=question, + steps=answer.split("\n\n"), + sep=self.search_config.separator + ) + ) + + batch_processor = BatchProcessor(inputs_for_prm, self.search_config.prm_batch_size) + processed_data = {} + + for batch_steps, batch_indices in tqdm( + batch_processor, + total=batch_processor.get_total_batches(), + desc="PRM Inference...", + ): + with torch.no_grad(): + # batch_steps = ['Let $a,$ $b,$ and $c$ be positive real numbers. Find the set of all possible values of\n\\[\\frac{c}{a} + \\frac{a}{b + c} + \\frac{b}{c}.\\]\n\nThis problem involves finding the range of an expression involving three variables.', 'Let $a,$ $b,$ and $c$ be positive real numbers. Find the set of all possible values of\n\\[\\frac{c}{a} + \\frac{a}{b + c} + \\frac{b}{c}.\\]\n\nThis problem involves finding the range of an expression involving three variables.\n\nOne possible strategy is to try to eliminate some variables and write the expression in terms of one variable only.'] + tokenized_batch = self.tokenizer( + batch_steps, + padding=True, + return_tensors="pt" + ).to(self.model.device) + # Get model outputs + batched_outputs = self.model(**tokenized_batch) + # Transform to probabilities, and extract the ones corresponding + # to the TRUE class (LABEL_1, which is the first class) + scores = batched_outputs.logits.softmax(dim=-1)[:, :, 0] + # The probabilities for the batch can be extracted by finding the prob + # of the last token, which should correspond to the + probs = scores[:, -1].tolist() # To extract them from cuda + # batched_outputs = self.pipeline(batch_steps) + + # Assign results back to original structure + process_results(probs, batch_indices, processed_data) + # process_results(batched_outputs, batch_indices, processed_data) + # Clear GPU memory + del batch_steps, batched_outputs, scores, probs + torch.cuda.empty_cache() + + # The "processed_data" comes sorted as a dict with the index, and the different + # scores. Now we group each group of answers to its N. + reshaped_output_scores = [] + counter = 0 + for _, answers in zip(questions, outputs): + scores = [] + for _ in answers: + scores.append(processed_data[counter]) + counter += 1 + reshaped_output_scores.append(scores) + + return reshaped_output_scores + + def load_prm(config: Config) -> PRM: if config.prm_path == "peiyi9979/math-shepherd-mistral-7b-prm": return MathShepherd(config) @@ -353,4 +437,7 @@ def load_prm(config: Config) -> PRM: if config.prm_path == "Skywork/Skywork-o1-Open-PRM-Qwen-2.5-7B": return SkyworkO1_7B(config) + if config.prm_path.startswith("HuggingFaceH4"): + return TRLPRM(config) + raise NotImplementedError(f"PRM {config.prm_path} not implemented") diff --git a/src/sal/models/utils.py b/src/sal/models/utils.py new file mode 100644 index 00000000..92c9442d --- /dev/null +++ b/src/sal/models/utils.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from functools import cached_property + +@dataclass +class Example: + problem: str + steps: list[str] + sep: str = "\n" + + @cached_property + def get_texts(self): + """Returns the lists with each problem and solution steps concatenated + with the separator. + """ + return [ + self.sep.join((self.problem, *self.steps[:i])) + self.sep + for i, step in enumerate(self.steps, start=1) + ] + + +class BatchProcessor: + """Helper class to allow passing batches to the model pipeline including different + problem and solutions steps. It allows assigning back the steps of the errors at the + end by finding the corresponding index of the problems in the batches. + """ + def __init__(self, data: list[Example], batch_size: int = 32): + self.data = data + self.batch_size = batch_size + self.current_idx = 0 + + # Create index mapping for steps + self.step_mapping = [] # [(dataset_idx, step_idx), ...] + for idx, item in enumerate(data): + for step_idx in range(len(item.steps)): + self.step_mapping.append((idx, step_idx)) + + self.total_steps = len(self.step_mapping) + + def __iter__(self): + self.current_idx = 0 + return self + + def __next__(self): + if self.current_idx >= self.total_steps: + raise StopIteration + + batch_indices = [] + batch_steps = [] + step_count = 0 + + while self.current_idx < self.total_steps and step_count < self.batch_size: + dataset_idx, step_idx = self.step_mapping[self.current_idx] + batch_indices.append((dataset_idx, step_idx)) + + # Here the steps have to be already generated + steps = self.data[dataset_idx].get_texts + batch_steps.append(steps[step_idx]) + + step_count += 1 + self.current_idx += 1 + + return batch_steps, batch_indices + + def get_total_batches(self): + """Return the total number of batches.""" + return (self.total_steps + self.batch_size - 1) // self.batch_size + + +def process_results( + results: list[dict[str, bool | str | int]], + batch_indices: list[tuple[int, int]], + processed_data: dict[int, list[dict[str, str | float | int]]] +) -> None: + """ + Assign results back to the original dataset structure. + + Args: + results: List of results from processing the batch, + the outputs from transformers.pipeline(X). + batch_indices: List of (dataset_idx, step_idx) tuples. + processed_data: Dictionary to store results, keyed by dataset index. + """ + for result, (dataset_idx, step_idx) in zip(results, batch_indices): + if dataset_idx not in processed_data: + processed_data[dataset_idx] = [] + # Ensure the list is long enough to insert at step_idx + while len(processed_data[dataset_idx]) <= step_idx: + processed_data[dataset_idx].append(None) + processed_data[dataset_idx][step_idx] = result diff --git a/src/sal/utils/parser.py b/src/sal/utils/parser.py index fa19fa02..b68bd96c 100644 --- a/src/sal/utils/parser.py +++ b/src/sal/utils/parser.py @@ -43,10 +43,17 @@ def parse_yaml_and_args( outputs = [] # strip other args list into dict of key-value pairs - other_args = { - arg.split("=")[0].strip("-"): arg.split("=")[1] for arg in other_args - } - used_args = {} + other_args_parsed = {} + for arg in other_args: + values = arg.split("=") + if len(values) == 1: + arg = values[0] # a boolean value, like --push-to-hub + value = "True" + else: + arg, value = values + other_args_parsed[arg.strip("-")] = value + other_args = other_args_parsed + del other_args_parsed # overwrite the default/loaded value with the value provided to the command line # adapted from https://github.com/huggingface/transformers/blob/d0b5002378daabf62769159add3e7d66d3f83c3b/src/transformers/hf_argparser.py#L327 From a2bca64ee2bbcc41ca44b19c820ef4f2bba90df8 Mon Sep 17 00:00:00 2001 From: plaguss Date: Tue, 18 Mar 2025 12:47:42 +0100 Subject: [PATCH 11/12] Remove extra newlines --- src/sal/models/reward_models.py | 1 - src/sal/utils/parser.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/sal/models/reward_models.py b/src/sal/models/reward_models.py index 2ca67364..c507a6aa 100644 --- a/src/sal/models/reward_models.py +++ b/src/sal/models/reward_models.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from itertools import accumulate from tqdm import tqdm diff --git a/src/sal/utils/parser.py b/src/sal/utils/parser.py index e45377b3..60df1b2b 100644 --- a/src/sal/utils/parser.py +++ b/src/sal/utils/parser.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import dataclasses import os import sys From f42ecf25ff1bb43c50471000b6ddcc3ec07dc0e6 Mon Sep 17 00:00:00 2001 From: plaguss Date: Mon, 24 Mar 2025 08:30:50 +0100 Subject: [PATCH 12/12] Fix style and lint --- src/sal/models/reward_models.py | 35 +++++++++++++-------------------- src/sal/models/utils.py | 6 ++++-- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/sal/models/reward_models.py b/src/sal/models/reward_models.py index c507a6aa..fbf5a9d3 100644 --- a/src/sal/models/reward_models.py +++ b/src/sal/models/reward_models.py @@ -15,8 +15,8 @@ from itertools import accumulate -from tqdm import tqdm import torch +from tqdm import tqdm from transformers import ( AutoModelForCausalLM, AutoModelForTokenClassification, @@ -25,7 +25,6 @@ PreTrainedTokenizer, ) - from sal.config import Config from sal.models.skywork_o1_prm.io_utils import ( derive_step_rewards, @@ -33,7 +32,7 @@ prepare_input, ) from sal.models.skywork_o1_prm.prm_model import SkyworkPRMModel -from sal.models.utils import BatchProcessor, process_results, Example +from sal.models.utils import BatchProcessor, Example, process_results CANDIDATE_TOKENS = [648, 387] STEP_TAG_ID = 12902 @@ -135,9 +134,9 @@ def score( # stripped_output_scores = [] TODO: strip out the reward for previous steps for output_score, output in zip(output_scores, outputs): - assert len(output_score) == len( - output - ), f"{len(output_score)} != {len(output)}" + assert len(output_score) == len(output), ( + f"{len(output_score)} != {len(output)}" + ) return output_scores @@ -350,10 +349,10 @@ class TRLPRM(PRM): def load_model_and_tokenizer( self, **model_kwargs ) -> tuple[PreTrainedModel, PreTrainedTokenizer]: - tokenizer = AutoTokenizer.from_pretrained( - self.search_config.prm_path + tokenizer = AutoTokenizer.from_pretrained(self.search_config.prm_path) + tokenizer.padding_side = ( + "left" # To extract the predicted token as the last token of the right ) - tokenizer.padding_side = "left" # To extract the predicted token as the last token of the right model = AutoModelForTokenClassification.from_pretrained( self.search_config.prm_path, device_map="auto", @@ -366,16 +365,13 @@ def load_model_and_tokenizer( def score( self, questions: list[str], outputs: list[list[str]] ) -> list[list[float]]: - inputs_for_prm = [ - Example( - problem=question, - steps=answers, - sep=self.search_config.separator - ) + Example(problem=question, steps=answers, sep=self.search_config.separator) for question, answers in zip(questions, outputs) ] - batch_processor = BatchProcessor(inputs_for_prm, self.search_config.prm_batch_size) + batch_processor = BatchProcessor( + inputs_for_prm, self.search_config.prm_batch_size + ) processed_data = {} for batch_steps, batch_indices in tqdm( @@ -386,9 +382,7 @@ def score( with torch.no_grad(): # batch_steps = ['Let $a,$ $b,$ and $c$ be positive real numbers. Find the set of all possible values of\n\\[\\frac{c}{a} + \\frac{a}{b + c} + \\frac{b}{c}.\\]\n\nThis problem involves finding the range of an expression involving three variables.', 'Let $a,$ $b,$ and $c$ be positive real numbers. Find the set of all possible values of\n\\[\\frac{c}{a} + \\frac{a}{b + c} + \\frac{b}{c}.\\]\n\nThis problem involves finding the range of an expression involving three variables.\n\nOne possible strategy is to try to eliminate some variables and write the expression in terms of one variable only.'] tokenized_batch = self.tokenizer( - batch_steps, - padding=True, - return_tensors="pt" + batch_steps, padding=True, return_tensors="pt" ).to(self.model.device) # Get model outputs batched_outputs = self.model(**tokenized_batch) @@ -396,7 +390,7 @@ def score( # to the TRUE class (LABEL_1, which is the first class) scores = batched_outputs.logits.softmax(dim=-1)[:, :, 0] # The probabilities for the batch can be extracted by finding the prob - # of the last token, which should correspond to the + # of the last token, which should correspond to the probs = scores[:, -1].tolist() # To extract them from cuda # batched_outputs = self.pipeline(batch_steps) @@ -421,7 +415,6 @@ def score( return reshaped_output_scores - def load_prm(config: Config) -> PRM: if config.prm_path == "peiyi9979/math-shepherd-mistral-7b-prm": return MathShepherd(config) diff --git a/src/sal/models/utils.py b/src/sal/models/utils.py index 92c9442d..36f55cbe 100644 --- a/src/sal/models/utils.py +++ b/src/sal/models/utils.py @@ -16,6 +16,7 @@ from dataclasses import dataclass from functools import cached_property + @dataclass class Example: problem: str @@ -25,7 +26,7 @@ class Example: @cached_property def get_texts(self): """Returns the lists with each problem and solution steps concatenated - with the separator. + with the separator. """ return [ self.sep.join((self.problem, *self.steps[:i])) + self.sep @@ -38,6 +39,7 @@ class BatchProcessor: problem and solutions steps. It allows assigning back the steps of the errors at the end by finding the corresponding index of the problems in the batches. """ + def __init__(self, data: list[Example], batch_size: int = 32): self.data = data self.batch_size = batch_size @@ -84,7 +86,7 @@ def get_total_batches(self): def process_results( results: list[dict[str, bool | str | int]], batch_indices: list[tuple[int, int]], - processed_data: dict[int, list[dict[str, str | float | int]]] + processed_data: dict[int, list[dict[str, str | float | int]]], ) -> None: """ Assign results back to the original dataset structure.