From fe0cea1ba40d359ff5f8f1624f198fdf96c8d39d Mon Sep 17 00:00:00 2001 From: Jason Gauci Date: Tue, 28 Jul 2020 13:16:41 -0500 Subject: [PATCH] update docs --- _images/reagent_banner.png | Bin 0 -> 3822915 bytes _sources/api/ml.rl.evaluation.rst.txt | 78 + _sources/api/ml.rl.models.rst.txt | 150 + _sources/api/ml.rl.prediction.rst.txt | 30 + _sources/api/ml.rl.readers.rst.txt | 46 + _sources/api/ml.rl.rst.txt | 4 + .../api/ml.rl.training.gradient_free.rst.txt | 30 + _sources/api/ml.rl.training.ranking.rst.txt | 30 + _sources/api/ml.rl.training.rst.txt | 26 +- _sources/api/modules.rst.txt | 6 +- _sources/api/reagent.evaluation.rst.txt | 110 + _sources/api/reagent.gym.agents.rst.txt | 30 + _sources/api/reagent.gym.envs.rst.txt | 54 + _sources/api/reagent.gym.policies.rst.txt | 46 + .../api/reagent.gym.policies.samplers.rst.txt | 38 + .../api/reagent.gym.policies.scorers.rst.txt | 38 + .../api/reagent.gym.preprocessors.rst.txt | 46 + _sources/api/reagent.gym.rst.txt | 42 + _sources/api/reagent.gym.runners.rst.txt | 22 + .../reagent.gym.tests.preprocessors.rst.txt | 30 + _sources/api/reagent.gym.tests.rst.txt | 69 + _sources/api/reagent.models.rst.txt | 166 ++ _sources/api/reagent.ope.datasets.rst.txt | 22 + _sources/api/reagent.ope.estimators.rst.txt | 54 + _sources/api/reagent.ope.rst.txt | 32 + _sources/api/reagent.ope.test.rst.txt | 61 + .../api/reagent.ope.test.unit_tests.rst.txt | 38 + _sources/api/reagent.ope.trainers.rst.txt | 30 + _sources/api/reagent.optimizer.rst.txt | 70 + _sources/api/reagent.prediction.rst.txt | 22 + _sources/api/reagent.preprocessing.rst.txt | 86 + _sources/api/reagent.replay_memory.rst.txt | 46 + _sources/api/reagent.rst.txt | 94 + .../reagent.training.gradient_free.rst.txt | 30 + _sources/api/reagent.training.ranking.rst.txt | 54 + _sources/api/reagent.training.rst.txt | 159 ++ .../api/reagent.training.world_model.rst.txt | 30 + _sources/api/reagent.workflow.rst.txt | 118 + _sources/continuous_integration.rst.txt | 7 + _sources/distributed.rst.txt | 8 +- _sources/index.rst.txt | 29 +- _sources/installation.rst.txt | 241 +- _sources/license.rst.txt | 2 +- _sources/rasp_tutorial.rst.txt | 129 +- _sources/usage.rst.txt | 218 +- api/ml.html | 195 +- api/ml.rl.evaluation.html | 269 ++ api/ml.rl.html | 1853 +------------ api/ml.rl.models.html | 305 ++ api/ml.rl.prediction.html | 245 ++ api/ml.rl.preprocessing.html | 717 +---- api/ml.rl.readers.html | 253 ++ api/ml.rl.simulators.html | 138 +- api/ml.rl.training.gradient_free.html | 274 ++ api/ml.rl.training.html | 1147 +------- api/ml.rl.training.ranking.html | 274 ++ api/ml.rl.training.world_model.html | 116 +- api/ml.rl.workflow.html | 434 +-- api/modules.html | 192 +- api/reagent.evaluation.html | 424 +++ api/reagent.gym.agents.html | 273 ++ api/reagent.gym.envs.html | 348 +++ api/reagent.gym.html | 443 +++ api/reagent.gym.policies.html | 327 +++ api/reagent.gym.policies.samplers.html | 380 +++ api/reagent.gym.policies.scorers.html | 288 ++ api/reagent.gym.preprocessors.html | 279 ++ api/reagent.gym.runners.html | 270 ++ api/reagent.gym.tests.html | 345 +++ api/reagent.gym.tests.preprocessors.html | 275 ++ api/reagent.html | 1315 +++++++++ api/reagent.models.html | 393 +++ api/reagent.ope.datasets.html | 327 +++ api/reagent.ope.estimators.html | 1556 +++++++++++ api/reagent.ope.html | 376 +++ api/reagent.ope.test.html | 802 ++++++ api/reagent.ope.test.unit_tests.html | 321 +++ api/reagent.ope.trainers.html | 507 ++++ api/reagent.optimizer.html | 561 ++++ api/reagent.prediction.html | 268 ++ api/reagent.preprocessing.html | 509 ++++ api/reagent.replay_memory.html | 1093 ++++++++ api/reagent.training.gradient_free.html | 273 ++ api/reagent.training.html | 347 +++ api/reagent.training.ranking.html | 282 ++ api/reagent.training.world_model.html | 273 ++ api/reagent.workflow.html | 474 ++++ continuous_integration.html | 229 ++ distributed.html | 24 +- genindex.html | 2463 ++++++++++------- index.html | 557 +++- installation.html | 180 +- license.html | 20 +- objects.inv | Bin 6232 -> 7714 bytes py-modindex.html | 602 ++-- rasp_tutorial.html | 127 +- search.html | 14 +- searchindex.js | 2 +- usage.html | 211 +- 99 files changed, 20663 insertions(+), 6178 deletions(-) create mode 100644 _images/reagent_banner.png create mode 100644 _sources/api/ml.rl.evaluation.rst.txt create mode 100644 _sources/api/ml.rl.models.rst.txt create mode 100644 _sources/api/ml.rl.prediction.rst.txt create mode 100644 _sources/api/ml.rl.readers.rst.txt create mode 100644 _sources/api/ml.rl.training.gradient_free.rst.txt create mode 100644 _sources/api/ml.rl.training.ranking.rst.txt create mode 100644 _sources/api/reagent.evaluation.rst.txt create mode 100644 _sources/api/reagent.gym.agents.rst.txt create mode 100644 _sources/api/reagent.gym.envs.rst.txt create mode 100644 _sources/api/reagent.gym.policies.rst.txt create mode 100644 _sources/api/reagent.gym.policies.samplers.rst.txt create mode 100644 _sources/api/reagent.gym.policies.scorers.rst.txt create mode 100644 _sources/api/reagent.gym.preprocessors.rst.txt create mode 100644 _sources/api/reagent.gym.rst.txt create mode 100644 _sources/api/reagent.gym.runners.rst.txt create mode 100644 _sources/api/reagent.gym.tests.preprocessors.rst.txt create mode 100644 _sources/api/reagent.gym.tests.rst.txt create mode 100644 _sources/api/reagent.models.rst.txt create mode 100644 _sources/api/reagent.ope.datasets.rst.txt create mode 100644 _sources/api/reagent.ope.estimators.rst.txt create mode 100644 _sources/api/reagent.ope.rst.txt create mode 100644 _sources/api/reagent.ope.test.rst.txt create mode 100644 _sources/api/reagent.ope.test.unit_tests.rst.txt create mode 100644 _sources/api/reagent.ope.trainers.rst.txt create mode 100644 _sources/api/reagent.optimizer.rst.txt create mode 100644 _sources/api/reagent.prediction.rst.txt create mode 100644 _sources/api/reagent.preprocessing.rst.txt create mode 100644 _sources/api/reagent.replay_memory.rst.txt create mode 100644 _sources/api/reagent.rst.txt create mode 100644 _sources/api/reagent.training.gradient_free.rst.txt create mode 100644 _sources/api/reagent.training.ranking.rst.txt create mode 100644 _sources/api/reagent.training.rst.txt create mode 100644 _sources/api/reagent.training.world_model.rst.txt create mode 100644 _sources/api/reagent.workflow.rst.txt create mode 100644 _sources/continuous_integration.rst.txt create mode 100644 api/ml.rl.evaluation.html create mode 100644 api/ml.rl.models.html create mode 100644 api/ml.rl.prediction.html create mode 100644 api/ml.rl.readers.html create mode 100644 api/ml.rl.training.gradient_free.html create mode 100644 api/ml.rl.training.ranking.html create mode 100644 api/reagent.evaluation.html create mode 100644 api/reagent.gym.agents.html create mode 100644 api/reagent.gym.envs.html create mode 100644 api/reagent.gym.html create mode 100644 api/reagent.gym.policies.html create mode 100644 api/reagent.gym.policies.samplers.html create mode 100644 api/reagent.gym.policies.scorers.html create mode 100644 api/reagent.gym.preprocessors.html create mode 100644 api/reagent.gym.runners.html create mode 100644 api/reagent.gym.tests.html create mode 100644 api/reagent.gym.tests.preprocessors.html create mode 100644 api/reagent.html create mode 100644 api/reagent.models.html create mode 100644 api/reagent.ope.datasets.html create mode 100644 api/reagent.ope.estimators.html create mode 100644 api/reagent.ope.html create mode 100644 api/reagent.ope.test.html create mode 100644 api/reagent.ope.test.unit_tests.html create mode 100644 api/reagent.ope.trainers.html create mode 100644 api/reagent.optimizer.html create mode 100644 api/reagent.prediction.html create mode 100644 api/reagent.preprocessing.html create mode 100644 api/reagent.replay_memory.html create mode 100644 api/reagent.training.gradient_free.html create mode 100644 api/reagent.training.html create mode 100644 api/reagent.training.ranking.html create mode 100644 api/reagent.training.world_model.html create mode 100644 api/reagent.workflow.html create mode 100644 continuous_integration.html diff --git a/_images/reagent_banner.png b/_images/reagent_banner.png new file mode 100644 index 0000000000000000000000000000000000000000..6510242eac667626adec403f19709f7ca335bdc1 GIT binary patch literal 3822915 zcmeF)2b@&Zy#U~Y4MY?b6%`9$FZ3n|sDNVcMvSH)O`?DWgC)tXQDe)~7-KXVjWMye zsIfOJv3Fy|-b=6{q9W`&<0|aV7Uu5G%ue~vZ{N$@dr$q&WrhDebM8sI?mTkYwyU=d zAuPMywj*{Aq4kp?EOu0z#p0Q&cXnak7s3%Qowvt?J@@!vl_7Ip{bJY-CszJxhbgGbtC?y~gq+n?Cw?=P;r+ubW{z4a2E9=zbh#ZNu8^7kw6wrRVQ+rH5H z?JgIddh9<=Jo(|dZyf!P1E0Hn&byDzzv20z?_D|Tyz8H6J-to4KX2P-w|@pktuk7>K^!D0C}*N^RT*t~UGg|B`ycHkM zUAu*!Z&2~>GokXlul{m$tFZ5%x~$ylg409i4&zQ75f0cqT=~S0PTo3<>KRrVzr%e4 z!wcO)&+Yaezis%}O=0@u%P(_XXwxqA-0j5UHwsG}9e)1cI_n$}j{8ekX`9dY9{kxR zS11o*A2d9-GN&lvu~SjhqT^%@9t}D`1p1mH(9Cw=*32#vt{>tj{M7Ny^rbJ z{`FaphH$~@E635!ns>zG%U%8WempI_3%Up2yoKwbM*M7?z!UZ3^wD+uoQ#M=w{SUXi_~JWn zzx~4Y_YK>3^uv4qdj5oael~sYc?W&{o5A1Cnf>~mZ?1bx@1Gnqa*3~Azvs~}cD!hf zC;qtfar?jg?r9^xy=j@MZ`XXL>wVkpf6J(iS9)u=6%RRe#5up8)%&8ZCk^}KM*msw z>8i1BzP9Py#fQGuFZ}BlLt7nvdbdk|GG_5Fci3S0QB$YCIVXhAZ<})We>ZHic;x}d zzVXDgs!z6bTRR-L}Md(};+FSW+q zOTK*fGT;8Z|8c|Cz3-l3%MWWiwsMU_f4BLt1pFC=vQ`>a>%kf`zxqI6s zRy<|h&JUfq(r#UMIKAJty>>r&&(3>oyXNOZ4qa^E3Hxnx>*1|7~}}@LMh&u=;V^T)51WW2g2y=d@KWIQPjz|GoN<-}nFfdrw|*K>L?Y8#?@f zp3~o7wRNBS4(i$c{(FaAyV<5opE2fQ3^W{-K|+G$Iyc+A$* zm+rsBv|*h`ueH^-qjns%^9$Qv+Ifw2`>ycP(gT)W{J7oj*`Vtgm+tWI^>1C_>-$#S zzjv<_Ms2(4I?t_mT8AFXAARER`v$DL;z`4MZ*a>W7TfOHfA0D8icfFz^yJ;w`C!w_ z&h5MH>g$~G&`V+keYL{`u7&Z*I8z z!@t;mwRe_(r|&yUjXS&dR(t*9oZVl!^zVDF(C@S#54-B`H|~DTZYNyYq3_}641Da^ zRkqvVqN^@E^O@BT=y<@KiPyY8_0|<$IdiR@&w6t2ryu!X*(>;ED^)`F;+2gItmj3Ubuh{GI^LG7c zk4}3`xNP%FPq=)-rBn9Y;-A0SXWmIm?Q`#r_wF+MpSxXf?a$u2{r=9&j9KmaBR1`F z;Idcz?(k#!-M9X=kG=S-Uv7Nc+4I)C?UW&xU-tKb!~1OCXYcEGpY^BlTeN=b>_fi# z>(rf>es9A|&YJa`lV4rwla1cosME!Lcf6(7xcv^i{Lg3odDIo#Ua{L1BR{&M|J5@u zIq%{-w!UqjxyLNG)dSmau-^tFC!F^9lIOj*$BTQsa!K1uR@rIDpxIY6=Stk8C$;sq5RVyVQ(!kFPrT zKm87wGI-4%56t*%x8Xmzcgx`yjXrbqS#KP7`Ylfl{>QW1{NFb1e?Rg5{oY#ltx5M! zy8j>RFSUM`^>^6m!<{}pXZM|Mxp#=i?i1pM3m}2kvvxsJ(vtlk2y+>b$Ggd+xTY zZo6cKOK#cujGeDI|G5eG{_C?FAG-0fn^ybRJ1;CXo;imsweecI- z+;PUQZ~gWE%^T8s*%Mw|?us>kyykBX`}d(M4f@fbSI>F;uODym>;HW5=MQ>J81=;R zPrT6Qts6T$veA7L&)ehNXGTuwIN_GVj-7SlDJSjl+ef;eHMIM=o$i0-!8_jGe(LN) ze)7gqPtQK@-0RQ1XXd3duKBqC#}hw3;pM+Q+V^i8oxe|?!=AkN!Qrp(_RdpVEc3TB zdmi}OmA~0#_%6pxxZ%BZdW`9D`rcPP|F4@ans)h|(Vui2`sL6=|Mu3M0_tq`$?>>H?N1q*a-l**k+F^}>-8a6v|Kx-2eC^wrvrc&P zyf>G5>*$VW_jvFp2Os(O$M5;#_)Au~tow?a4*K(!JD#}nW`Fv0tK~Pobd}$)_j!kD z>z&&6$a~hjXF}H=PxXK5#21F|xA9s#f4=R;J011M|DAq&za!WEPoIl6edm_rHn{8$ zx9_;@j<1|M==@dA`DN!xy@w3Cqt~&2JpB)c47aovjH;n$~cjt8)y7EaUU%k?dj%Tf0@xq2LAMsA76Q(?T*I!=#)4wmf;=WtPJ~963 z@gKbVYPa+EUGC49pY`rp%S~SP#h3oQ>}x;z_srQ}Zat&NS&wZvBZ-@UT3)*pTFh5&#w9O zkUe)l?YT`p8oAs9Yu`ElyicZeTeknQd;Q;E?|*ZhgGU_s_6Kv8er?WL%MNuu*;ean6a-tzHL zcieFF_&FbRI)A|9uY7UGk)N$n_1v@P4;=E!9kY&_GUq>YX1}(|E5A8(gL#iXa@&j3 zHu(6e+jqJBXJ2eNbE_AJzjDH}TkJdkpO}C8?9W-`Q_{~?(e)Rm~ zPo4eL-_QE|tTXPK^s9^JeK`H2)vlRw|2_Awb>~Az^y&4}FMc$4=H1g?p0?&E|9Gwa zE^q&;bH#>L2Yh|)>^+{mWbPTOopDCh5{sWb>)SDXy3Gh-(7=g%j+#1ZryU24nKG&S z=&@7wAJ_eeNe9Qjh=wqDiz5ymJ?6l1Q#>)!o#>bcXw z!^a&kaoeK~9k<6(JMTH>r~}6g9NTq^p=}2rF(_6rY24J&osXDw(B#7g9WkWqcl8d6 zpHruMbnX0ImZ=90>AF>_!Oo+0+O_lWDTj{h+^>88Zex1&*{t(`f!%xc8@O5TzMFRL z-LqH!9z6&4=+(Dd&t8Li4jk08SLg3j*P(6Wr@@Dg9Y1LI5!-y<&yu1N#_T|!=@a5=$LU^|8m^qsq56&iLqlA)N}CR zhaU8u&)6|N#vL?nQY>;gcpCr%tY zsQ+f;dv7+j-}r8Q`t<76t#8ks^;hITo0V*JfC`k)$lQezpL8q4^;;|C5Hvst&ky~p(K)_1d>`*$1I zbN@cwdiEVXuy6kX`}gm&*_a=isHd7bO>H~zuy{WmS>ry7ciVTrd-fQ2NbR#fTsUar zcfUgp9)0Lx<5GV#hjjhn&*p+(ItzZMq{@8vM|{xeF{$4^L&v0U;BjLI_o#JtVYe2P z{Jz-4dZsfYg6jHf6mPhVX$Op(+}Q13UDbEF8m{E9DdVS38-3`wttP}fB(p!8)%Dgc znBKtP9;x5g2TjbuR2(+?m(84rjQ(7e$V4o>lgAt|dh&#EV~6(m;U@m!L}qhWJ)^b% zbp7xrw(g(C!DGh^8b9UGNu#F@oj7UqgmFC%o;+c2kA)`}wmEp=@1fyS4w`c4PE*E? z8`@`ZkNQs6TPnuicC8M-4Lxl4k;6J~xApM;y$1I0->rA|UNx(!k*`KQssGdsiobE< z-9P%fKk2bdopb5=N2Pvq9CY}1`3Lmw8B>p5ssI0d(VCaji|*5S(VCaji!Lm=*W`&) zhsIyY-*q(Ybd6G}{~ize4t0a{(zWZfSeK!6LGXCYY$AZ7T*7-fM;1_N-KP((l zHfv3*OCxC@jUVD_+$ZfSt~8Lw4{#F#_ zAB1pt&k#QRdHj;;*&%eC^2eY3XPYo+`Hi+4vDKbOv&`I2ByAgDiI(+fB*pk z1PBlyK!5-N0t5&UAV7cs0RqhsNNqGO5g7|0t5&UAV7cs z0RjXF5FkK+009C72xLJ(zysvtms009C72oNAZfB*pk1PBlyK!5-N0s`*t0RjXF z5FkK+009C72oNAZfB*pk1PBlykOct&_bec)f&c*m1PBlyK!5-N0t5&UAV7cs0RjXF z2)Mfk2oNAZfB*pk1PBlyK!5-N0t5&UAV7dX76b&`vw)}y0t5&UAV7cs0RjXF5FkK+ z009C72xKhKs&als#+hX-IssAlqMI@uCP07y0RjXF5FkK+009C72oNAZfB=C)6cBJP z#97fN0t5&UAV7cs0RjXF5FkK+009C72oNYb0Ri`-n=&0HK!5-N0t5&UAV7cs0RjXF z5FkK+0D(dj5O6QVS|ckW@g_J*k-d1PBlyK!5-N0t5&UAV7cs0RjXF5FkLH9svROdQhAq zK!5-N0t5&UAV7cs0RjXF5FkK+009C?1q9raipfua009C72oNAZfB*pk1PBlyK!5-N z0tD(25OA*t#VG;=2oNAZfB*pk1PBlyK!5-N0t5&UAdpl*z&)v$`~(OPAV7cs0RjXF z5FkK+009C72oNAZpdJAM_j*vAB0zuufpQh7tg0y2p&K^=0t5&UAV7cs0RjXF5Fn5p z0a5qtII4vJ0RjXF5FkK+009C72oNAZfB*pk1j<}Mz`e|G0K+FhfB*pk1PBlyK!5-N z0t5&UAV7csf$RtfxM#;vEd&SBQ+a0RjXF5FkK+009C7 z2oNAZfB*pk1PBzCfPi~p&4=C)AV7cs0RjXF5FkK+009C72oNAZfIyK72)GyN#OXEx z0t5&UAV7cs0RjXF5FkK+009C72o#opfO}!hhu#n%K!5-N0t5&UAV7cs0RjXF5FkK+ zK#>XvxEJZf>2`4lR900KM@RLN009C72oNAZfB*pk1PBlyK!5;&>K)~I7K!5-N0t5&UAV7cs0RjXF5FkK+009C7vLGPfo&`iz5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0eANR0RjXF5FkK+009C72oNAZfB*pk1PBnwf`EW~ z77$fIfB*pk1PBlyK%iU&Dyu5Wb?C-TfB*pk1PBlyK%jmBQTO^?oFzbj009C72oNAZ zfB*pk1PBlyK!5-N0*M3!+!JxhOn?9Z0t5&UAV7cs0RjXF5FkK+009C7>K71juiwR4 z0t5&UAV7cs0RjXF5FkK+009C72oNBUNI<|n5tqya2oNAZfB*pk1PBlyK!5-N0t5&U zAV8pg0Ri{=U7RIAfB*pk1PBlyK!5-N0t5&UAV7csfzlPItg6V-;6&YX1glyC1PBly zK!5-N0t5&UAV7cs0RjXF5GXGJ0r&EnZ=)qZfB*pk1PBlyK!5-N0t5&UAV7csfgA}4 zxaSB~wFC$dAV7cs0RjXF5FkK+009C72oNApUIGH{pce!P5FkK+009C72oNAZfB*pk1PBlyP~-vv?nORDMK0t5&U$V8yBsv?tY@(>_E zfB*pk1PBlyK!5-N0tE6bAnKlHaLo}QK!5-N0t5&UAV7cs0RjXF5FkK+KxqmHxR>U< z8WjNo1PBlyK!5-N0t5&UAV7cs0RjXF}NPqwV0t5&UAV7cs0RjXF5FkK+009Cy z7Z7mIIk5%^5FkK+009C72oNAZfB*pk1PBlyK%g811l-GEmW`1B0RjXF5FkK+009C7 z2oNAZfB*pk1adAQ;GT104GfP0HifesKLK!5-N0t5&UAV7cs0RjXF5FkK+Kq&|a zxR=5#8Uq0W1PBlyKp<-Zl~om4t4l=$2oNAZfB*pk1PBl)Ljh6uGMscnCP07y0RjXF z5FkK+009C72oNAZfB=E)3kbMp-&BnR2oNAZfB*pk1PBlyK!5-N0t5&UAW((^0`6rv z>4r>z009C72oNAZfB*pk1PBlyK!5-N0@)W3aL>M}8VL{}K!5-N0t5&UAV7cs0RjXF z5FkLH3SnE(L-1PBlyK!5-N0t5&UAV7dX$q7_eRg~PA45&B-MBR&X-t?LP z0RjXF5FkK+009C72oNAZfB*pk1PV+*z`ejGM0W@fAV7cs0RjXF5FkK+009C72oNAZ zpg08t+>3MG^qK$x0t5&UAV7cs0RjXF5FkK+009C73QR!2y}%|!cL)$5K!5-N0t5&U zAV7cs0RjXF5FkLHI0XdUi*w%eng9U;1PBlyK!5-N0t5&UAV7cs0RjXHOhCZBz$Qd@ z2oNAZfIyK5R900KNl$f?009C72oNAZfB*pk1PJ6)K-4{-+S(#OfB*pk1PBlyK!5-N z0t5&UAV7csfsz#va4*@ZH827M2oNAZfB*pk1PBlyK!5-N0t5&U$ftmSdp@4r>z009C72oNAZfB*pk1PBlyK!5-N z0@)W3aL>M}8VL{}K!5-N0t5&UAV7cs0RjXF5FkLH3SnE(L-1PBlyK!5-N z0t5&UAV7cs0RjZFFCgHaeN#0OAV7cs0RjXF5FkK+009C72oNAZfIt}v2)LKwq#H5; z0t5&UAV7cs0RjXF5FoHffy%0iMdp!#009C72-Fo2b+2pU3;_ZJ2oNAZfB*pk1PBly zK!5-N0t5&UNGu@Wo|sGl0t5&UAV7cs0RjXF5FkK+009C72oNApS3tnMu8A`Q2oNAZ zfB*pk1PBlyK!5-N0t5&UAV46ofPi~qG6e__AV7cs0RjXF5FkK+009C72oNAZfIwXV z0r$El&JZ9#fB*pk1PBlyK!5-N0t5&UAV7csfy4p=?up41AV44|0+m%2IjL8r1PBly zK!5-N0t5&UAV7cs0RjZ_DIn^ePi<`xAV7cs0RjXF5FkK+009C72oNAZfI!I#2)LK* z)EXE80t5&UAV7cs0RjXF5FkK+009C72;@^hz&)SZ+9E)J009C72oNAZfB*pk1PBly zK!5;&k`)keFWIRzFaiV!5FkK+009C72oNAZfB*pk1PBnwr+|QaKDD()fB*pk1PBly zK!5-N0wpIc)*;QoVz|0O_x009C72oNAZfB*pk1PBlyK!5-N0?iN*aBl{lq67#KAV7cs0RjXF z5FkK+009C72oNAZ;0FN#_a7wuF98As2oNAZfB*pk1PBlyK!5-N0t5&UXoi4*do%D9 zB|v}x0RjXF5FkK+009C72oNAZfB*pkKL`l8{~+Oii$tKZs-j4Gs+$A|5FkK+009C7 z2oNAZfB*pk1PGL&fT(*JPP!ozAV7cs0RjXF5FkK+009C72oNAZfI#*I1l+T4szw3? z2oNAZfB*pk1PBlyK!5-N0t5&UC_@1O_cEMxLnc6g009C72oNAZfB*pk1PBlyK!5;& z>hD3k>0RjXF5FkK+ z009C72oNAZfB*pk`40Ri{2m}Y|{K!5-N0t5&UAV7cs0RjXF5FkK+0D)W!2)O53Sp5VD5FkK+ z009C72oNAZfB*pk1PBlyP!<9L?qxB}21$Sb0RjXFl%POmRYeJo%8&>UAV7cs0RjXF z5Fk+O0;2B4K7;yCfB*pk1PBlyK!5-N0t5&UAV7cs0RjafAmCmQ)1VUs2oNAZfB*pk z1PBlyK!5-N0t5&UAW-ZA0`A2=gZfW^009C72oNAZfB*pk1PBlyK!5-N0tF!;;9d~Z zpc4cL5FkK+009C72oNAZfB*pk1PBlyQ0xK%?!`WX`cHrW0RjXF5FkK+009C72oNAZ zpr`~Yt161BzdB1Gp@67+LNM705FkK+009C72oNAZfB*pk1PBlyK!8Af0!!3)(m4VI z2oNAZps50jEf&`78rm(_)bhD6_sRdlyKm;cneSR?6*anqW!vYexy+lHH!pnfcKBpw z<|X7LK!5-N0t5&UAV7cs0RoE@5O7~4kqiU~5FkJxV}Tf4KK!Jx>KYjrXrYWRJ{?Xz zBFy?Ux6OC#6plD0bn2YjR+?OURaLn4>Tv$4A%2hu5FkK+009C72oNAZfIv0`Y$=ls zMRgD$K!5;&YJthe6hX8-c6Ed9;ir>wK4}O3s&J$2aqz8Lg^_!N!Q15AK?4K`5FkK+ z009C72oNApHUa|fWi!{ON0T5`R)PEI?oasmVh5FkK+009C72oNYR0a5n?n-JY0K!5;& z;uV-XH(Ymd(RBY`7v{XHd1#`UV~CFgZoQ_Nbtq1N009C72oNAZfB*pk1QrOis@iMd zf&(825FkK+0D)!+#J$0XZX5ax4vT;HZt}LvgiU%ktHLZ6e)+lZ$;?!J-+UGB{C9Zf zku28TU{!-ggaKQIC6){;bqrnBZ!lXvFMK*P%y=%<;3pr2e_tBje>u(aK z{I*v0kK7whnO5tJ0|EpH5FkK+009C72oNBUC4ukO-({&T)es;+fB*pkix8OkVYuP4 zMdZoK<^DsnQcVski4@O1p2Pa9t6pWboHwBX0t5&UAV7cs0RjXF5Fqe_fIZxQknq0* z2oNAZfB*pk1PBlyK!5-N0)-?{SyfR;-O(2U1PEj(AnKkWlxzeD5FkK+009C72oNAZ zfB*pk1PBlyK%g-J0r$pqTq8h$009Ec68Prp^x}&zo_@x0nn1hd!rJSGrCO(*c1VB# z0RjXF5FkK+009C73SYoR?u9=`dPsl(0RlxUFn4Zx*W0vBKjXNjz@Gbu__Nqz;ma?= z*(Zbt@2YvxX95HW5FkK+009C72oNAp-~v{-7x)C}9svRb2oNY6fgS_GkH)5=?J>1& z7baDN6*`p7;0=@j0RjXF5FkK+009C7Nx=^e1PBlyK!5-N0t5&UAW+x>0`7%9KYB-i0D=4oR902wuQ4qVXh`6T&(g1a_Hp_d z#{>uvAV7cs0RjXF5FkLHfCWU|3wUyLjQ{}x1PGLsz+E?l&puf=;u+6|=bl`6=o0|~ z1PBlyK!5-N0t5&UD0l&HPA&K;(m?_Q2oNC91c7h9PA|0h;^}7!?fBD~;ixfT*Pnz{ zI)#^?3)fu~s^%A39}BuatFILfIj)6!SXGsJ>+QUG;j1|zer4g~k5WJHzY|`c5oUeb z!hO?xGX?r@5%wCBX&omX9%g=!X=eEd5FkK+009C72$Yb3fO`o|rlAlZK!5;&1p;&D zF8KKUhc<1~&lKSCNAHJoPbt8D6-SqrY8}>Iw{W`}Bk%EB3r{~Bo_aWZ_-^6$Q_mU_ z*m}1FH`vgT;hGB@s>yi*1PBlyK!5-N0wpXU;9kO$YiI-r5FkLHUz&!z% ztON)UAV7cs0Rkl?AmCm?lW8af2oNApssgPR3vJtm*-uX65r=Mx)c)8_M z8(D0&RcOCrm^~{za!~~PO@0M`&-@|)v z<*#WiHBDgfHX;5Z$~<-I9J+28UVS0+5^@qCK%jX7l~onZt3YW21PBlykWB$m_iSpa zivR%v1PEkVVClAD-(Q5G+l3gT{^Ij+>($}PzlCq>>=~2gR`XI_h8xSTylOb+tguqY z@B6gOvSG^|LZ89mn1jNbujHkp8p)YJya%?}K6QJ?_3r=tQ}|?N&RSA&vjyTbB~5Xq zyQq6oH2Db-AV7cs0RjYyUm(e9mEs?*5fC6ifB=DF5{Ti*qfQIk?4F9Y$JB1Qu=~%$ zq%vE>^^0FEjJ7YByk*;mgMS+`4B0OzO0)mrN$ z*us5*z~)=~L%ION2Lc2L5FkK+0D||;P3%_xp<(G|!kpPnEUp*<0+|aOaAa6#qs+613;qzEeLO*aSqTszK!5;& zj0FVTGp3S}0D)?O%BnKkK-$C;Xs*EgdFeGQ)jIu5BgfA_S^w(N=bmh&P)S{S>6uVf z72-zU^`;k}srR&#MK7@U65+sKg}7b+zb`5Jv6R37n!Lt6j(y3c!e(2ByKX1}216iF z=mM**k?e&FZQB+4i1d#D0RjXF6s*tDgJUUK*009C7N=M-Co5EYKrVrUYHNlSS59rVFnd;b_pLDN(*{TN`KRHuU!`_JZofkK zQ#P^VerB5gsS*5#0T?_fE9*<7_BgJUR|{M3 z8h-VMaK`0f`yYkYZL%=dIbWt%vgDHKXB-nK3W4~IkbMp)ik_BQCkJm6T4m!O3!OTL zt{avbj`0+)KxI`$@pfL%2@oJafB=D_5wOg?Xr@X>2@oJap!5Y+Sv~yx;IP}z!jS2msm1v)FZv@Qmxa^I3_UjgK*hjv&_&cg!pxiWtR^fRtal#32Su?i#Pi( zoZIf1dWY^CFK4-{s%u=J`J2its6VcBfAxh0M?MfBK!5-N0t5&UC}DwS|NEzeZ48Y7 z0RjXFw1`05vwh5AVVm8qsg7TYh9sY4f(;p%(sK*Ba z1PBlyK!5-N0tGLSjjxIp{2+CZ009C72o#^dnq5NWnPGBeSYf3EonLq3aN_x)d;bJQ zN;_*+RcVi}2uAnK*WsB*!g>D}4%j99{g2^`&zsoi9s|O9n>4YkVoeZ;Ytplr;&Vj5 zm$K-oDgp!u5FkK+0D%G)u)@87Cr8%^5FkLHR0ZaK6JB^K%>E**ylUvOL5Qn&8%l9e z@PIAD?bn7IFApEQ(@?I|`HriHU4I&e?G#$Y9X$#!)qV48d?V(9)Oo#BFfOjBjT=5M zw|rQvRhVBDzWpYA{#p3=qwwCF;oUdGJFkbgUr+q{%r9pzXyp43ZI(_y<2XM8Uw;*D zxHQ~*bNI!tLhnIMX>;d&!~Y%Cl!D466^Q${4;h)X&?5534QifvU=ev-CP07y0RjXF z5Fk+e0s`*EKZiy@fB*pk#U=3m+u@i=A?}@Bot9}AdT$=$7S#={-i?9j?eNPYhdc9FhlLt!>qf~e96`!?lm8u5X6_ouD(|NbGTaV z)AQ%2cJY4s+3?b{sm*pidaw57%#NDBpLk~N$W_0<=bwg?4i7&X8}=Aoe}+cR_8S^j z=ny`c*+>?bvLmo*?_}St$)weOLqdFYblsOaXHr%k0t5&UAV7cs0RqJ%AmCm+b5(3T zt*ojjwl3>00RjZ_ClEK-SP*iLt;WUf_ui8F*`Qna(YVm9Z~Z-q5%mpwq<*SXj6Z+% zWr%yW7jIgweK_sn(6Lj35ybBUbXh<3v-NJNQXjn^o_jnz`$Ty5@g_##6SOUB_5|W1 zn9I%$bLWOp1^*RLNtIrJGE_CF-7oVLDuVXp5# z#mHk(N895t;@dQvnc}WITWp{DiAnIinV9)6e)R=m&8}hd5y@KazlDH%GO;G)Zrvso z8P7$Ear|7rsW%thRwn@h1PBlyK!5;&Bmx5NNwnl8K!5-N0*M5^{31R3l1qlF`RONX z9zS+p>NS(Q{WR>nUx=BToZ|ZTHf@_+y7&s#48DKzzwr8u)X#_S7W7^_#iyL@9{=^^ z(7j(+wta$nBW`8u`X#*`n0+x`{4otxNT-nZfUi^I)VCN5Yq1y)!o zwXJ=8Xfff4Fn(Ie466hO68BSz>)aEixVO^Ay%Lp>Ik7;Z{bb?;!Vlifw23%<@omr& z0RjXF5FkK+0D+Pgu)@8hr`O;J5FkLHcm@9VK{)s1aQV4m#4f3Qs~dW^Xw3nBIy2mL zLn_oBmw4|rrskC*`250C;qm*!t1qUOrO*0q{c?Ob(SC)n>YAa`+F^|@VV#XqAEXi! zxbyljYC?#6m}H*fBKHUH$~?QA$pn_~5PI}aP~`5LQd%SV6! z0RjXF5FkLH2n7V(i*V9(nE(L-1jh-#5ulCHHI2vY-`1pLc8{<%~Il) zHJ{83@4ppZelGm?vGCHfi`cKGFx}pJGn{!`c1ou+{j)kc7ZuxhDYv6w(2?lS&(uku=&=ZRmOj^;#Y7Uy)V4|dV+v^oWA(p1dJTh^G3glBDZrwj`URZqb1Z}R+A;H@^6VzMwS>xLC%v1HIGS&GX zzB@s{z5R-z$AIs@s*-tgauOgwfB*pk1PBl)5&;4CBAF=NBtU=wfnpR`cjE%;^~OEJ z)h*YZ7!yxRafR}o*QK4y!eLca_~^aVPjy;ZzMxve5b+duv=#F@%G#TOaGJxrwR z3dH@}ha`DARlT1~bsyrpBxZgPI&{pmM?<$2aL?3AJ^};?5FkK+0D)o?$oR!b#Wpbg zB|v}x0Rk;Au-#tieW)9EPS260;}3s`I4`KLd;hTdS__U^{zKf7K1Nz40#RUgoiAiG z&Qk9eCK@l|>LLX;?HyKJd6BsmW>~PFOgt4IbUkqA!b^+%)NiP*pB9NE0|5dA2oNAZ zfIyK7wD=15%BqSYrPFN!1PBlyuxNp0mk$GnrDwbD;`B2)I)3~=^{hG_xkvchX*p^z zakVo)N?b&NQWdDX4N~JHZPOB)BrUUSSZ&SJenoM!qNUn=A2go(P5A1|)Ki0x-VYzX z6W)J2%$?hK-wXF@TzZ~)n)hvZ^uAizW7K`eeQKR)CtLy zh1J#waYvkZ8^oX3|El@n%h@45NR1mK#b=1|1}NdJ=}Z3R&*W~4d$X;+X6jB@sbgr} zCUqOcG=E;|T_hiW6h3$-yz_eaWM(Ec$wPnu0RjXFl)ONTN8L+a*Z>I-AV8p81%~Yy zmRPdZ;A2$r?we|z$=yL*L4NBsVV?tPwXo$5;nKe}@lB<*Dsn&|s{%=1b{m6S`AN%n z2z@pW8}$t9ZIX(%FEYi|-fzDiUU(|R*TFvZQ26YVMV2U(3^9NhBjA~*|2`Jd9+Na- zYneh0xW`>~HtZI94h)<2Nxe;W$)y%y6Qi!rJ{F#NG{imaXMVVd%s-hFmTSL=JT)(0 zc5e9J2Q^RR<8v|I{PC-0T{lR8bKk8YZjaPRT5PeLP;j_10e-b*aT~9~qJy}n_A5LxmqtJio4+jnW@0rInaKUK;1PBlyK%g-J z0r$pqTq8h$009C`6o|o^t#?f?8>8)C&q+U-yW=~q3ww?UOV$Yb$FOEx#Cl6h?C1XV zSLp+dfs8z)IkVFnNqY*Ce%WzL2(1my>K^vLnX%97a3x!*^`)g z%`T~*_{Tr4fPVIg)NU&=s8{zstXZlMKF5t-5`Ax=?k+ObTgzm=$bFOEp~EVRj!owu zCw@6;gP5Xv^L+UrI_w4I48VtC{W$f`!aePX>!bcWU z`^n5pt>dnn674DzA5IM3CS3c^y4AGUnfS|Lx1WX`M}@d~NRv~X_xP=d_!EP<7oQcL ze6Yzi<-1_n-u!9MD|85>CWN8eg}9Yalhe}6g!o_ExQ*NWcZ5sN_NM11Ln=ss009C7 zic}yki_41CMz;wNAV7dXlLfZfExqWokEb?x&2zf>iu49MbxuV9B?5&b5bw#PfvrVu z+0lg2#VFgPitvB`OnnQa`O)^89f;4Cb{G}@czO8w!Qb!uT=Qxne2$xk^yr_UBmaFg zwRd^lRK2TA-SZ8dNqZdFP`3P>?>#7-dQsS8bhD!EHM<%&qB-i+aOD3bdmnAh8uIqJ zoNxY`$nhai+?3|@%fnVXCjO;Vvz`T?V}`o$3I075?NYupQvI&1sz}iN_+%k&n|#I~{3d%1Oa=23eT$&^jb>g$v;G8R z+)N?g{0jyVUyFP6X<_ss$u?M8P`r^3@hRRe`-M|34C`*(NG_KM5FkK+0D)2xNb+~< zQX03h5FkK+0D;^IjMz0jp!Uhk@W{QnYp}7JZ@x19YTv=h_7F}l@0dVV1s3`0zO>ff ze>FXpG6wmxaTINq2FNFQm*$+Ms(l z{+zJ#YFVi)aV7D$#IH`T=iyx9ike*@=^FEf_LHgZNPJiELrE67$9+6{3~2Tknp8pD z&F0sqhh=O3x^7ZUnHGvK(fHLL3L@t2=3g*pG1?v{CBEo1Zz=8)f9UaI?s@ap5CH-N z2oPuy0Ri_GVb>@D0t5(@u0Y+zbw@`M&|;n)(>tU8;35qU`2&^DppRvuilwloq%R`frh1=N>oEmk1Cb zK!5;&5)}||FVTrLECK`w5NHvB_^bwT>maAK&FAE#}_@nU9^o57jpS(XcUBq#I z1!Ayi?BR*OY4Oe1;pOL=+tIkR`IjfPaL{wX7=AplC1dWJ_6{-hoOx<|Kbh)!UU(+i zt}^|GhNatP-i0jWj2nMVJt4%!^(~TO%)JG7@OSgqob>o*uY7*Rp=Q%tY#(;sx8@0- z2@oJafIv$KR902AM6>16rl@2nOckJzP=>p8rXbcuQ6o61yo zH0jndan1LDt#UZVx>fHpA;i#k874P>K~4HT2ZVJtDySam3;_ZJ2oNBUEdc@dY)Ptz z009C7%3dI@)ZAjbbcMLay{anxWd4tzdm_B~N_w+#4{eD+iwN`^5`I3VDJ>`6vm~vJ zxQ)s|zYZNbrk!s2!!c7^;617NoBf$wwpiG+cA&%;^Uu@dxRS=PuAp<4~CE4tEWiM&64ew^z7r|fA7~* zk5dE)5FkK+K(hq|+?!3P0s;gG5Fk)3u+47i68GO8zW6--WJ?|2du!PDz_3iah3zlC zMA&-QaOL?6&-g?j*8=fXsWIXi3O2>fz2l2iPp^(Gr2rdn^B3TxRCg)9 zJNfo&Lwr1+dHUBs>d#wwX0m{L4D>&8PyJOnOMn0Y0t5&&Pe8!Ed45V0AV7csfog#b zdxW*tNyljMcc#|t(ARUq^nZsPM%6r*3b;NWF4&!#-G5VZ&iZS$ur)Qo$#9d`F54gvA<9=Lu2)M`9#O0aX`~^Gj@lWEGJA_+X z`VH#^>!V`?2oNAZpg08t+>3MG^qK$x0t5<8pziYaS6>LPJYQJdy!ooI{ob{WhWZod=&e`7yKkmg;Q&bW@$KVFF4>%l7nY5`9!1?3 za`;4m009C7Ny%c8A7zhv`K%j*M;)W2LZI#}Kf?DHV-MJ6m32_71UIS}&H7-TX zL)5*cR?^q~vVNh(!fH)MsT{V^>g7L!w!3go%e0xlKL1Fj*swexUBKkvT(KTYn~ggH0k2|)Ux<{ zQeULWJPqA0w*mL~Te|sQMfmcIaQ98&q3Pkpr^CE?^$ae)kG6Zi?;gDj4{`6VLP&1@ zCd|ScFNZs?4-enn;QIxZSTbzXBefsW;E~OZyvK($xqjNz1OY_|5FkK+0D)!*6vp2I zn=t@I2@oJafI!O&4BIKhe|y(VvuB04i+cg4o32O?xW|ye+UtfGT*ztq>Z|nX+t%1p zKK)wuj$@!UEiK(P?L=80uD(_{<-%+V#A%H`HQyJImD;cRTm5ee%&_tex`&N>Wtewi zws`aW^G_kZayV{jlX;4N_Amcyn0{lJT$x(h--NWwejz@t`?k)l+k~nbDzavm(0RRv zF4l8C>83K(RXliSvVi;Gt;0XgNG&4HQR=;Uvnq}=5_jSH_a&(7lC^+YKG1PBlyKp-0e0`A#RR0jb9 z1PGL~z&5+5k3RmxyP-ELre|%5;}`+_dQPo|muek`Orq3p zXvoo~Pl>kBsoCckF+XM6BDRsKnZNGmxGui(z|=bE{3kbm=H};|9B#ih^X&EH{OG;Z zM8|J7G$F;We8tUwBmx8o6s$mHRYk$}O~(inu7Id};m(bo5g5m z&^cKN5FkK+0D=4o2)O6ZT1x~75Fk*30$cBz{<5@B{uds-x6nr#Uj~_$I<6k#200Re z!V!oso~(PLChcC=4MWFH2@1uKXx6t=j+?s2*KH3PLbejMH0t5;`V96yz-Tjv17t`m>D}c^5 z+J(1X4RPgZT8i&al?W7qK-|dWufGrf`bQS}(r-wzO5!TlT`|T`mvmt{v_%Ags1#E}K+0feZv@e-TbRJY;KM zrn);mHQt3Bz4Lg^>~PObjklbBwQakwMh;fICVgjW3=qd}?KCGP-Bl*;XgzqF<}_G; z+2vQP|9nGdb^5IZYI;r;8flaVpr?F;LRfUy!mUWUV*q9!;$-k zCmyKxV&aqW9h_Y^NKoqa8R6~M668;m_1;?(Wlv>ZzgyxWjTeX!z4%-$QFVx<)?f&f|3(O*<36jkL_NY3CMk_~7)MY`OmQBlY~^T|`%!blJ_HA={n*4)H5B zSx?jdow&-Rw`k%96d*u=009C7@+TnRo5tmKBT2vu1Y(R=&-V? zB2oV4XO1iK>n`Hla{FX^PBgz8Y9LT85Z^F**}389aUs5AGFxfAO_OcSk~BD+sGl?b z8)ki$s6<#j+soVsZygp(@-Kk6sZ4WrabGanq;H~Kvuo(Q-h$Fu_|Sv}#LqpEg?8&! z5d*wWJyiE}_RqNaGc)a%l5&$i67y$OZZ?YS%>5%K!5-N0tE6ZkmT>&dF`0S z2oNAZfIwjj)LrBL?wj@Oj$X)vxbwR3?c7=&TzZ)>bh}z-91zHZKwQ^;=zbyYo1NX4 zAgr@dvc5n6WU~B;a>v!)NjLFcWwk^tEIM=2<>SrYRHnL5&paAt&suc%YUW7V)tY7N zer~s1-E)gL6Tfkhqx9l4IjXo;wQl}e91R@A+w#p<8o1W%)6YMZtct~#NS0r20t5&U zAV8pi1q9p+cye@&009C73RWNnrt9v9eA_i4E(k566kqN3pWD;>yX_t!Zlo*`$d5o= zM||AD;e^A&dv9jH!L`>(P-k3v{PJ@N@;4#t%g-k()Ui{t{Eg>cqf6+LWS5;-Tk|)S zsqRwz`ohC^H{QGSt8p*)M2qp$%hx_`o#bV))kQyiw{{iTJIckYdF%Gh&0p7~?wQwK zs(UVLXI^_TYgMU;009C72oNYX0Ri`7n=$<*K!5;&LJ)|nFWc0P;C%CSn0`YcbS+sQ z>TZM+qjDSfN>+*71d%7-nt z4_j=XxZvL7!lU}%JfcsJj8jV@M{r3OcyALo~ zsxo1~A8^Sa0+Iv-L?kJK5=6-vK@pK)W>;~|y8eb;MP^L9#ubb!tE(uuhzp2_sbbF7GKChozhuK$YMHqAu1rlXXAto4&j93O0t5&UAV7dX zZv`g&)4I0}sht1;0t5)uUm%5~x7#f{jMIM|kN-V;vtHNt{538*E8FFr-k81Pnm}8D zU;Z?nD!w^PI@z&M?6qyv)7&f?VxFd|x_`9APK8Hs!RdvIvyP+_IP)14}a@3;0^%-1PBlyK%kET z0`7gp))WB(1iB+I-WcmnMQSN0u;UxD9TQEwdp_&Q`4SoddPm_LH`Z%A@z&)AHmhlP8@GkxtAllvaLWthz>_5^XXM z>ez@jUHQLTJv}oxwcz2OO5S|}1PBlyK!8AB1Vr8Y!mJSj1PBlq0)ey*#(EoP$8pk8 z**o>UKH3ilD1WJ4rCfB=Ec z1=8j!X({f+lw#1Q{%YclVfgvPBeOekFSSf;w8bzC%^3P1klxLdNa>Wv-QHTF+zdrZ z>?uD(jdlBYo_S;Q9qZQVDf+wR&QpBQo7;5{SbO+;^IWKv009C72oNBUQ$WByCzNai z2oNAJ+5$VY9G80GnQ{L;!-kR$fNfnyy7QjHHcaE`k-#-qw0VB=wDp?3V#%fRWR$0X zz&h(kI+k`gQhM#zqrLvy`Yf_&+uLpuAkZa&@y1w}YElgW0t5&UsIGvhdv)hV&j=78 zK%goDtF94itefr3FrAq?acsx`V&YZv=e5_57p$JW>6$=WfnWZ#?d=(ErhS{z5vtuy zPdwh;`s!C(pO=RB6llSPVw0_V+QLv(?%^M*=`WQLpD?H0?dFhia~ zA3ySNq3pw!d4UCoEkeCT2YcjH?O{mo4|90FD4lmooO?>468*^hzWjk1*j*a-kU!j@M z6#@hZ5FkKcL|9F{NE)ese^m_3!1c{c%FBPtS-g#rMcg25vx^Tg^1@15SYU;jw;-a(K)>OZnODq*H+oXQo zp0UU2Eke0~d!yhA_ZfRq@eHH<@K=0lbdLZ50t5&U7!85R|BN4vF&ia;Q4tt#jE%}r zjgvrc1=iUhR$MjP#EC~{?{xQi-IvE#4ry*?lL@(}R_2?px!n9~KW%f8Yj^w5y0y%5 z&yV#xh~uUA1(~$qf|(oryPRd!L65O7K09u{zDygh+&Wf!ahZaxW$wMJ_1<)M=I_^Q zpy@i%rO!6oKGG|U;Y;Z|Kj|Fn5~=9M=INV7y-N!(+Pk`+u6dLn{+{D%fB*pk1PBly zFtflg{s4M9|N8Y-6dH zZ7MTYzueGfw|vcJZiQ`s=;^r?eb``slpp@K6Y3@b0t5&UAV8qM0s`*+_0}2z0t5)u zNFeQ_zVVjX{+@JH_D&bC(*pL7d@dHqvGhBwk^JCTn`m7M z+6tU^THJ9<+fQb=nI5yXUp7OYJRheR>pg$Xlkw?1&l{G2`%0_DYHJo4c)hZwul|%+ zykB%5sl}etLzO-R+(-7|@8Ptmng9U;1PBlyP(VPyy#SW11PBlyFj@lJ?$+GGWny~p zZ*ktK6K{0+=Ph<_vxCi4ZM<-;*mRqzKJ3E%w5fGwUxMbg_AN{PyU>yR6QQ;014Y;RPGQa!_H#v2I z=PWt(XWjc`G#~!%`KgHj0RjXF5FpTrfPi}^dddVR6s_aopk2x()h{E%~$MJGf5GoqTk*{MNg~f(vJFx+c(8Af3N^ zPr-d%=G2o`xUc`pwpA9pdBfEO^3OLbR_)UE@daA#cGh(8!{$47yV8o)x8=@-JM_py z#fqdw^u60+XsKn2b*5bI5q$W|K`BLm009C72oUIjfPi}sP*q8Q009C+An@{6$I{DA z8b{iex-|ejDMx1)FIuO~Mi2)c5Qlyz&OId#`DXm*zh@gramlpjd+*XI$3%+wPfUv} z7Q?-McH&_D+@U~vc#ir>$vWQjo=D%Q=rCRXr(zYn{8h#BS1|WF>t}y)Rj`vCc4?z6 zVxfYs{fe$|Pd)72uh#VCq&}qKc=*Q~V|}pF1OWmB2oNAZpeF*N?mb~uDFFfm2-IF+ z$CeFLF1sM^ysd}hYHjf^*Tw~BJlo<~$G6!Td#5)(o1yDJ{q4Rs`{=Z;#qWB%DjXVt zGmeY8xA zia&%ee#vwdcJkS}8+1}SZ`l!i`18`pM}PnU0t5&U=()hGp0}U@0t5&UAW%DjtzqTW zV(s5_)OoSEq07Ee=~iSNuNIYaQuCC zu0yHk7JKD3aoBg~UUlt<6<3M=EOQ^Vhreqxq#^R2dp8y}4@BV9VBQFh zZs%U&kcG5OR?!{5XAms@@p-34`ua|xlzwLxJs|e}dkeMD%>3r}L^?vXiz&TB+NR|| z?k?7!scNJi{><225+Fc;009C71}f0i|A-vuNHk7>009C7x+9Rbc3E`sNmZo-Ak!+{ z-ldmr6loKiOgibP%#Ein)2=d^ly=Nmf0N9;ZeE^uay)qdqzY3Is`U-Qq#P~?Ob|Hf z$Vl&sN~C3$k8L{(P^P_)itgXN-zQ@6=a%Sf-XbYj{MPqJ*Ed^VVySpp@!tdI7GGVS zcbvKN-SgMD`VYBtKa;WO+l9Msf2L&a|6P6UczyT2@i71V@!`+LeDmdQzT=Fe^YC{} zr2+y32oNAZfIxQz1l+spS*7yG05jNd)o;q-{%%Z~3xBo`NaQ9{U#Ce>+oc`gY1` z$7b%$aGAcs@sZDUy1ag>Hqslb_kJo~_m+77XJVgsM~c9AD{Z!YnKjvIiFp;$*~NqZ z=|qWD*C^2U+ioh5d2ZId-xV8gIrpmk4{4&`w|}t?mv8KulB4tRKQm(gTTCGBOD7Q^ zK!5-N0=*Fsb?=Rlwo!wUFInQl#)N78KWEoL`-y9LMA1yQX=Pn%`?tNWiQ>d4282|5!-H*91 zv1EMao6nxS@P>Vx>z*?wIFvE^zcNLaeWyY<-w^2qUKP@XXT+0F7Vb!)A^+u?!u6*D zY0IvE{(PhZsyj((Z=??#5U<|8lR8S49g&B>q|V3}%Y376K(GbJg2GR&sR+I z;dCF(H!D6g9{YVf7F>9`Y*T%fPGJ4~e-_?$eyZAv-XDdBzo^L&=YQhyVuM?Ktz!A* zCP07y0RjYa3kbO9rjn5W0RjYuULXbaS{ETszu3G(lN8EHd$VNHslSTp|Jj_Df}Y$G zyWDEm=;_zPkN-uco6WBRyl9>5&1tSPgX-dD8q+kIK5AW9o!F)R3;gJ} z_uPi66L3#mI;Z%qGD|KU>AMIy)8Fom8%r&6pHu(#yT?I?$E&x?*;+GO`aQGz+vDqp zwfWXmfm%AwI`j{J$3nwXK{}?oP}*ecSbT{>C1fT*fB*pk1ab-pxaWkDjQ{}x1V%t$ zt6j3YMW=tho^?VGN50YKkzz&@Q=<{5v;@;9W|;GH3b?29TqmaVwxGj(4<^>u`g1yj z@0J^~w|6YEtp4w_Clz0K(MgxRyJ+32qTEW=-t}-qYECDZp8o5SW8HP1j!rX8UsCz= zAImhIc53>@x_A+#Evs{rEO0(-!XG<=F1zd zj@xgZIa}XyTy%D%<06WtEp{$Y;*xUH1^qmPFB;qMVzO{*<>O7y~uG{3#!{~^|Nx^_=jx!29h)`0aT=fz!j%-lp;t$F@w*$8{u<0Y+*?_E0K$avSs zCRLx-dmj4j-kvBssXZ4p7x?8*BkiG7;u{7jB=m-N#32WjAO8_Qi2V;NQ#|eXly;ih zX4mGsw=<8At1pknQ!~_)_UvA3z2;u+uiQFsyfy7z^Qr%cAAGIMU*UDukLN8nLzh}V zKDRRa$;|27BM(Iih;Oo0Yfq=Uv+)+`)A7dGLJLRE)x8&<6>0PQ4%6D}$EVki6y8i- zzW(aCD&`r^5dRIxGyKuXb9avag2-s3S4jVH zSfsD>{rU2E@PU~9SmyJ%>-Ov&*8~UHO5py7pCG6l=dMuDUEcif&z>b$q<#y^{h-DZ04jYvNajb*lyS z47or$5#X#7W9wbY40DIwBkhopHl{9@{%~QWKkIE=u5|N!)AYU|{YhJn|LL;-i?rSn zDZG_-%9%Yo7F;mW<}oSsy~4`P4b@W^yu`}U^fKa~{w0=QDSr07NMCo!pN3#lnK^?^ z``i?bx+mw2x6J!xr0f3_X>osv9q(J`B7Hrn`8>3KP{}(({P0)OdtjaV%e7^iUS|1t z%loD*xYw(u{Ky9c2oNAZfWXuO0`61W_=Erf0t7}%pfzZ9^&g_#KVNgYwXUlF>uquA z1vyJM_sO{7&#~HzC*@D;=6e@#Ps>Qp_)YA*=cE>sn}GWy5f{@69DPV^xpRR7FXvX4 zLf(7q8wY=R?v27f{OG{=+E2p zFiDH)i>CD3Bfb2nTKe6&&0`mezN<*Tg7WYFb<)wzw;CguhVtPb!Fim1$k$#u{pUU< zK!5-N0tEUdFzqIUed~pW2oNAZfWTk{($*0hHTNf&lu~G>cPX86y47p4)u-**XWX)4 zVwn?KPBTphp=EZ4pP0Y%pVQVdne>9yW9elx_o{W-=7`oJ%ZIBqnELnTj$7ilr8pu&fKWPW!jh}?fOzQ zrNdK~d|uu{X=k9(+#+KrAO5_vJ~H_pdLXX3Vq`{ZpacjIAV8qf0s`)po*tbeK!5;& zJ_)S);z%1QPfTh5_TQd9@kW<_e)aa*7N2;$@VSy{E9UeM?1^c<`PytpKQTjxKi_^+ zw639BXM8`Wzt)RKBU&ol+%{c(f|ei_eUigPcO#y-mhfYImJ@b*ZrxZ8v0Xs z*%e~(C2}>DzAjKSrMG&e-+2^mt7!gnPAXcWd5CHLgJaXL*~1Sm+{ADc7`}&pIOe6h zqp0j(?d}d!Z3GAqAV8q<0s`)ppCTP3K!5;&J_xL`dbW{2T-4m1xpyg@#h6LyB-97) z&)l2gGKH{DJ2snVyWJvft=_wI*@fAPTT88AIaW+&U3SOdU4&_O3ot|d(jzn zCV^XSh>OlHDer= zm&x2{e|hg++50QB995gm(eZWK^Cq*H{=U28$tOFmR|TC3{O^C4ET47>+WXxl%g!lw z=WX%zPeqDARx_o)CZBmv+;m;8-lcELq;IwqP3NClv_${%r+GQAc=ytw(dkdllveqF z=L?a($TWg!m>&KSoW49mPVeka{Z*cf@(>_EfB*pkJr&3wb?<36R8D{Z0RjXDDzNC{ z+2&iL!`U1iT(_=qPe-(-gIY_ZblB=8=Vr_8^v3L+E?z(Qw`^q#Et0*{_WHKJWN)vr zPWDdMuUB6?TV>ivTpGB*4Ohoy7nJP9Mw`bP>y|7#r`Q#j#5c!_9h%Tld$|jzIQ#zZ zj@xd^TVV5@^5&eIFTHI_->9i_I`5P+y=y)CdYW49yfqFQ8`w*i6!A~n+YBlV&BH$^ z!0NYq#1A5c-6aA92oNAZV4wm5?gQo5H~|6#2#lb>V$X@RH%2CvTi!mWlGf8q*IX}F zT_e*%*De=s39rw7ELY_>UX#uCk`1$Wx_Eg4S0q(nQ@35LK6o2Ze_@D2`S3VJ~haF5)d!aSmbL%N_H0j*3{rMp+ z057q~eaWTcr3GK$C5mtC-`8EQ)6jnKwMy*xQTKG{9-oX)%4 z^BPJ7+*6;n*tu}WetUX+?t_ueb?8G%=UM*8{&CV#eP}9I6ZL)ga}96gGyVL(;+R86 zUI73B0t5&U7?^;7`@rZmOn?9Z0u>UtvE_)=v{-V11-swFJ>L~eES0UcHC&#}G1GO5 z7-r4@OuH+*`xB9(a@|d7KbiGknXU5LE3?b?!5sNOCz1fxPB(VECvNfgs z?sKlk&z5xE1A+AAjBBnaS#R3LJ#FS*AdNT1rp%rG_I&S)G5-E&-S=q9GF|#0?WunB zPvT>5DLsS1@-;MwI&!HKOfe;`-fB=Dd33U4(rHf_T?)CA4)w4xTJ2rdA^@9nL#>;#IDn7hE_t-ZEFt zOr~@i^jH5SzV+F-|DMbzUARoY@qTkeeEjVLc%zfE_qBWYbB;^4B#>U|ed%NIfP+*$ z4Fm`fAV8qs0s`*+7S|pD0t5(DQ6TNF+q(8JMXOs+2`%0G6Ek`$8Cw?LYhhBo6E2ooCf?Apa6ECRi*IPdhvE^Nx3>A>1Cmj+2j$bYO%;fB*pk1PJs?K)}6U z(%K|2pmb7xX&^oFZ5_ex%-75ayMr%l=SeLA+^ zz0Liiw`R>B|NQw_c+qURqMPFU?|*0i*+0)b@sZERa?j7!T>5&E#p07+iL~4`lhVev zDXt_{N+8WhdRJ6B?ep$P=i_!UrLE3W$o2hu#5c#A2fKFhfw||-=5FmDcw>C+U*m?B zjjRd{VZ+Va{7sy(hTolAay9*oHTNRJP1iNwkQGg7Kd?>?V@_WcO0QeeHxE+q{mCba zHvVk>6xmGLpZ!35{zH*}+pTBIxBN7_)Yl+ds+r+NEgt?E#^B?20%?W&e|-LGa%lw?8?fV}XE3TR?`^VokR~AbZ6gdC1 z*z+ITOj(|;F1KQAy=$CuOrE?2@;vc)oO4q1&(h1rdK<-h8^=pFj3t&VP{W+8>8ms8 zFUVyVHp8!fzPxMU`*L4fIs2r-{U2^br8|J{DI zcv{zX#1A3`n%w_Crt0AwTxkxc%lrl{GV`W%6G=B$jPi(w;(8U;CGi zOcMCVzZ5IKFx#a^ABj)Az05I;*@~)touYU}H$h5EaTCtKQ+lH_tL!R}C!XljejRB` zrB&C6Rn~}QmTw+dnpRS$w>t|g5c4nawBI8SN7@ZM?E{|P8{KwO{N>u_@dmx%O@D*t zZTU4d<0L-vNTe44HA?f%iiIYh7?G#*U4I@@I5Hj6n|=+hxXS;{`_j#MPm`W@iFxRO z=6d0DAZj{?BmMc)<&pO7$&(H_rPyE9w{IJd<*PYQ>m~A3=i#5e^XZFCvxC7qy< z{vTP7Qu=R+MOwZSHho*^Oq7A7`RAYVsQ5v3X&??7B|v~cr3JzKh8WpE;=i&`_l}YOr)N#vwm#1TfB0cHg6(Q?LYcP9RG{OVzILgfX$X)Q3-hBhtdeX;KPp-*$66 za9^agxtWw!yrqp^R$O(uQKYE-cfZilJ{eoSrn!2)ZAv@t{PNH^>x5iiElBH4H{B+7 zfBTdhlcezd|9rn~J8lvnK%n0O>1z_-IjTUT=`HCyUsE8ztON)UAV7cs0Rn>-5O5## z3}~MK0RjZ7CXmiIO^dHv_a^B|Khw9^(}|3I*ik%fK>dj?H}^v5MN0iY@PNYm)3o*^ zeVHJArC_R*tS-N(`G(`B>zjLZOnfo0=wi(!{41>*Yp)+K+qlgp`E%<`o5rU-fx3A9 z<=mS6K!5;&5&|#XC_ZygfpY1b-B0dYAit~x2oNAZfB*pkgBB2QAM^}pp8x>@1S%+y zV(y>#O7nEF?xu8T=s^d>%{O$n)+uXC3)SEJ-gwR4?Kgy(ve--?q-|u<>1$~N@UEn^ z56nA1_ViZ-x>Di9N{%@+e)c`l_K75ZCO}}o0=vH>-tze+=>Nybji8#hl>j2mz4kk0t5&UAV6Ty0#>*WdIq#lfB*pkRTfCQvA^~G zvCanN`}oL1aq6*g?9ZB;q6{#tw{h&eXS{sVNayvHNDn_4zd0g~`+20{^dc>>WW3|U zvDpsYsr&MaJOt_wy`#zxS@V;*$8?xpC21@%Z0+K8W;oW3P90^p4}kYvLC_jEm3i=0g0Q z4_gBS2#k!t>)z6QlUpc#{O$3VYYUZ-nE(L-1PBlyKwz)}0`7zL*E#_L1PIheAZ;+S z(rV2URhL~M7G5M4d}be%#~z6%pL}{458NO3{xxp7G48mf>M`Fmgr$~^=PVKPwyd>$ z;_>D&rFY*Mk3Kwr>3QK=vGuO8={Av;g62%=^+tNJar&>DThurtH7A;E1PJt5V1WhW zn@7Y_%M@t#u}5Ry?YnkxMuCQ9B|v}x0RjXF5GXGo;9lNNNdg225FkLHR|4tv#tT-D zHP?$3SBYhxAB!v+i!AnRTWNcn^ylus#;rHTbyqe2bn7J7XWQ@-0Rp2ZF#iHQ{5H?t z`^6srP|Wo5i{kSiE>=Wt0t5&UAV7csfk6w*8gvJ=Pk;ac0t5&UXc0&Y)zdBtHd1JT zaf1MXo(QCue7n9Wc6)1l{a@p%%evph>T5SEFPg5rv}g(W2@oJafB*pk1O_fp>iD#Q z_gV)C5FkK+009C72oNC96@j!x*hfAWZ~H)VC#wU#6+7l95FkK+009Dn6|lm6u>M*nK!5-N0t5&UAV8px0xP^A zKJle8+q}$vG%h(muDm3!y)yoKM?84{)16M|t){b4Q=I)p>%@8+#|kTVI+(^&@zJ-& ztv7aBkMaZv5FkK+009Dn5U|315bD|_K!5;&nhT6K#%ewS-6ueRz~~CR;??oNu~=lW zG6HGY`lj17|IA5GJRWKHmrmBQPt}Q2j}>yC%Eo;H1PBlyK!8B?1w`GeKSz2Z30t5&UAV8qD0s`)}ojM&S zK!5-N0t5&UAV46GKsp}v%fBanyI(yo&aqh`A=%+3cAV7cs0RjXFOf4YbKDCWc z2oNAZfB*pk1PBlq4uRhs5l8;G(nGoW^7!G`E8Rn#BS3%v0RjXF5Xd1Q;GP3X76Jqa z5FkK+009C7>Mro}gX5P!t>_?bzCOPC$$0$l744pG5g;&J0^^Ob;Tn+95FkLHf&!xM z6`T}ZB0zuu0RjXF5FkLH2LeC)UL5kx9@N`el{Z}%pZ$RS+dD&4ng9U;1PBlyP!$0I z_o~clIkjXXK!5-N0t5(D zUtm`CyQqf*2oNAZfB*pk1PBo5n!vfI#2vT9NB=FBS-xv^maFLKL*nQE6;C}?uC!7F z2oNAZfB*pk6%??-y@HdXO9TiIAV7cs0RjXF^itruKgB2B5f_}%%QmL1`+@u8E1!)2 z`MzlTv`juEK!5-N0t5)uPhg(Lo}1RM*LqBV009C72oNAZfB=Ec1vcL?_WejKzC`CW z6{z8?6XKAA;=#WaC?YEX0t5&UAV7dXO#}qoYcf%~Nq|5t1;!g=wd}V}6Cgl<0D(~x zSak8&d%xJ>4KZ);zXN>3)p6)|BW>O;5gU^pyYs0t5&UAV7cs0Rp)N(xUeCMyJ`o^5fB*pk1PBlyK!5-N0t5&UAV7e?(*mOI9t8pf2oNAZ zfB*pk1PBlyK!5-N0t5&UAkYN?0rxH-s)7Ik0t5&UAV7cs0RjXF5FkK+009C72ne`) z1_%%!K!5-N0t5&UAV7cs0RjXF5FkK+KoaFw0t5&UAV7cs0RjXF z5FkK+009C72#mIXfct3AztIyQK!5-N0t5&UAV7cs0RjXF5FkK+K$ipr+`9y+8Uh3e z5FkK+009C72oNAZfB*pk1PBlqZ2Y;*^2nF$aeK!5-N0t5&UAV7cs0RjXF5FkK+KzjiJ_x3LC5+Fc;009C72oNAZ zfB*pk1PBlyK!5;&LIMKrg}7uUK!5-N0t5&UAV7cs0RjXF5FkK+009E+1q9sNySPh$ z009C72oNAZfB*pk1PBlyK%n~qRzeo&=~>*2oNAZfB*pk1PBly zK!5-N0t5&UsH=d0dtGNvzX=c^K!5-N0t5&UAV7cs0RjXF5FkLHQUU_*m6{HnAwYlt z0RjXF5FkK+009C72oNAZfB=EI3JAE@b>{S&009C72oNAZfB*pk1PBlyK!5-N0t6~0 zAmCo9>ChPh1PBlyK!5-N0t5&UAV7cs0RjXF5U8twfO}nMPQM8dAV46Gz<6UUPd0f7 z5FkK+009C72oNAZfB*pkT@ny=?-HbH2oNAZfB*pk1PBlyK!5-N0t5&UAV6TW1q9qj zd;X1{009C72oNAZfB*pk1PBlyK!5-N0tC7wAmH95NYxM^K!5-N0t5&UAV7cs0RjXF z5FkK+z-S8yxR3Vy8$AI61PBlyK!5-N0t5&UAV7cs0RjXFbV)$Ky-Sd)AwYlt0RjXF z5FkK+009C72y{Pj0t5&UAV7cs0RjXF5FkK+009C72(%XvaBuJ8E&&1r2oNAZfB*pk1PBnQ zgTQ!WtPb7OM*;*05Fju@0;29CG}(qqfB*pk1PBlyK!5-N0t5&UAV7cs0Rp`f5OD9E zt(plCAV7cs0RjXF5FkK+009C72oNAZV1xt&+(&4#4V3@^0t5&UAV7cs0RjXF5FkK+ z009C7dM6;@-aA`06Cgl<009C72oNAZfB*pk1PBlyK!Csq2?)54&}17b0RjXF5FkK+ z009C72oNAZfB*pk1PJs_K)}6swrUO@2oNAZfB*pk1PBlyK!5-N0t5&U z=#qe_dzTLMWEUYD8DPXYu85FkK+009C7 z2oNAZfB*pk1PBnQw19wnrKd;d2oNAZfB*pk1PJt5V7xKb=N2_bfB*pk1PBlyKw!88 zMBRsLK8=O|0RjXF5FkK+009C72oNAZfB*pk1O_A^;65OD%@QC$fB*pk1PBlyK!5-N z0t5&UAV7e?a0v*w57&Ge4FLiK2oNAZfB*pk1PBlyK!5-N0t5&QNI<}SK=7I+K!5-N z0t5&UAV7cs0RjXF5FkK+0D<8W5O5!^`7{~=1PBlyK!5-N0t5&UAV7cs0RjXFjHJML zW2_qE7Im-2Jm>`h0t5&UAV7cs0RjXF5FkK+009C72-I9az`f=ZsQUy65FkK+009C7 z2oNAZfB*pk1PBlyPz?bA_iD_8UJxKafB*pk1PBlyK!5-N0t5&UAV7dX%>@M9Yd(Rx zPk;ac0t5&UAV7cs0RjXF5FkK+009Ek5D;*$#ysc+0RjXF5FkK+009C72oNAZfB*pk z1PIhzK)}7`6R7(H2oNAZfB=EP2#hzz2Gg)s2@oJafB*pk1PBlyP;CKG_iE3N-Vq=` zfB*pk1PBlyK!5-N0t5&UAV7dXO#}qoYcf%~Nq_(W0t5&UAV7cs0RjXF5FkK+009Ek z77%c+_WbA_0RjXF5FkK+009C72oNAZfB*pk1PIhbK)}5w6Q!F32oNAZfB*pk1PBly zK!5-N0t5&UAW&@q0rzUpkKPd=K!5-N0t5&UAV7cs0RjXF5FpS^f$_#zH>*<@fffN# z_ZAX22oNAZfB*pk1PBlyK!5-N0t5&UAV7dXaRCAM;$}(^AV7cs0RjXF5FkK+009C7 z2oNAZfB=CO0Ri_G5;q7CAV7cs0RjXF5FkK+009C72oNAZfIx8p0r%o&N)RAGfB*pk z1PBlyK!5-N0t5&UAV7csfffM)_ZAX22oNAZfB*pk1PBlyK!5-N0t5&UAV7dXaRCAM z;$}(^AV7cs0Rs62#v5b#Gs{VU009C72oNAZfB*pk1ZpoJ>R$UPGynnw2oNAZfB*pk z1PBlyK!5-N0t5&UsDglidlhCu9|#a2K!5-N0t5&UAV7cs0RjXF5FkLH_5uR#wVy%* zAV7cs0RjXF5FkK+009C72oNAZfB=Ci2ne`WVHWg(009C72oNAZfB*pk1PBlyK!5-N z0t9L=AmCp6DKr2A1PBlyK!5-N0t5&UAV7csf$j^8H^#bOsTv6o7zF`Q_feQ-V_EfB*pk1PBlyK!5-N0tBipAnIQ2`O!N91PBlyK!5-N0t5&UAV7cs z0RjXF5U7cOfO}0QN;e4*AV7cs0RjXF5FkK+009C72oNAZpxOcg?$w?jy(2(?009C7 z2oNAZfB*pk1PBlyK!5;&ng|HE*JPq}lK=q%1PBlyK!5-N0t5&UAV7cs0RjZ7Eg;}t z?fKC=0t5&UAV7cs0RjXF5FkLH+5+Q^v1)fv?+6eeK%mb8qV9bL*Bk)?1PBlyK!5-N z0t5&UAV7cs0RjXF45xsA`*6;yQ4t_OfB*pk1PBlyK!5-N0t5&UAV7dXp9KWm`wXr* z0t5&UAV7cs0RjXF5FkK+009C72oM-f0Ri{noL8eFK!5-N0t5&UAV7cs0RjXF5FkK+ z0D(RW2)OqdTyq2n5FkK+009C72oNAZfB*pk1PBlyFq{Gc?!!5+MnzzF1;!g=!#g-5 zBS3%v0RjXF5FkK+009C72oR{ffT(-zr_cZh5FkK+009C72oNAZfB*pk1PBlyK%fc& z0`66q1$`hufB*pk1PBlyK!5-N0t5&UAV7csf!YfQxYvFP4S)av0t5&UAV7cs0RjXF z5FkK+009C7svscXUWHlE2Lc2L5FkK+009C72oNAZfB*pk1PBnQy?}sw?WfQH2oNAZ zfB*pk1PBlyK%fo+gMBO_TRGt6<0t5&UAV7cs0RjXF5FkK+ z009C72qXakcaH%90t5&UAV7cs0RjXF5FkK+009C72oUIkfPi}!5LH2d009C72oNAZ zfB*pk1PBlyK!5-N0t5uyJp%*?5FkK+009C72oNAZfB*pk1PBlyK%ff(0`6TvR0RP7 z1PBlyK!5-N0t5&UAV7cs0RjXF5D;+p3=FBjcw=lxM`l1l%h;Il4xG009C72oNAZfB*pk1PBlyK!5-N0`(CPaIeo? z=_vsM1PBlyK!5-N0t5&UAV7cs0RjXFR9Ha3y~2~DYXk@oAV7cs0RjXFjG(}HV{8OR zZO8-&5FkK+0D<8V5Op7(xik_21PBlyK!5-N0t5&UAV7cs0RjXF5E!6VwaIe^e=neq_1PBlyK!5-N0t5&UAV7cs z0RjXF)Kfsfy`J-?*8~Uz`e?|qi+NV5FkK+009C72oNAZfB*pk1PBlyP#XaO z_u5RAjuIe1fB*pk1PBlyK!5-N0t5&UAV7dXl?4Rct2{gUMt}eT0t5&UAV7cs0RjXF z5FkK+009EE5fE^%%~a_q0RjXF5FkK+009C72oNAZfIu68@y1x2yKWI6KwwY;qV9u| z*De791PBlyK!5-N0t5&UAV7cs0RjXF43&U@`%q1%!4M!ofB*pk1PBlyK!5-N0t5&U zAV7e?pacZm2PLmv0t5&UAV7cs0RjXF5FkK+009C72oM-50Ri`+noffuK!5-N0t5&U zAV7cs0RjXF5FkK+0D(aX2)GYQUb_Sc5FkK+009C72oNAZfB*pk1PBlyFjN8p?n5=5 z219@Vfj$U~H^%zViY5pUAV7cs0RjXF5FkK+009E^6cBZ<=e+4P0RjXF5FkK+009C7 z2oNAZfB*pk1PD}2K)}6X6QVl=2oNAZfB*pk1PBlyK!5-N0t5&UAW%;M0rz^&n_d$j zK!5-N0t5&UAV7cs0RjXF5FkK+K*a;wo9AV8pmfT(*3G(`vyAV7cs z0RjXF5FkK+009C72oNAZfIwD2z&$JCng9U;1PBlyK!5-N0t5&UAV7cs0RjXFln@Yb zFM*~A0RjXF5FkK+009C72oNAZfB*pk1PBnw3JADoMO+ggK!5-N0t5&UAV7cs0RjXF z5FkK+0D%$$0`4Wy6d^!>009C72oNAZfB*pk1PBlyK!5-N0$Bk8_pFF(0tC7wFy0vJ zQcbEMK!5-N0t5&UAV7cs0RjXF5Fk(+0a5qbOqGrjAV7cs0RjXF5FkK+009C72oNAZ zfIyW61l+4UJNia|009C72oNAZfB*pk1PBlyK!5-N0<{qkaIei&=_mmL1PBlyK!5-N z0t5&UAV7cs0RjXFR9Qg4y~?wrZv+SsAV7cs0RjXF5FkK+009C72oNAp8vz0L+Dw&> z5+Fc;009C72oNAZfIwda#v5aOX-6Xj2oNAZfWUAHh`JBwyc!h&0t5&UAV7cs0RjXF z5FkK+009C72=rM%z`f7lnj=7f009C72oNAZfB*pk1PBlyK!5;&;S>;XAI^C-Dgp!u z5FkK+009C72oNAZfB*pk1PBo5vw(nmpTRXpfB*pk1PBlyK!5-N0t5&UAV7cs0RqD* zAmBcn^J-KC2oNAZfB*pk1PBlyK!5-N0t5&UAkb$40rx(GYpz9LyfM~t&kX_u2oNAZ zfB*pk1PBlyK!5-N0t5)uQ$WVwaIe^e=neq_1PBlyK!5-N0t5&UAV7cs0RjXF)Kfsfy`J-?*8~Uo%N zM1TMR0t5&UAV7cs0RjXF5FkK+009DZ6sS$WeY`PNM_+v=K!5-N0t5&UAV7cs0RjXF z5FkK+0D(~ws7=&;lqSVk2@oJafB*pk1PBlyK!5-N0t5&UAV8pg0s`*!n=?HoK!5-N z0t5&UAV7cs0RjXF5FkK+0D%e$2)I{pQgn#`0RjXF5FkK+009C72oNAZfB*pk1nMUs z;9kEu(_;bz2oNAZfB*pk1PBlyK!5-N0t5&UsGxv=dj%&&mk1CbK!5-N0tALiV7xIl zRO2xi0t5&UAV7cs0RjZ36A*QuPRC~i2oNAZfB*pk1PBlyK!5-N0t5&UAV46efPi~W zDA@=QAV7cs0RjXF5FkK+009C72oNAZfWUMD0`Al4_>2Gn0t5&UAV7cs0RjXF5FkK+ z009C72;>wHaL)-P8vz0Y2oNAZfB*pk1PBlyK!5-N0t5&Um`*^zeL5YV5gWaz`X-JMF|ifK!5-N0t5&U zAV7cs0RjXF5FkK+z%v2@?$1d0UjhUO5FkK+009C72oNAZfB*pk1PBly(1Cz}dk1)m z5+Fc;009C72oNAZfB*pk1PBlyK!5;&X9NV?pONst1PBlyK!5;&!3z9&@iiV8Y!9?f vfB*pk1PBlyK!5-N0-X!Y6VYh=`_. +It's a pretty basic setup. You should follow the local testing instructions in ``.circleci/config.yml``. diff --git a/_sources/distributed.rst.txt b/_sources/distributed.rst.txt index af1da430..7d054b27 100644 --- a/_sources/distributed.rst.txt +++ b/_sources/distributed.rst.txt @@ -12,20 +12,20 @@ How distributed training works With a single GPU and model, training follows this process: 1. Compute the loss from a minibatch of data (the forward pass of the model) -2. Backpropgate that loss through the model to compute gradients (the backward pass of the model) +2. Backpropagate that loss through the model to compute gradients (the backward pass of the model) 3. Sum the gradients across the minibatch. 4. Run the optimizer by calling the "step()" function. Now assume we have several GPUs, and they each have exactly the same model. -#1 and #2 are `embarassingly parallel `_ and can be distribued to many nodes. +#1 and #2 are `embarrassingly parallel `_ and can be distributed to many nodes. As long as we can sum across nodes to complete #3 (this is known as an 'all-reduce'), then each node can run #4 on the same gradients, and the resulting models will again be identical. This is the premise behind distributed training. Training on a single node ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Using multiple GPUs on a single node is relatively straightforward. When running either the dqn_workflow or the parametric_dqn_workflow, -set the "use_all_avail_gpus" parameters in the input config (the json file) to true. Horizon will detect the number of available GPUs and +Using multiple GPUs on a single node is relatively straightforward. When running either the dqn_workflow or the parametric_dqn_workflow, +set the "use_all_avail_gpus" parameters in the input config (the json file) to true. ReAgent will detect the number of available GPUs and run on all of them without any additional effort. Training on multiple nodes diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt index 0f571b59..228d7948 100644 --- a/_sources/index.rst.txt +++ b/_sources/index.rst.txt @@ -3,7 +3,7 @@ -.. image:: ../logo/horizon_banner.png +.. image:: ../logo/reagent_banner.png :alt: Banner @@ -11,9 +11,8 @@ ReAgent: Applied Reinforcement Learning Platform ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. image:: https://ci.pytorch.org/jenkins/buildStatus/icon?job=horizon-master - :target: https://ci.pytorch.org/jenkins/job/horizon-master/ - :alt: Build Status +.. image:: https://circleci.com/gh/facebookresearch/ReAgent/tree/master.svg?style=svg + :target: https://circleci.com/gh/facebookresearch/ReAgent/tree/master -------------------------------------------------------------------------------------------------------------------------------------------------------------------- @@ -26,7 +25,7 @@ workflows to train popular deep RL algorithms and includes data preprocessing, f counterfactual policy evaluation, and optimized serving. For more detailed information about ReAgent see the white paper here: `Platform `_. -The source code is available here: `Source code `_. +The source code is available here: `Source code `_. The platform was once named "Horizon" but we have adopted the name "ReAgent" recently to emphasize its broader scope in decision making and reasoning. @@ -59,9 +58,15 @@ License ReAgent is released under a BSD license. Find out more about it here: :ref:`license`. -.. image:: ../logo/horizon_logo.png - :alt: Logo - :width: 200px +Citing +~~~~~~ + +@article{gauci2018horizon, + title={Horizon: Facebook's Open Source Applied Reinforcement Learning Platform}, + author={Gauci, Jason and Conti, Edoardo and Liang, Yitao and Virochsiri, Kittipat and Chen, Zhengxing and He, Yuchen and Kaden, Zachary and Narayanan, Vivek and Ye, Xiaohui}, + journal={arXiv preprint arXiv:1811.00260}, + year={2018} +} Table of Contents ~~~~~~~~~~~~~~~~~~~~~ @@ -77,15 +82,19 @@ Table of Contents :caption: Advanced Topics Distributed Training + Continuous Integration .. toctree:: :caption: Package Reference - Workflow + Evaluation + Models + Prediction Preprocessing + Readers Simulators - Thrift Training + Workflow All Modules .. toctree:: diff --git a/_sources/installation.rst.txt b/_sources/installation.rst.txt index 5d84fefc..cad211b4 100644 --- a/_sources/installation.rst.txt +++ b/_sources/installation.rst.txt @@ -3,168 +3,105 @@ Installation ============ -Anaconda -^^^^^^^^ +ReAgent CLI & Python API +^^^^^^^^^^^^^^^^^^^^^^^^ -First, install anaconda from here (make sure to pick the python 3 version): `Website `_. +We have CLI to launch training & Python API to use programmatically, e.g., in your own script or Jupyter Notebook. +To install this component, you will need to have Python 3.7+ installed on your system. +If you don't have that, you can either install it via `pyenv `_ or +`conda `_. To verify that you have the right version, +type the following command on your shell: -Next, we're going to add some channels that we need for certain software: +.. code-block:: bash + + python --version + +Once you make sure you have the right version, you can simply clone this repo and pip install + +.. code-block:: bash + + git clone https://github.com/facebookresearch/ReAgent.git + cd ReAgent + pip install ".[gym]" + + # install nightly torch (change cpu to cu101/102 if fit) + pip install --pre torch torchvision -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + +If you don't want need gym dependencies, you can remove :code:`[gym]` + +To verify your setup please run `tox `_. + +.. code-block:: bash + + pip install tox + tox + +Spark preprocessing JAR +^^^^^^^^^^^^^^^^^^^^^^^ + +If you don't want to rebuild the JAR, you can grab `the pre-built version from CircleCI `_, +under `the artifact section of end_to_end_test `_. + +To build from source, you'll need JDK, Scala, & Maven. We will use `SDKMAN! `_ to install them. + +.. code-block:: bash + + curl -s "https://get.sdkman.io" | bash + source "$HOME/.sdkman/bin/sdkman-init.sh" + sdk version + sdk install java 8.0.252.hs-adpt + sdk install scala + sdk install maven + +If you are testing locally, you can also install Spark + +.. code-block:: bash + + sdk install spark 2.4.6 + +Now, you can build our preprocessing JAR + +.. code-block:: bash + + mvn -f preprocessing/pom.xml clean package + +RASP +^^^^ + +RASP (ReAgent Serving Platform) is a decision-serving library. It also has standlone binary. It depends on libtorch, +which cannot be statically linked at the moment. Therefore, we don't have a pre-built version. + +To build the CLI, you'll need `CMake `_ and the following libraries: + +- Nightly build of `libtorch `_ +- `boost `_ +- `gflags `_ +- `glog `_ +- `eigen `_ + +If you don't have those requirements, one easy way to get them is through `conda`. +We recommend `miniconda `_ for this purpose. +If you want to install those requirements via conda, you can type this in the shell: .. code-block:: - conda config --add channels conda-forge # For ONNX/tensorboardX - conda config --add channels pytorch # For PyTorch + conda install --file rasp_requirements.txt -Clone and enter Horizon repo: +To get libtorch, please go to `pytorch `_. +Please make sure to download the "Preview (Nightly)" since our code is tested with that version. +Below, we assumed you put the extracted file at :code:`$HOME/libtorch`. + +You will also need to make sure to init git submodules .. code-block:: - git clone --recurse-submodules https://github.com/facebookresearch/Horizon.git - cd Horizon/ + git submodule update --force --recursive --init --remote -If you already cloned the repo without submodules, they can be added by running this command inside the repository" +Now, you are ready to build .. code-block:: - git submodule update --init --recursive - -Install dependencies: - -.. code-block:: - - conda install --file requirements.txt - -Set JAVA_HOME to the location of your anaconda install - -.. code-block:: - - export JAVA_HOME="$(dirname $(dirname -- `which conda`))" - - echo $JAVA_HOME # Should see something like "/home/jjg/miniconda3" - -Install Spark (the mv command may need to be done as root): - -.. code-block:: - - wget https://archive.apache.org/dist/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz - tar -xzf spark-2.3.3-bin-hadoop2.7.tgz - sudo mv spark-2.3.3-bin-hadoop2.7 /usr/local/spark - -Add the spark bin directory to your path so your terminal can find ``spark-submit``\ : - -.. code-block:: - - export PATH=$PATH:/usr/local/spark/bin - -Install OpenAI Gym if you plan on following our `tutorial `_\ : - -.. code-block:: - - pip install "gym[classic_control,box2d,atari]" - -Download libtorch from https://pytorch.org/get-started/locally/ and extract it to $HOME/libtorch - -As of pytorch 1.3, libtorch is broken on OS/X. To fix (mac only): - -.. code-block:: - - cp ~/miniconda3/lib/libiomp5.dylib $HOME/libtorch/lib/ - -And now, you are ready to install Horizon itself. To install the serving platform: - -.. code-block:: - - mkdir serving/build - cd serving/build - cmake -DCMAKE_PREFIX_PATH=$HOME/libtorch .. - - -Next we must package the models. We use "pip install -e" on the root directory of the repository to create an ephemral package. This means that you can make changes to Horizon and they will be reflected in the package immediately. - -.. code-block:: - - pip install -e . - -At this point, you should be able to run all unit tests: - -.. code-block:: - - pytest - -Docker -^^^^^^ - -We have included a Dockerfile for the CPU-only build and CUDA build under the docker directory. -The CUDA build will need `nvidia-docker `_ to run. - -To build, clone repository and cd into the respective directory: - -.. code-block:: - - git clone https://github.com/facebookresearch/Horizon.git - cd Horizon/ - -On macOS you will need to increase the default memory allocation as the default of 2G is not enough. You can do this by clicking the whale icon in the task bar. We recommend using at least 8G of memory. - -On macOS, you can then build the image: - -.. code-block:: - - docker build -f docker/cpu.Dockerfile -t horizon:dev . - -On Linux you can build the image with specific memory allocations from command line: - -.. code-block:: - - docker build -f docker/cpu.Dockerfile -t horizon:dev --memory=8g --memory-swap=8g . - -To build with CUDA support, use the corresponding dockerfile: - -.. code-block:: - - docker build -f docker/cuda.Dockerfile -t horizon:dev . - -Once the Docker image is built you can start an interactive shell in the container and run the unit tests. To have the ability to edit files locally and have changes be available in the Docker container, mount the local Horizon repo as a volume using the ``-v`` flag. We also add ``-p`` for port mapping so we can view Tensorboard visualizations locally. - -.. code-block:: - - docker run -v $PWD:/home/Horizon -p 0.0.0.0:6006:6006 -it horizon:dev - -To run with GPU, include ``--runtime=nvidia`` after installing `nvidia-docker `_. - -.. code-block:: - - docker run --runtime=nvidia -v $PWD:/home/Horizon -p 0.0.0.0:6006:6006 -it horizon:dev - -If you have SELinux (Fedora, Redhat, etc.) you will have to start docker with the following command (notice the ``:Z`` at the end of path): - -.. code-block:: - - docker run -v $PWD:/home/Horizon:Z -p 0.0.0.0:6006:6006 -it horizon:dev - -To run with GPU, include ``--runtime=nvidia`` after installing `nvidia-docker `_. - -.. code-block:: - - docker run --runtime=nvidia -v $PWD:/home/Horizon:Z -p 0.0.0.0:6006:6006 -it horizon:dev - -Depending on where your local Horizon copy is, you may need to white list your shared path via Docker -> Preferences... -> File Sharing. - -Once inside the container, run the setup file: - -.. code-block:: - - cd Horizon - ./scripts/setup.sh - -Now you can run all the tests: - -.. code-block:: - - python setup.py test - -or try running one specific test: - -.. code-block:: - - python setup.py test -s ml.rl.test.constant_reward.test_constant_reward.TestConstantReward.test_trainer_maxq + mkdir -p serving/build + cd serving/build + cmake -DCMAKE_PREFIX_PATH=$HOME/libtorch -DCMAKE_CXX_STANDARD=17 .. + make diff --git a/_sources/license.rst.txt b/_sources/license.rst.txt index 9136877b..ade8cacc 100644 --- a/_sources/license.rst.txt +++ b/_sources/license.rst.txt @@ -5,7 +5,7 @@ License BSD License -For Horizon software +For ReAgent software Copyright (c) 2017-present, Facebook, Inc. All rights reserved. diff --git a/_sources/rasp_tutorial.rst.txt b/_sources/rasp_tutorial.rst.txt index 2c87d75e..39ed2850 100644 --- a/_sources/rasp_tutorial.rst.txt +++ b/_sources/rasp_tutorial.rst.txt @@ -43,7 +43,7 @@ we will always show bacon (with it’s score of 1.1) and never show ribs (with a score of 0.9). This means we will never know the true performance of recommending ribs and can’t improve our system in the future. This is known as the cold-start or explore-exploit problem -(TODO: Citations). +( https://arxiv.org/abs/1812.00116 ). To avoid that problem, we will use the SoftmaxRanker, which will show bacon 52% of the time and ribs 48% of the time. The SoftmaxRanker @@ -52,11 +52,11 @@ operator is based on the softmax function: :: >>> import numpy as np - >>> + >>> >>> def softmax(x): ... e_x = np.exp(x - np.max(x)) ... return e_x / e_x.sum() - ... + ... >>> print(softmax([1.1, 1.0])) [0.52497919 0.47502081] @@ -121,11 +121,6 @@ We will be using the built-in web service directly for this tutorial. The simulator code can be found at: serving/examples/ecommerce/customer_simulator.py -Here is our RP config file. This tells RP where to find decision plans -and models (coming later): - -(RP config) - Makin’ bacon ------------ @@ -133,7 +128,7 @@ In one terminal window, start the RP server: :: - ➜ ReAgent git:(master) ✗ ./serving/build/RaspCli --logtostderr + ➜ ./serving/build/RaspCli --logtostderr I1014 17:23:19.736086 457250240 DiskConfigProvider.cpp:10] READING CONFIGS FROM serving/examples/ecommerce/plans I1014 17:23:19.738142 457250240 DiskConfigProvider.cpp:42] GOT CONFIG multi_armed_bandit.json AT serving/examples/ecommerce/plans/multi_armed_bandit.json I1014 17:23:19.738286 457250240 DiskConfigProvider.cpp:46] Registered decision config: multi_armed_bandit.json @@ -144,12 +139,11 @@ In one terminal window, start the RP server: I1014 17:23:19.739843 131715072 Server.cpp:58] STARTING SERVER Then in another, run our simulator. The simulator will spawn many -threads and call RASP 10,000 times (this will take a few minutes to -complete): +threads and call RASP 1,000 times: :: - ➜ ReAgent git:(master) ✗ python serving/examples/ecommerce/customer_simulator.py heuristic.json + ➜ python serving/examples/ecommerce/customer_simulator.py heuristic.json 0 200 100 @@ -164,21 +158,20 @@ complete): Action Distribution: {'Ribs': 471, 'Bacon': 529} As expected, we recommend Bacon 52% of the time and Ribs 48% of the -time. We get an average reward (in this case, average # of clicks) of -0.3555. +time. We get an average reward (in this case, average # of clicks) of about 0.36. This is our baseline performance, but can we do better? From the log, we can see that more bacon recommendations were clicked on: :: - ➜ ReAgent git:(master) ✗ cat /tmp/rasp_logging/log.txt | grep '"name":"Ribs"}]' | grep '"reward":0.0' | wc -l + ➜ cat /tmp/rasp_logging/log.txt | grep '"name":"Ribs"}]' | grep '"reward":0.0' | wc -l 390 # Ribs not clicked - ➜ ReAgent git:(master) ✗ cat /tmp/rasp_logging/log.txt | grep '"name":"Ribs"}]' | grep '"reward":1.0' | wc -l + ➜ cat /tmp/rasp_logging/log.txt | grep '"name":"Ribs"}]' | grep '"reward":1.0' | wc -l 88 # Ribs clicked - ➜ ReAgent git:(master) ✗ cat /tmp/rasp_logging/log.txt | grep '"name":"Bacon"}]' | grep '"reward":1.0' | wc -l + ➜ cat /tmp/rasp_logging/log.txt | grep '"name":"Bacon"}]' | grep '"reward":1.0' | wc -l 266 # Bacon clicked - ➜ ReAgent git:(master) ✗ cat /tmp/rasp_logging/log.txt | grep '"name":"Bacon"}]' | grep '"reward":0.0' | wc -l + ➜ cat /tmp/rasp_logging/log.txt | grep '"name":"Bacon"}]' | grep '"reward":0.0' | wc -l 253 # Bacon not clicked This makes sense since, from our simulator definition, most people @@ -197,7 +190,7 @@ Generates this plan: :: - ➜ ReAgent git:(master) ✗ cat serving/examples/ecommerce/plans/multi_armed_bandit.json + ➜ cat serving/examples/ecommerce/plans/multi_armed_bandit.json { "operators": [ { @@ -232,7 +225,7 @@ Running with this new plan gives: :: - ➜ ReAgent git:(master) ✗ python serving/examples/ecommerce/customer_simulator.py multi_armed_bandit.json + ➜ python serving/examples/ecommerce/customer_simulator.py multi_armed_bandit.json 0 200 100 @@ -252,7 +245,7 @@ again: :: - ➜ ReAgent git:(master) ✗ python serving/examples/ecommerce/customer_simulator.py multi_armed_bandit.json + ➜ python serving/examples/ecommerce/customer_simulator.py multi_armed_bandit.json 0 200 100 @@ -291,8 +284,8 @@ instance of RaspCli: … I1014 17:45:36.613893 6602752 Server.cpp:58] STARTING SERVER ^C - ➜ ReAgent git:(master) ✗ rm /tmp/rasp_logging/log.txt - ➜ ReAgent git:(master) ✗ ./serving/build/RaspCli --logtostderr + ➜ rm /tmp/rasp_logging/log.txt + ➜ ./serving/build/RaspCli --logtostderr I1014 17:48:49.674149 144418240 DiskConfigProvider.cpp:10] READING CONFIGS FROM serving/examples/ecommerce/plans I1014 17:48:49.678155 144418240 DiskConfigProvider.cpp:42] GOT CONFIG multi_armed_bandit.json AT serving/examples/ecommerce/plans/multi_armed_bandit.json I1014 17:48:49.679606 144418240 DiskConfigProvider.cpp:46] Registered decision config: multi_armed_bandit.json @@ -308,14 +301,14 @@ Now let’s run the heuristic model a few times to generate enough data :: - ➜ ReAgent git:(master) ✗ for run in {1..10}; do python serving/examples/ecommerce/customer_simulator.py heuristic.json; done + ➜ for run in {1..10}; do python serving/examples/ecommerce/customer_simulator.py heuristic.json; done 0 200 ... 900 Average reward: 0.36 Action Distribution: {'Bacon': 516, 'Ribs': 484} - ➜ ReAgent git:(master) ✗ wc -l /tmp/rasp_logging/log.txt + ➜ wc -l /tmp/rasp_logging/log.txt 10000 /tmp/rasp_logging/log.txt RASP’s logging format and the ReAgent models’ input format is slightly @@ -323,70 +316,43 @@ different. Fortunately, there’s a tool to convert from one to the other: :: - ➜ ReAgent git:(master) ✗ python serving/scripts/rasp_to_model.py /tmp/rasp_logging/log.txt ecom_cb_input_data/input.json - ➜ ReAgent git:(master) ✗ wc -l ecom_cb_input_data/input.json - 10000 ecom_cb_input_data/input.json + ➜ python serving/scripts/rasp_to_model.py /tmp/rasp_logging/log.txt /tmp/input_df.pkl + ... + INFO:__main__: ds mdp_id sequence_number state_features action reward action_probability possible_actions metrics + 0 2019-01-01 1287515757457242569 0 {0: 0.0, 1: 1.0} Ribs 0.0 0.475021 [Bacon, Ribs] {'reward': 0.0} + 1 2019-01-01 -1441171268272508658 0 {0: 0.0, 1: 1.0} Ribs 0.0 0.475021 [Bacon, Ribs] {'reward': 0.0} + 2 2019-01-01 -267723109738500267 0 {0: 0.0, 1: 1.0} Bacon 1.0 0.524979 [Bacon, Ribs] {'reward': 1.0} + 3 2019-01-01 7619952535038766490 0 {0: 0.0, 1: 1.0} Ribs 0.0 0.475021 [Bacon, Ribs] {'reward': 0.0} + 4 2019-01-01 -2393212434904546228 0 {0: 0.0, 1: 1.0} Bacon 0.0 0.524979 [Bacon, Ribs] {'reward': 0.0} + Since we are using the contextual bandit or RL model, we need to build a timeline: :: - rm -Rf spark-warehouse derby.log metastore_db preprocessing/spark-warehouse preprocessing/metastore_db preprocessing/derby.log ; /usr/local/spark/bin/spark-submit \ - --class com.facebook.spark.rl.Preprocessor preprocessing/target/rl-preprocessing-1.1.jar \ - "`cat serving/examples/ecommerce/training/timeline.json`" - ... - 2019-10-14 19:04:18 INFO ShutdownHookManager:54 - Shutdown hook called - 2019-10-14 19:04:18 INFO ShutdownHookManager:54 - Deleting directory /private/var/folders/jm/snmq7xfn7llc1tpnjgn7889h6l6pkw/T/spark-2b6a4171-cb60-4d5e-8052-87620a0677a2 - 2019-10-14 19:04:18 INFO ShutdownHookManager:54 - Deleting directory /private/var/folders/jm/snmq7xfn7llc1tpnjgn7889h6l6pkw/T/spark-927dae4a-6613-4a28-9d88-4d43a03d1cf3 - ➜ ReAgent git:(master) ✗ + # Set the config + ➜ export CONFIG=serving/examples/ecommerce/training/contextual_bandit.yaml -The spark job creates a directory full of files, so we must merge into -one file for training & evaluation: + # First clean up derby database from last run + ➜ rm -Rf spark-warehouse derby.log metastore_db preprocessing/spark-warehouse preprocessing/metastore_db preprocessing/derby.log + + # Run timeline operator + ➜ ./reagent/workflow/cli.py run reagent.workflow.gym_batch_rl.timeline_operator "$CONFIG" + +The `Click `_ command submits a Spark job that uploads the timeline table to Hive. +Now we can train the contextual bandit. :: - ➜ ReAgent git:(master) ✗ mkdir -p training_data - ➜ ReAgent git:(master) ✗ cat ecom_cb_training/part* > training_data/train.json - ➜ ReAgent git:(master) ✗ cat ecom_cb_eval/part* > training_data/eval.json + ➜ ./reagent/workflow/cli.py run reagent.workflow.training.identify_and_train_network "$CONFIG" + ... + I0524 112136.208 model_manager.py:213] Saved torchscript model to model_1590344496.torchscript -Now we run our normalization. Any time we use a deep neural network, we -need normalization to prevent some large features from drowning others. - -:: - - ➜ ReAgent git:(master) ✗ python ml/rl/workflow/create_normalization_metadata.py -p serving/examples/ecommerce/training/cb_train.json - - WARNING:root:This caffe2 python run does not have GPU support. Will run in CPU only mode. - INFO:ml.rl.preprocessing.normalization:Got feature: 0 - INFO:ml.rl.preprocessing.normalization:Feature 0 normalization: NormalizationParameters(feature_type='BINARY', boxcox_lambda=None, boxcox_shift=0.0, mean=0.0, stddev=1.0, possible_values=None, quantiles=None, min_value=0.0, max_value=1.0) - INFO:ml.rl.preprocessing.normalization:Got feature: 1 - INFO:ml.rl.preprocessing.normalization:Feature 1 normalization: NormalizationParameters(feature_type='BINARY', boxcox_lambda=None, boxcox_shift=0.0, mean=0.0, stddev=1.0, possible_values=None, quantiles=None, min_value=1.0, max_value=1.0) - INFO:__main__:`state_features` normalization metadata written to training_data/state_features_norm.json - -Now we can train our contextual bandit: - -:: - - ➜ ReAgent git:(master) ✗ rm -Rf "outputs/*" ; python ml/rl/workflow/dqn_workflow.py -p serving/examples/ecommerce/training/cb_train.json - INFO:ml.rl.json_serialize:TYPE: - INFO:ml.rl.json_serialize:{'gamma': 0.0, 'target_update_rate': 1.0, 'maxq_learning': True, 'epsilon': 0.2, 'temperature': 0.35, 'softmax_policy': 0} - ... - INFO:ml.rl.workflow.page_handler:CPE evaluation took 0.26366519927978516 seconds. - INFO:ml.rl.workflow.base_workflow:Training finished. Processed ~6555 examples / s. - INFO:ml.rl.preprocessing.preprocessor:CUDA availability: False - INFO:ml.rl.preprocessing.preprocessor:NOT Using GPU: GPU not requested or not available. - /Users/jjg/github/Horizon/ml/rl/preprocessing/preprocessor.py:546: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - elif max_value.gt(MAX_FEATURE_VALUE): - /Users/jjg/github/Horizon/ml/rl/preprocessing/preprocessor.py:552: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - elif min_value.lt(MIN_FEATURE_VALUE): - INFO:__main__:Saving PyTorch trainer to outputs/trainer_1571105504.pt - INFO:ml.rl.workflow.base_workflow:Saving TorchScript predictor to outputs/model_1571105504.torchscript - -At this point, we have a model in ``outputs/model_*.torchscript``. We -are going to combine this scoring model with an e-greedy ranker. The -e-greedy ranker chooses the best actions most of the time, but sometimes -chooses random actions to explore: +At this point, we have a model saved at ``model_*.torchscript``. We +are going to combine this scoring model with an Softmax ranker. The +ranker chooses the best actions most of the time, but rarely +chooses other actions to explore: :: @@ -439,14 +405,14 @@ put the model there so we can find it: :: - ➜ ReAgent git:(master) ✗ mkdir -p /tmp/0 - ➜ ReAgent git:(master) ✗ cp outputs/model_*.torchscript /tmp/0/0 + ➜ mkdir -p /tmp/0 + ➜ cp model_*.torchscript /tmp/0/0 Let’s run with our model: :: - ➜ ReAgent git:(master) ✗ python serving/examples/ecommerce/customer_simulator.py contextual_bandit.json + ➜ python serving/examples/ecommerce/customer_simulator.py contextual_bandit.json 0 200 100 @@ -472,4 +438,3 @@ All of the decisions made so far have been pointwise: we don’t consider repeat visitors. ReAgent can also optimize for long-term value in sequential decisions using reinforcement learning, but that is out of the scope of this starting tutorial. - diff --git a/_sources/usage.rst.txt b/_sources/usage.rst.txt index 8f03e488..edc69569 100644 --- a/_sources/usage.rst.txt +++ b/_sources/usage.rst.txt @@ -3,25 +3,61 @@ Usage ===== -Horizon is designed for large-scale, distributed recommendation/optimization tasks where we don't +ReAgent is designed for large-scale, distributed recommendation/optimization tasks where we don't have access to a simulator. In this environment, it's typically better to train offline on batches of data, and release new policies slowly over time. Because the policy updates slowly and in batches, we use *off-policy* algorithms. To test a new policy without deploying it, we rely on *counter-factual policy evaluation (CPE)*\ , a set of techniques for estimating a policy based on the actions of another policy. -Before we get started using Horizon as it is intended, let's begin with a traditional RL setup with a simulator where we can trivially evaluate new policies: +Quick Start +----------- + +We have set up `Click `_ commands to run our RL workflow. The basic usage pattern is + +.. code-block:: + + ./reagent/workflow/cli.py run + + +To train a model online with OpenAI Gym, simply run the Click command: + +.. code-block:: + + # set the config + export CONFIG=reagent/gym/tests/configs/cartpole/discrete_dqn_cartpole_online.yaml + # train and evaluate model on gym environment + ./reagent/workflow/cli.py run reagent.gym.tests.test_gym.run_test $CONFIG + + +To train a batch RL model, run the following commands: + +.. code-block:: + + # set the config + export CONFIG=reagent/workflow/sample_configs/discrete_dqn_cartpole_offline.yaml + # gather some random transitions (can replace with your own) + ./reagent/workflow/cli.py run reagent.workflow.gym_batch_rl.offline_gym $CONFIG + # convert data to timeline format + ./reagent/workflow/cli.py run reagent.workflow.gym_batch_rl.timeline_operator $CONFIG + # train model based on timeline data + ./reagent/workflow/cli.py run reagent.workflow.training.identify_and_train_network $CONFIG + # evaluate the model + ./reagent/workflow/cli.py run reagent.workflow.gym_batch_rl.evaluate_gym "$CONFIG" + + +Now we will describe how the above commands work, starting with a traditional RL setup with a simulator where we can trivially evaluate new policies: 1 - On-Policy RL Training ------------------------- -Open AI Gym is a set of environments: simulators that can run policies for a given task and generate rewards. If a simulator is accessible, on-policy training (where the latest version of the policy makes new decisions in real-time) can give better results. To train a model on OpenAI Gym, simply run: +OpenAI Gym is a set of environments: simulators that can run policies for a given task and generate rewards. If a simulator is accessible, on-policy training (where the latest version of the policy makes new decisions in real-time) can give better results. We have a suite of benchmarks on OpenAI Gym, which is listed in ``reagent/gym/tests/test_gym.py``'s ``GYM_TESTS``. To train a model on OpenAI Gym, simply run the Click command: .. code-block:: - python ml/rl/test/gym/run_gym.py -p ml/rl/test/gym/discrete_dqn_cartpole_v0.json + ./reagent/workflow/cli.py run reagent.gym.tests.test_gym.run_test reagent/gym/tests/configs/cartpole/discrete_dqn_cartpole_online.yaml -Configs for different environments and algorithms can be found in ``ml/rl/test/gym/``. +Configs for different environments and algorithms can be found in ``reagent/gym/tests/configs//__online.yaml``. While this is typically the set up for people conducting RL research, it isn't always practical to deploy on-policy RL for several reasons: @@ -31,52 +67,47 @@ While this is typically the set up for people conducting RL research, it isn't a #. We want to evaluate the behavior of the policy offline and then keep the policy constant afterwards to reduce the risk that the policy will degrade at odd hours. #. We are building on top of traditional recommender systems that typically rely on a fixed, stochastic policy. -For these reasons, Horizon is designed to support batch, off-policy RL. Let's now walk though how to train a model with Horizon: +For these reasons, ReAgent is designed to support batch, off-policy RL. Let's now walk though how to train a model with ReAgent: 2- Offline RL Training (Batch RL) --------------------------------- -The main use case of Horizon is to train RL models in the **batch** setting. In batch reinforcement learning the data collection and policy learning steps are decoupled. Specifically, we try to learn the best possible policy given the input data. In batch RL, being able to handle thousands of varying feature types and distributions and algorithm performance estimates before deployment are of key importance. +The main use case of ReAgent is to train RL models in the **batch** setting. In batch reinforcement learning the data collection and policy learning steps are decoupled. Specifically, we try to learn the best possible policy given the input data. In batch RL, being able to handle thousands of varying feature types and distributions and algorithm performance estimates before deployment are of key importance. + +In this example, we will train a DQN model on Offline ``CartPole-v0`` data, where Click command config should be set to + +.. code-block:: + + export CONFIG=reagent/workflow/sample_configs/discrete_dqn_cartpole_offline.yaml + + +We now proceed to give pseudo-code to sketch out the main ideas of our batch RL workflow. -In this example, we will train a DQN model on Offline ``Cartpole-v0`` data: Step 1 - Create training data ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -First we need to generate the data required to train our RL models. For this example we generate data from the ``Cartpole-v0`` environment in OpenAI Gym. In practice, end users would generate a dataset in a similar format from their production system. +We first generate data from a random policy (chooses random actions) run on the ``CartPole-v0`` environment. +In particular, the following Click command runs 150 episodes of ``CartPole-v0`` (max steps of 200) and stored the pickled dataframe in ``/tmp/tmp_pickle.pkl``, which you may inspect via ``pd_df = pd.read_pickle(pkl_path)``. .. code-block:: - mkdir cartpole_discrete + ./reagent/workflow/cli.py run reagent.workflow.gym_batch_rl.offline_gym $CONFIG - python ml/rl/test/gym/run_gym.py -p ml/rl/test/gym/discrete_dqn_cartpole_v0.json -f cartpole_discrete/training_data.json - -Let's look at one row of data to see the expected input format: +The command essentially performs the following pseudo-code: .. code-block:: - cat cartpole_discrete/training_data.json | head -n1 | python -m json.tool + dataset = ReplayBuffer() + for epoch in range(num_episodes_for_data_batch): + run_episode & store transitions - { - "mdp_id": "10", - "sequence_number": 0, - "state_features": { - "2": 0.0021880944, - "1": -0.015781501, - "0": -0.031933542, - "3": 0.04611974 - }, - "action": "0", - "reward": 1.0, - "possible_actions": [ - "0", - "1" - ], - "action_probability": 0.9, - "ds": "2018-06-25" - } + df = dataset.to_pandas_df() + df.to_pickle(pkl_path) -The input data is a flat file containing a JSON object per-line separated by newlines (the first line is pretty-printed here for readability). This is human-readable, but not the most efficient way to store tabular data. Other ways to store input data is parquet, CSV, or any other format that can be read by Apache Spark. All of these formats are fine, as long as the following schema is maintained: +In practice, end users would generate a dataset in a similar format from their production system. For this example, the data is stored as a pickled Pandas dataframe. + +This is human-readable, but not the most efficient way to store tabular data. Other ways to store input data are parquet, CSV, or any other format that can be read by Apache Spark. All of these formats are fine, as long as the following schema is maintained: .. list-table:: :header-rows: 1 @@ -110,16 +141,14 @@ The input data is a flat file containing a JSON object per-line separated by new - A unique ID for this dataset. -Note that JSON does not support integer keys in objects so in our JSON format we replace the ``map`` with ``map``\ , but even in this case the keys must be strings of integers. +Once you have data on this format, you can move on to Step 2. -Once you have data on this format (or you have generated data using our gym script) you can move on to step 2: - -Step 2 - Convert the data to the ``timeline`` format +Step 2 - Convert the data to the ``Timeline`` format ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Models are trained on consecutive pairs of state/action tuples. To assist in creating this table, we have an ``RLTimelineOperator`` spark operator. Let's build and run the timeline operator on the data: -First, we need to build the Spark library that will execute the timeline. Apache Spark is a platform for doing massively-parallel processing. Although we are running this on a single file, Spark is designed to work on thousands of files distribued across many machines. Explaining HDFS, Hive, and Spark are beyond the scope of this tutorial, but for large datasets it's important to understand these concepts and that it's possible to run Horizon in a distributed environment by simply changing the location of the input from a file to an HDFS folder. +First, we need to build the Spark library that will execute the timeline. Apache Spark is a platform for doing massively-parallel processing. Although we are running this on a single file, Spark is designed to work on thousands of files distribued across many machines. Explaining HDFS, Hive, and Spark are beyond the scope of this tutorial, but for large datasets it's important to understand these concepts and that it's possible to run ReAgent in a distributed environment by simply changing the location of the input from a file to an HDFS folder. .. code-block:: @@ -133,49 +162,56 @@ When running spark locally, spark creates a fake "cluster" where it stores all o # Clear last run's spark data (in case of interruption) rm -Rf spark-warehouse derby.log metastore_db preprocessing/spark-warehouse preprocessing/metastore_db preprocessing/derby.log -Now that we are ready, let's run our spark job on our local machine. This will produce a massive amount of logging (because we are running many systems that typically are distributed across many nodes) and there will be some exception stack traces printed because we are running in a psuedo-distributed mode. Generally this is fine as long as the output data is generated: +Now that we are ready, let's run our spark job on our local machine. This will produce a massive amount of logging (because we are running many systems that typically are distributed across many nodes) and there will be some exception stack traces printed because we are running in a psuedo-distributed mode. Generally this is fine as long as the output data is generated. To do so, run the following Click command: .. code-block:: - # Run timelime on pre-timeline data - /usr/local/spark/bin/spark-submit \ - --class com.facebook.spark.rl.Preprocessor preprocessing/target/rl-preprocessing-1.1.jar \ - "`cat ml/rl/workflow/sample_configs/discrete_action/timeline.json`" + ./reagent/workflow/cli.py run reagent.workflow.gym_batch_rl.timeline_operator $CONFIG - # Look at the first row of training & eval - head -n1 cartpole_discrete_training/part* - head -n1 cartpole_discrete_eval/part* - -There are many output files. The reason for this is that Spark expects many input & output files: otherwise it wouldn't be able to efficiently run on many machines and output data in parallel. For this tutorial, we will merge all of this data into a single file, but in a production use-case we would be streaming data from HDFS during training. +The command essentially performs the following pseudo-code: .. code-block:: - # Merge output data to single file - mkdir training_data - cat cartpole_discrete_training/part* > training_data/cartpole_discrete_timeline.json - cat cartpole_discrete_eval/part* > training_data/cartpole_discrete_timeline_eval.json + # load pandas dataframe + pd_df = pd.read_pickle(pkl_path) - # Remove the output data folder - rm -Rf cartpole_discrete_training cartpole_discrete_eval + # convert to Spark dataframe + spark = get_spark_session() + df = spark.createDataFrame(pd_df) -Now that all of our data has been grouped into consecutive pairs, we can run the normalization pipeline. + # run timelime operator + json_params = make_input_to_timeline_operator() + spark._jvm.com.facebook.spark.rl.Timeline.main(json_params) -Step 3 - Create the normalization parameters + +Now that our data is a Spark table in Hive storage, we're ready to run the training workflow (Steps 3-4). These steps are altogether accomplished with the following Click command: + +.. code-block:: + + ./reagent/workflow/cli.py run reagent.workflow.training.identify_and_train_network $CONFIG + + +We now proceed to describing this command and present some pseudo-code. + + +Step 3 - Determine normalization parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Data from production systems is often sparse, noisy and arbitrarily distributed. Literature has shown that neural networks learn faster and better when operating on batches of features that are normally distributed. Horizon includes a workflow that automatically analyzes the training dataset and determines the best transformation function and corresponding normalization parameters for each feature. We can run this workflow on the post timeline data: +Data from production systems is often sparse, noisy and arbitrarily distributed. Literature has shown that neural networks learn faster and better when operating on batches of features that are normally distributed. ReAgent includes a workflow that automatically analyzes the training dataset and determines the best transformation function and corresponding normalization parameters for each feature. We do this via ``ModelManager.run_feature_identification``, where ``input_table_spec`` points to a Spark table with the timeline data. .. code-block:: - python ml/rl/workflow/create_normalization_metadata.py -p ml/rl/workflow/sample_configs/discrete_action/dqn_example.json + model: ModelManager__Union + manager = model.value + manager.run_feature_identification(input_table_spec) -Now we can look at the normalization file. It's a JSON file where each key is a feature id and each value is a string-encoded JSON object describing the normalization: + +The normalization is a Python dictionary where each key is a feature id and each value is NormalizationData. +An example of this, in JSON format, is .. code-block:: - cat training_data/state_features_norm.json | python -m json.tool - { "0": "{\"feature_type\":\"CONTINUOUS\",\"mean\":0.5675003528594971,\"stddev\":1.0,\"min_value\":-0.1467551738023758,\"max_value\":2.1779561042785645}", "1": "{\"feature_type\":\"CONTINUOUS\",\"mean\":0.42259514331817627,\"stddev\":1.0,\"min_value\":-1.3586808443069458,\"max_value\":1.8529225587844849}", @@ -183,28 +219,76 @@ Now we can look at the normalization file. It's a JSON file where each key is a "3": "{\"feature_type\":\"CONTINUOUS\",\"mean\":0.02947876788675785,\"stddev\":1.0,\"min_value\":-2.194336175918579,\"max_value\":2.164193868637085}" } +NB: ``reagent/workflow/training.py`` is what the pseudo-code in Steps 3 and 4 are trying to depict. Models should subclass ``ModelManager`` and implement all abstract methods (including ``run_feature_identification`` and ``query_data``) to be added to our registry of models. + Step 4 - Train model ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +To train the model, we first save our Spark table to Parquet format, and use `Petastorm `_'s PyTorch DataLoader, which can efficiently read Parquet formatted data. We do this via ``ModelManager.query_data``, which each ``ModelManager`` in our registry of models must implement. In this step, we also process the rewards, i.e. computing multi-step rewards or computing the reward from ``metrics`` columns directly. + +.. code-block:: + train_dataset = manager.query_data( + input_table_spec=input_table_spec, # description of Spark table + sample_range=train_sample_range, # what percentage of data to use for training + reward_options=reward_options, # config to calculate rewards + ) + # train_dataset now points to a Parquet + Now we are ready to train a model by running: .. code-block:: - # Store model outputs here - mkdir outputs + # make preprocessor from the normalization parameters of Step 3 + batch_preprocessor = manager.build_batch_preprocessor() - python ml/rl/workflow/dqn_workflow.py -p ml/rl/workflow/sample_configs/discrete_action/dqn_example.json + # read preprocessed data + data_reader = petastorm.make_batch_reader(train_dataset.parquet_url) + with DataLoader(data_reader, batch_preprocessor) as dataloader: + for batch in dataloader: + trainer.train(batch) -Note that, even in the OpenAI Gym case, we aren't running the gym at this step. We are taking a batch of data that we generated previously and training by looping over that data and interatively learning a better policy than the policy that generated the data. + # Store model outputs + torchscript_output_path = f"model_{round(time.time())}.torchscript" + serving_module = manager.build_serving_module() + torch.jit.save(serving_module, torchscript_output_path) + + # store for later use + training_output.output_path = torchscript_output_path + +Note that the model is trained purely on the randomly generated data we collected in Step 1. +We are taking a batch of data that we generated previously and training by looping over that data and interatively learning a better policy than the policy that generated the data. +Effectively, this is learning to perform a task by observing completely random transitions from an environment! While doing so, we are not even building a dynamics model of the environment. + +NB: We can do the same for the ``eval_dataset`` if we want to perform CPE during training as a diagnosis tool. Step 5 - Evaluate the Model ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Now that we have trained a new policy on the offline ``Cartpole-v0`` data, we can try it out to see how it does: +Now that we have trained a new policy on the offline ``CartPole-v0`` data, we can try it out to see how it does: .. code-block:: - python ml/rl/test/workflow/eval_cartpole.py -m outputs/predictor* + ./reagent/workflow/cli.py run reagent.workflow.gym_batch_rl.evaluate_gym $CONFIG + +which performs the following pseudo-code + +.. code-block:: + + # load our previous serving module + jit_model = torch.jit.load(saved_serving_module) + + # wrap around module to fit our gymrunner interface + policy = create_predictor_policy_from_model(env, jit_model) + agent = Agent.create_for_env_with_serving_policy(env, policy=policy) + + # run Agent on environment, and record rewards + rewards = evaluate_for_n_episodes( + n=num_eval_episodes, env=env, agent=agent, max_steps=max_steps + ) + + +Even on completely random data, DQN can learn a policy that obtains scores close to the maximum possible score of 200! + Step 6 - Visualize Results via Tensorboard ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/api/ml.html b/api/ml.html index b0b60d43..53ca62cc 100644 --- a/api/ml.html +++ b/api/ml.html @@ -8,7 +8,7 @@ - ml package — Horizon 1.0 documentation + ml package — ReAgent 1.0 documentation @@ -34,9 +34,7 @@ - - - + @@ -50,7 +48,7 @@ - Horizon + ReAgent @@ -89,25 +87,19 @@

Advanced Topics

Package Reference

-