From 520ec28f82f17c9d48eb871003fe86935f73470f Mon Sep 17 00:00:00 2001 From: Mike Wattsm Date: Wed, 20 Nov 2019 21:20:55 +0300 Subject: [PATCH] reinforce without correction added for testing --- docs/source/data.rst | 3 + docs/source/examples/your_data.rst | 3 +- ...of REINFORCE inside recnn (optional).ipynb | 159 ++++++++++++++---- .../1. Basic Reinforce with RecNN.ipynb | 109 ++++++++---- recnn/data/dataset_functions.py | 25 +-- recnn/nn/algo.py | 4 +- recnn/nn/models.py | 12 +- recnn/nn/update/reinforce.py | 33 ++-- 8 files changed, 253 insertions(+), 95 deletions(-) diff --git a/docs/source/data.rst b/docs/source/data.rst index 6210265..719f946 100644 --- a/docs/source/data.rst +++ b/docs/source/data.rst @@ -33,6 +33,9 @@ dataset_functions What? +++++ +Chain of responsibility pattern: +refactoring.guru/design-patterns/chain-of-responsibility/python/example + RecNN is designed to work with your dataflow. Function that contain 'dataset' are needed to interact with environment. The environment is provided via env.argument. diff --git a/docs/source/examples/your_data.rst b/docs/source/examples/your_data.rst index cc5fec5..715aee3 100644 --- a/docs/source/examples/your_data.rst +++ b/docs/source/examples/your_data.rst @@ -53,7 +53,8 @@ Here is how default ML20M dataset is processed. Use this as a reference:: Although not required, it is advised that you return all of the arguments + kwargs. If the function is finishing this may work fine, but if you are using **build_data_pipeline**, you need to do it as I said. Look in -reference/data/dataset_functions for further details. +reference/data/dataset_functions for further details. Chain of responsibility pattern: +refactoring.guru/design-patterns/chain-of-responsibility/python/example Toy Dataset +++++++++++ diff --git a/examples/2. REINFORCE TopK Off Policy Correction/0. Inner workings of REINFORCE inside recnn (optional).ipynb b/examples/2. REINFORCE TopK Off Policy Correction/0. Inner workings of REINFORCE inside recnn (optional).ipynb index df8c42a..0ea560d 100644 --- a/examples/2. REINFORCE TopK Off Policy Correction/0. Inner workings of REINFORCE inside recnn (optional).ipynb +++ b/examples/2. REINFORCE TopK Off Policy Correction/0. Inner workings of REINFORCE inside recnn (optional).ipynb @@ -82,6 +82,7 @@ " value_counts = df['movieId'].value_counts() \n", " print('counted!')\n", " \n", + " # here n items to keep are adjusted\n", " num_items = 5000\n", " to_remove = df['movieId'].value_counts().sort_values()[:-num_items].index\n", " to_keep = df['movieId'].value_counts().sort_values()[-num_items:].index\n", @@ -169,7 +170,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a353201895b5418cb9425666c19f62da", + "model_id": "8fe5a74d03574848a23df2d133aa375e", "version_major": 2, "version_minor": 0 }, @@ -190,7 +191,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3c8ddb1b04894cb7bd9c986d92eee2a9", + "model_id": "f1158eb3e6c64420a782fe6ba7734f92", "version_major": 2, "version_minor": 0 }, @@ -211,7 +212,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e1efae29b9a44d7c880a925fe9ea2ae5", + "model_id": "b968f7f55ac44de48bb8326926dec633", "version_major": 2, "version_minor": 0 }, @@ -301,9 +302,7 @@ " def reinforce_with_correction():\n", " raise NotImplemented\n", "\n", - " def __call__(self, policy, optimizer):\n", - " \n", - " # R = torch.tensor([0]).to(cuda)\n", + " def __call__(self, policy, optimizer, learn=True):\n", " R = 0\n", " \n", " returns = []\n", @@ -315,10 +314,11 @@ " returns = (returns - returns.mean()) / (returns.std() + 0.0001)\n", "\n", " policy_loss = self.method(policy, returns)\n", - "\n", - " optimizer.zero_grad()\n", - " policy_loss.backward()\n", - " optimizer.step()\n", + " \n", + " if learn:\n", + " optimizer.zero_grad()\n", + " policy_loss.backward()\n", + " optimizer.step()\n", " \n", " del policy.rewards[:]\n", " del policy.saved_log_probs[:]\n", @@ -414,7 +414,7 @@ " \n", " if step % params['policy_step'] == 0 and step > 0:\n", " \n", - " policy_loss = params['reinforce'](nets['policy_net'], optimizer['policy_optimizer'])\n", + " policy_loss = params['reinforce'](nets['policy_net'], optimizer['policy_optimizer'], learn=learn)\n", " del nets['policy_net'].rewards[:]\n", " del nets['policy_net'].saved_log_probs[:]\n", " print('step: ', step, '| value:', value_loss.item(), '| policy', policy_loss.item())\n", @@ -441,7 +441,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9ad153b869184df5bc62261dd9192b75", + "model_id": "a534228c3f504c48bd4e23abcf1c5072", "version_major": 2, "version_minor": 0 }, @@ -464,36 +464,123 @@ "name": "stdout", "output_type": "stream", "text": [ - "step: 10 | value: 31.62990379333496 | policy -12072.251953125\n", - "step: 20 | value: 27.167705535888672 | policy -12296.3212890625\n", + "step: 10 | value: 24.99452781677246 | policy -2379.3642578125\n", + "step: 20 | value: 25.838939666748047 | policy -7025.1083984375\n", "step 30\n", - "step: 30 | value: 31.044828414916992 | policy -24366.44921875\n", - "step: 40 | value: 25.81496238708496 | policy -36721.953125\n", - "step: 50 | value: 25.57366943359375 | policy 36845.19921875\n", + "step: 30 | value: 21.31504249572754 | policy -6453.48779296875\n", + "step: 40 | value: 24.557893753051758 | policy -17982.73046875\n", + "step: 50 | value: 24.8399715423584 | policy -2648.5126953125\n", "step 60\n", - "step: 60 | value: 24.099489212036133 | policy 9254.177734375\n", - "step: 70 | value: 24.482318878173828 | policy -2431.529296875\n", - "step: 80 | value: 24.085397720336914 | policy -12564.220703125\n", + "step: 60 | value: 22.109338760375977 | policy -1317.727783203125\n", + "step: 70 | value: 22.422393798828125 | policy 3406.505859375\n", + "step: 80 | value: 23.30132484436035 | policy -23820.28515625\n", "step 90\n", - "step: 90 | value: 23.831710815429688 | policy -5823.1826171875\n", - "step: 100 | value: 21.566062927246094 | policy -16230.359375\n", - "step: 110 | value: 23.551292419433594 | policy 12860.2568359375\n", + "step: 90 | value: 23.076168060302734 | policy 9186.025390625\n", + "step: 100 | value: 22.222537994384766 | policy -3152.271484375\n", + "step: 110 | value: 22.635631561279297 | policy 18650.52734375\n", "step 120\n", - "step: 120 | value: 21.037578582763672 | policy 13131.966796875\n", - "step: 130 | value: 26.000120162963867 | policy 26710.12109375\n", - "step: 140 | value: 21.62142562866211 | policy 6173.89404296875\n", + "step: 120 | value: 22.85053062438965 | policy 67.15323638916016\n", + "step: 130 | value: 23.42321014404297 | policy 15718.7001953125\n", + "step: 140 | value: 19.7723445892334 | policy 22986.236328125\n", "step 150\n", - "step: 150 | value: 22.930984497070312 | policy -47639.875\n", - "step: 160 | value: 22.709178924560547 | policy -770.7249755859375\n", - "step: 170 | value: 23.399124145507812 | policy 9165.1201171875\n", + "step: 150 | value: 17.86973762512207 | policy 11883.517578125\n", + "step: 160 | value: 20.086488723754883 | policy -14132.6865234375\n", + "step: 170 | value: 21.994441986083984 | policy 28941.62109375\n", "step 180\n", - "step: 180 | value: 21.017301559448242 | policy 36191.46875\n", - "step: 190 | value: 24.664167404174805 | policy -2960.500244140625\n", - "step: 200 | value: 21.502689361572266 | policy -12015.58984375\n", + "step: 180 | value: 20.82435417175293 | policy -18888.984375\n", + "step: 190 | value: 20.39084243774414 | policy 12952.0\n", + "step: 200 | value: 19.08588409423828 | policy 15089.16796875\n", "step 210\n", - "step: 210 | value: 21.28272819519043 | policy -23168.14453125\n", - "step: 220 | value: 21.275829315185547 | policy -12109.36328125\n", - "step: 230 | value: 21.715091705322266 | policy 1788.224609375\n" + "step: 210 | value: 19.787445068359375 | policy 11130.337890625\n", + "step: 220 | value: 21.866870880126953 | policy -29572.3984375\n", + "step: 230 | value: 19.001218795776367 | policy -7762.5263671875\n", + "step 240\n", + "step: 240 | value: 17.587854385375977 | policy -17405.8046875\n", + "step: 250 | value: 18.876882553100586 | policy 9699.81640625\n", + "step: 260 | value: 17.76979637145996 | policy 53937.65625\n", + "step 270\n", + "step: 270 | value: 18.574525833129883 | policy 5737.3056640625\n", + "step: 280 | value: 19.043319702148438 | policy 2095.352294921875\n", + "step: 290 | value: 19.431182861328125 | policy 21138.427734375\n", + "step 300\n", + "step: 300 | value: 17.907007217407227 | policy 10494.392578125\n", + "step: 310 | value: 19.347442626953125 | policy -863.209716796875\n", + "step: 320 | value: 19.445966720581055 | policy -3958.390625\n", + "step 330\n", + "step: 330 | value: 16.458955764770508 | policy -3471.0068359375\n", + "step: 340 | value: 17.300609588623047 | policy 24490.462890625\n", + "step: 350 | value: 16.507837295532227 | policy 16351.6865234375\n", + "step 360\n", + "step: 360 | value: 16.999927520751953 | policy 15888.359375\n", + "step: 370 | value: 16.343154907226562 | policy 7932.8515625\n", + "step: 380 | value: 17.079055786132812 | policy -4479.4638671875\n", + "step 390\n", + "step: 390 | value: 17.001665115356445 | policy -5233.06640625\n", + "step: 400 | value: 16.83679962158203 | policy 14890.7744140625\n", + "step: 410 | value: 15.065675735473633 | policy -3753.981689453125\n", + "step 420\n", + "step: 420 | value: 15.776702880859375 | policy 5760.9619140625\n", + "step: 430 | value: 14.647445678710938 | policy -10753.4560546875\n", + "step: 440 | value: 15.86405086517334 | policy 2819.04052734375\n", + "step 450\n", + "step: 450 | value: 16.01703453063965 | policy 23311.623046875\n", + "step: 460 | value: 15.3170166015625 | policy -10775.216796875\n", + "step: 470 | value: 14.39520263671875 | policy -45444.4921875\n", + "step 480\n", + "step: 480 | value: 14.270796775817871 | policy 29882.931640625\n", + "step: 490 | value: 13.62187385559082 | policy -37586.625\n", + "step: 500 | value: 14.875505447387695 | policy 13346.44921875\n", + "step 510\n", + "step: 510 | value: 16.35289764404297 | policy -2197.68701171875\n", + "step: 520 | value: 14.109804153442383 | policy -6195.9072265625\n", + "step: 530 | value: 14.597068786621094 | policy -24810.94140625\n", + "step 540\n", + "step: 540 | value: 13.869283676147461 | policy -9439.814453125\n", + "step: 550 | value: 12.897269248962402 | policy 4875.93994140625\n", + "step: 560 | value: 13.858412742614746 | policy -5171.6552734375\n", + "step 570\n", + "step: 570 | value: 13.509696006774902 | policy -4896.9716796875\n", + "step: 580 | value: 13.47771167755127 | policy 20225.587890625\n", + "step: 590 | value: 14.150710105895996 | policy 27916.4375\n", + "step 600\n", + "step: 600 | value: 13.883435249328613 | policy 26529.67578125\n", + "step: 610 | value: 14.753575325012207 | policy 16934.59375\n", + "step: 620 | value: 14.87264633178711 | policy 8398.638671875\n", + "step 630\n", + "step: 630 | value: 12.911136627197266 | policy -20880.380859375\n", + "step: 640 | value: 12.59472942352295 | policy 1997.0396728515625\n", + "step: 650 | value: 15.05930233001709 | policy -13963.5927734375\n", + "step 660\n", + "step: 660 | value: 13.511848449707031 | policy -13226.11328125\n", + "step: 670 | value: 13.056660652160645 | policy -6386.6376953125\n", + "step: 680 | value: 14.399893760681152 | policy -17728.984375\n", + "step 690\n", + "step: 690 | value: 12.360898971557617 | policy -30248.330078125\n", + "step: 700 | value: 10.530476570129395 | policy 15213.7294921875\n", + "step: 710 | value: 12.827935218811035 | policy -11777.169921875\n", + "step 720\n", + "step: 720 | value: 11.752530097961426 | policy 18676.767578125\n", + "step: 730 | value: 13.420793533325195 | policy 38.823768615722656\n", + "step: 740 | value: 11.595123291015625 | policy 493.12945556640625\n", + "step 750\n", + "step: 750 | value: 12.754773139953613 | policy 2673.26953125\n", + "step: 760 | value: 11.162399291992188 | policy 19384.373046875\n", + "step: 770 | value: 11.600396156311035 | policy -1110.976318359375\n", + "step 780\n", + "step: 780 | value: 9.937241554260254 | policy 406.34869384765625\n", + "step: 790 | value: 11.99508285522461 | policy 14375.361328125\n", + "step: 800 | value: 11.615348815917969 | policy -10188.333984375\n", + "step 810\n", + "step: 810 | value: 10.723864555358887 | policy -2762.9443359375\n", + "step: 820 | value: 10.420403480529785 | policy 3851.626953125\n", + "step: 830 | value: 14.302266120910645 | policy -16290.2451171875\n", + "step 840\n", + "step: 840 | value: 11.320375442504883 | policy 20128.3125\n", + "step: 850 | value: 10.443620681762695 | policy 758.8353881835938\n", + "step: 860 | value: 10.812860488891602 | policy 3802.49267578125\n", + "step 870\n", + "step: 870 | value: 10.839583396911621 | policy 5098.1533203125\n", + "step: 880 | value: 11.484695434570312 | policy -30153.146484375\n" ] }, { @@ -504,7 +591,7 @@ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mwriter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwriter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m debug=debug, learn=True, step=step)\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mplotter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_losses\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mreinforce_update\u001b[0;34m(batch, params, nets, optimizer, device, debug, writer, learn, step)\u001b[0m\n\u001b[1;32m 4\u001b[0m learn=False, step=-1):\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrecnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_base_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0mpredicted_action\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicted_probs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'policy_net'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mreinforce_update\u001b[0;34m(batch, params, nets, optimizer, device, debug, writer, learn, step)\u001b[0m\n\u001b[1;32m 4\u001b[0m learn=False, step=-1):\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrecnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_base_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0mpredicted_action\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicted_probs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'policy_net'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/Documents/RecNN/recnn/data/utils.py\u001b[0m in \u001b[0;36mget_base_batch\u001b[0;34m(batch, device, done)\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 221\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'reward'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 222\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 223\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/Documents/RecNN/recnn/data/utils.py\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 221\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'reward'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 222\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 223\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " diff --git a/examples/2. REINFORCE TopK Off Policy Correction/1. Basic Reinforce with RecNN.ipynb b/examples/2. REINFORCE TopK Off Policy Correction/1. Basic Reinforce with RecNN.ipynb index 6c71e80..1adccd2 100644 --- a/examples/2. REINFORCE TopK Off Policy Correction/1. Basic Reinforce with RecNN.ipynb +++ b/examples/2. REINFORCE TopK Off Policy Correction/1. Basic Reinforce with RecNN.ipynb @@ -9,6 +9,8 @@ "import torch\n", "import torch.nn as nn\n", "from torch.utils.tensorboard import SummaryWriter\n", + "import torch.nn.functional as F\n", + "from torch.distributions import Categorical\n", "\n", "import numpy as np\n", "import pandas as pd\n", @@ -45,39 +47,23 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [], - "source": [ - "def embed_batch(batch, item_embeddings_tensor, *args, **kwargs):\n", - " return recnn.data.batch_contstate_discaction(batch, item_embeddings_tensor,\n", - " frame_size=frame_size, num_items=num_items)\n", - "\n", - "def prepare_dataset(**kwargs):\n", - " recnn.data.build_data_pipeline([recnn.data.truncate_dataset,\n", - " recnn.data.prepare_dataset], reduce_items_to=5000, **kwargs)\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "action space is reduced to 26744 - 21744 = 5000\n" + "action space is reduced to 26744 - 25744 = 1000\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "86a87f7c902c40b09b3a4ef75bb51a9e", + "model_id": "19e6f28b52204d98b7164b16325a8c6c", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "HBox(children=(IntProgress(value=0, max=18946308), HTML(value='')))" + "HBox(children=(IntProgress(value=0, max=12840344), HTML(value='')))" ] }, "metadata": {}, @@ -93,12 +79,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "df8d673dcb5540fab5f1448b0bfd1dc8", + "model_id": "b2fc7accdae64a6392ce64518e1a6adc", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "HBox(children=(IntProgress(value=0, max=18946308), HTML(value='')))" + "HBox(children=(IntProgress(value=0, max=12840344), HTML(value='')))" ] }, "metadata": {}, @@ -114,12 +100,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4b05ece30858432f940afddba40de114", + "model_id": "b2176330700441c7bcc09c11e5ebf4c1", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "HBox(children=(IntProgress(value=0, max=138493), HTML(value='')))" + "HBox(children=(IntProgress(value=0, max=138475), HTML(value='')))" ] }, "metadata": {}, @@ -134,6 +120,15 @@ } ], "source": [ + "def embed_batch(batch, item_embeddings_tensor, *args, **kwargs):\n", + " return recnn.data.batch_contstate_discaction(batch, item_embeddings_tensor,\n", + " frame_size=frame_size, num_items=num_items)\n", + "\n", + " \n", + "def prepare_dataset(**kwargs):\n", + " recnn.data.build_data_pipeline([recnn.data.truncate_dataset,\n", + " recnn.data.prepare_dataset], reduce_items_to=1000, **kwargs)\n", + " \n", "# embeddgings: https://drive.google.com/open?id=1EQ_zXBR3DKpmJR3jBgLvt-xoOvArGMsL\n", "env = recnn.data.env.FrameEnv('../../data/embeddings/ml20_pca128.pkl',\n", " '../../data/ml-20m/ratings.csv', frame_size, batch_size,\n", @@ -143,25 +138,79 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "value_net = recnn.nn.Critic(1290, num_items, 2048, 54e-2)\n", - "policy_net = recnn.nn.Actor(1290, num_items, 2048, 6e-1)\n", + "value_net = recnn.nn.Critic(1290, num_items, 2048, 54e-2).to(cuda)\n", + "value_net.save_limit = 15\n", + "policy_net = recnn.nn.DiscreteActor(1290, num_items, 2048).to(cuda)\n", "\n", - "cuda = torch.device('cuda')\n", "reinforce = recnn.nn.Reinforce(policy_net, value_net)\n", "reinforce = reinforce.to(cuda)\n", + "\n", "plotter = recnn.utils.Plotter(reinforce.loss_layout, [['value', 'policy']],)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step 990\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6IAAAF4CAYAAABOwkTnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3zT1f7H8VeS7j1pgbYUyiobVBSEGFxR3BM3bhTc83pd6OXnuO6FIioq4sRxuQ7ijBH3ZJZVoFBWKZRS6E7y+yPFi8hI26Tfpn0/Hw8e0O9859x6k0/O+Z5j8nq9iIiIiIiIiLQUs9EBREREREREpH1RISoiIiIiIiItSoWoiIiIiIiItCgVoiIiIiIiItKiVIiKiIiIiIhIi1IhKiIiIiIiIi1KhahIK2Oz2i+0We31RucQERERH5vVvspmtd+xy89Om9X+gpGZREJdmNEBRERERERCzKmAvjQWaQYVoiIiIiIijeB0ObYYnUEk1KkQFQkgm9V+GfAwkOl0Oap22X4rcA2QAzwHHA50AtYDbwL3OF2Omr1c80LgBafLEbbLtixgDTDK6XI4G7Z1Bx4EjgC8wM/AjU6XY35gX6WIiEjosFntTmAFUAJcCkQAbwNXO12OKpvVHg78CzgfSAeWA5OcLsfr+7nmcqfLceku2yYAE4A8oBxwOV2O021W+z3AWU6Xo9du15gGdHW6HLYAvVSRkKJnREUC6218b3An77b9fOA1fAXiRuAcIB+4DrgI+Gdzbmqz2jOAOfjeZEcChwBLAKfNak9vzrVFRETagNOBVHzvkecCJ+L78hbgPuAyfO/J/fC9X79ms9qP8PfiDcXmg8BkoD9wDPBHw+6pQJ7Naj9sl+PjgTMa9om0S+oRFQkgp8tRbrPa/wNcALwBYLPahwB9gTFOl8MD3LHLKatsVnseMB64uxm3vhJY5XQ5rty5wWa1XwOMxveG+3gzri0iIhLqtgBXOF0ON1DQMPHQUzar/TZ8I5aud7oc7zQce5/Naj8IuB34Yn8XtlntscAtwJ1Ol+PpXXb9BuB0OYptVvvH+Irdrxv2nQPUAO82/6WJhCYVoiKB9yowy2a1Zzpdjg34ekN/dbocC+HP4buXArlALL7/Dps7OuEg4ACb1b59t+3RQI9mXltERCTU/dRQhO70Lb4RTHkNf7t2O/5r4DY/r90XiAI+3ccxU4CZNqv9aqfLUYavKJ3udDmq/byHSJujQlQk8BzAJuBcm9X+BHA2vmE/2Kz2M4BngH/ge5Pbhm9ozv/t43qePWwL3+1nM75vba/aw7HljQkvIiLSDph2+9m7h/27b9uffR3/Cb5Hc863We0u4ABgbCOvL9KmqBAVCTCny+G2We2v4xueWwCk0DBMF7ACvztdjkd3Hm+z2nP3c8kSwGKz2jOcLsfGhm1DdjvmF+BCYO2ukySJiIgIAAfZrHbLLr2iw4BaoBDfENnDgIW7HG/d7ed9WQRUA3ZgjxMEOl0OT8O6o5cBvYDvdo6UEmmvVIiKBMcrwA34ejo/cbocmxq2LwEusVntJwELgOPxrUW2Lz8BFcADNqv9PnzDiO7a7ZingUuAD2xW+yR8M+pmAccCHzldju+a/5JERERCVirwTMNIpW74Zsmd6nQ5dtis9ieBf9ms9k34Jhg6AzgJOMqfCztdju02q/0RYKLNaq8CPsP3aMxop8tx/y6HvohvPoiewOUBel0iIUuz5ooEgdPlmIfvzWwQvmdGd5oCTAemAb8DBwMT93OtLfiG9x4CzAPuxDcpwq7HbMT37W4p8B6+gncG0AXfEjEiIiLt2Ux8X+rOwbds2sf87730dnyz1z6Orxf0POA8p8ux34mKdnFnw3WuwfdF86fsNnrJ6XKsBz4EqvDNsi/Srpm83sYOfxcRERERCQ17WvPTwCw/AT86XY6rjc4iYjQNzRURERERCSKb1d4B33DfIfhGOYm0eypERURERESCayNQBlzrdDkKjQ4j0hpoaK6IiIiIiIi0KE1WJCIiIiIiIi2qxYfmTrjpLhO+ZSW2tfS9RUSkTUoAip95+F4N8WkivTeLiEiA7fe92YhnRLOA1QbcV0RE2q4cfOvnStPovVlERAJtn+/NRhSi2wD+dfsNREdF+X2S2+2moKCA/Px8LBZL0MK1FWov/6mt/Ke2ahy1l/+a2lZV1dXc+X+PgnrymkvvzS1A7eU/tZX/1FaNo/byX7Dfmw2bNTc6Koro6Ma92UVEhBMdHaVfGj+ovfyntvKf2qpx1F7+U1u1DnpvDi61l//UVv5TWzWO2st/wW4rTVYkIiIiIiIiLUqFqIiIiIiIiLQow4bmioiISGjwer243e6/bXe73Xi9Xurr62mv65JbLBZMJpPRMUREQo56REVERGSv6urqqK2t3eM+s9lMXl4eZnP7/ThRV1dHVVUVHo/H6CgiIiFFPaIiIiIGslnt0cB8IM3pciQ1bAsDHgHOx/el8UxggtPlqAnE/sZwu91E7WUmXa/Xi8ViISwsrN33ClZVVREdHW10DBGRkNF+v8IUERFpHe4Finfb9k/gMKAf0APoC9wfwP1+8Xg87bq3szEsFku7HZ4sItIUencRERExiM1qHwKM5u9F4qXAJKfLsc7pcmwCJgIX2ax2c4D2+0WFqP/MZvMen6MVEZE909BcERERAzQMn50KTNhtexKQDfy+y+bfgCQg22a1lzdnP1C0t0xut/svxZTb7cZsNu+1p2/ndvUE+trA4/Hsc4jyzrZVwbp/aiv/qa0aR+3lv6a2lb/HqxAVERExxo3APKfL4bRZ7bZdtsc3/F2+y7atu+zzNHP/XhUUFBAREf7nz16vl7y8vP0uZF5dXb3P/e2B2+2msLDQr2dlCwoKWiBR26C28p/aqnHUXv5rbFvV1tb5dZwKURERkRZms9rz8PWEDt7D7oqGvxOB0oZ/J+2yr7n79yo/P5/o6P9NTFRfX4/ZbCYsbM8fF7xeL9XV1URFRbWJyYouHHs548ZdwrDhBzf63Pr6evLz8/faVuArVgsKCsjPz99vcd/eqa38p7ZqHLWX/5raVlVV1cCs/R6nQlRERKTljQTSgYU2qx0gAkiwWe0bgFOBNcAgoLDh+MH4ejXXOF0Oj81qb/L+fYWyWCx/+bCxc8jt/opMk8lkeCE65swLuOrqKxg5cniTr/HKq1ObfK7JZPpb++2Nv8eJ2qox1FaNo/byX2Pbyt9jQ7YQnf3rJr6Yt5lumTF0y4iha2Y0uR2iiQjTpAoiItLqvQXM3uXn4cA0fMXjFuAF4Hab1f49UIdvsqFpTpdj57Db5u5vV3Y+v6kPnSIiezfj63VU13q45KisFrlfyBainVKjyEmPZunaHcz+rZT1W2qIDDczvHcSowakMKJPMnFRIfvyRESkDXO6HFVA1c6fbVb7FsDrdDk2NPx8H5AGLMQ3w/07+JZk2am5+9uUu++aRMnGEv51z/2YLWYOO2wkf/wxjxNOOJZvv/2BFYUreeLJh6io2M7UqdNYW7yWiMhIhg07mKuuvoKYGN/6n7v2qn7yyafMfOd9bLaRvPfeLLweD6eedhIXjD3X4FcrIhJ4S9fuYPJHq5l0fo8Wu2fIVmoDcuMZkPu/OReqatzML6rgy3lbeHxWEf96s5BhvZM457BODO4Wb/iQIRERkb1xuhxO/vccJ06Xox64puHPno5v1v7mcHu8bKnwTUThe0a0jqhaS1DeZ1Piw7GY93/de+69429Dc8eceQGffPwp991/D9k5WdTX17O4YAk33XQted27UVq6mdv/OZHpr8xg3JWX7vG6q1YWYTnCxjszX2P58hVcNf56hg07mB49uwf0dYqIGKm61s2dM5Zx3EHpjOqf2mL3DdlCdHfRkRaG9kxiaM8kbj61K3NXVvDRzyVcNWURfbLjGHtEZ0bkJ6kgFRERaYYtFXUcd++vLXKvj+46gPTEiCaff/IpJ9AlNweAiIgIBgzs/+e+jIwOnHb6yfzn/f/u9fyExATOOXcMAL179ySvezeWLl2uQlRE2pQn/1uE2+Pl+hNzW/S+baYQ3ZXFbGJIXgJD8hK43J7NDNd6bp++lKzUKK4/KZeDeiQaHVFERCQkpcSH89FdBwDBnzU3JT58/wftQ2Zmxl9+XrJkGVOnvERh4QpqamrxeNwkJSXt5WxISUn+y89RUVFUVlY2K5OISGvyzaIyPvixhBeu7kd0ZMs+R98mC9FdZSRHcsNJuVx8RGde+XIt1zxfwJEDU7n2xC6kJTT9W1YREZH2yGI2/dlL6fV6qYpwEx0dYfiII/Oe7r/btnsn3of9mCO5d9JdxMRE88knn/LytNdaKKGISOtSuq2WSW8tZ5w9mz7ZcS1+/3YzxWxSXDjXnpjLazcMYMPWGs548A/enrMet8drdDQRERFppuSUZNYWr93nMZWVlcTFxRETE03xmrXMfPu9FkonItL6PPrBKrpmxHDeqE6G3L/dFKI75XWMYcr4vtx4Ui5THcVcNWURpdtqjY4lIiIizXDe+Wfxwfv/5bjRp/LvBx7d4zE33nQtM995n2PsJ3H/fQ9x+BGjWjiliEjrULqtlq/mb2bCcTl+TQoXDG1+aO6emM0mjh/agUN6J3HXjGWc+8hcJp7dg2G99/6ciIiIiLRew4cfwvDhh+zzmBEjhzOiYVbdnc49b8yf/37r7Vf//Pexxx7Nscce/Zdjn3jyoQAkFREx3ie/biInPZp+OS0/JHendtcjuqu0hAieGteHM0d05IYXF/PUh0XUu9vlWt8iIiIiItIOeL1eZv1YwokHdzD0+f52XYiCb9KFS47KYvKVfXD8VsrVzxewrbLe6FgiIiIiIiIBN29VBcWbazh2SLqhOdp9IbrT4G4JvHxdf2pqPVz85HzWlFYZHUlERERERCSgZv1YgrVvcrOXyGouFaK7SEuIYPL4PvTsFMtFTyzg9xXbjI4kIiIiIiISEDuq3Xw+dzMnHtzB6CgqRHcXFW5h0nk9OH14Blc9t4jZv24yOpKIiIiIiEizfT63lPjoMA7pZfwkre1y1tz9MZtNXHFsDlmpUdz7ViHbquo5c0RHo2OJiIiIiIg02awfSxh9YLphS7bsSoXoPhw/tAMJMWH8c/pSKqrcXHxkZ0NnlhIREREREWmKlRsrmV+0nXvO6WF0FEBDc/fL2i+Fxy/LZ/pXa3lsVhEej9foSCIiIiIiIo3y359KOCAvgay0KKOjACpE/XJg90QmX9GXT37ZxKS3C3GrGBURERERkRDh8XiZ/Vspxx1k7JItu1Ih6qc+OXFMmdCXH5Zs5e4Zy6h3e4yOJCIiIg3GnHkB33zzXbOv88knn3LJxVcGIJGISOuxeO0OyrbXMbJvstFR/qRCtBG6Zcbw3Pi+zF1VwT+nL6O2XsWoiIiIiIi0bnMWldE/N57EGGPXDt2VJitqpJz0aKZM6Mv4Zxdx68tLuH9sT6LCLUbHEhERabfuvmsSJRtL+Nc992O2mDnssJFcd/1VTH3+JeZ88x01NbUMGTKQa6+/iqSkRLxeL89PeQnH7M+pqqoiITGBy8ddTE52Fo8+8iTuejfH2E8CYOoLz5CdnWXwKxQRaZ45i8o4cmCq0TH+QoVoE3RKieL5q3zF6A0vLOahi3oTG6ViVERE2of6yircVVWAl7qqaizRVcD/ZpUPT0zAHBZGbVkZYbGxmCMiqCsvx1Pv3uP1TGYTEcnJeOrrqSvfRkRKcqNmqb/n3jsYc+YFXHX1FYwcOdy3beJ94PXywkuTiYyM5InHJ/PgA49w/wP38svPv/H5Z18yZepTpKensWlTKdu3b6dr11xuuPEaZr7zPi++9GzTG0hEpBUp3VbL4uIdTDy7u9FR/kKFaBN1SIzkufF9ueb5Aq6asojHLu1NUmzr6eoWEREJlnX/mcWaN9/e6/5BTzxCbG4uf1x/M93HjyNl6EEsmnQ/25cu2+Px4cnJDH35BaqKi/nj2hsZ9u5bmMKa/hFl69atOL9y8cGst0lISADgsssv4pSTxrBjxw7CwsOoqa1l1coikpISSU9PIz09rcn3ExFpzb4tKKNTSiRdM6KNjvIXKkSbIS0hgufG9+WGFxcz7pmFPDUunw6JkUbHEhERCapOJ51IxtFHAV6qq6qJio5i9x5RgEGPPURYbCwAfe64bZ89ogDRWVkc+NJUTJbmjTLasH4jXq+Xc86+8C/bwyPCKSnZxODBA7n00gt5edp0Vty5ikGDBzB+wuUagisibdKcRWWM6NO4kSYtQYVoMyXEhPHU5fn849WlXPb0Qp4el092Wuv6tkFERCSQwmKiCYuJxuv14q6qIiI6eo8fcCKS/zc7Y3hi4n6vaw4LIzI1pUmZzLvcPyMzA7PZzMx3Xyc6es/r5Z140nGceNJxVFZW8uwzU3nowcd48ulH/nIdEZFQV1Pn4ael5TwwtpfRUf5Gs+YGQHSkhYcv6kW/nDgue3oh81ZVGB1JRESkXUlOSWZt8Vrfv5OTsB52KI8/9jRlZVsBKCvbitP5DQCLC5Ywf/5C6urqiIiIIDIqErPF/Od1Npduprq62pgXIiISQL8VbgNgSF6CwUn+Tj2iARIeZubec3vwzEdFXDl5Ibed3o3jh3YwOpaIiEi7cN75Z/Hk45OZPv0NDrOO4JZbb+TladO5ctw1lJdvIzk5iWHDD8ZmG8mOykqenTyVtcXrsFgs9OrdgxtvuhaAIUMG0bdfH8447Vw8Xi9TpjxFVnZng1+diEjTfFtQxtCeiUSGt77+RxWiAWQxm7jmhFy6d4zlvncKWba+kquP70KYRcN8REREgmn48EMYPvyQv2wbP+Fyxk+4/G/HHnDAYF54cfIerxMWFsb/3TcxGBFFRFqU1+tlzqIyLjqydX6ZpkI0CEYfmE6XDlHcPG0JKzZUMvGc7qTGRxgdS0RERERE2omVG6tYt6WGQ/OT93+wAVpfH20b0TcnnleuG0BNvYcxD/7Bhz+X4PV6jY4lIiIiIiLtwJxFZfTOiiUtoXV2iKkQDaL0xAieu7IvVxybw8Pvr+Sa5wtYt0WTH4iIiIiISHDNKfAt29JaqRANMrPZxOmHZvLWzYMIs5g466G5PPVhERu31hgdTURERERE2qCKqnrmr6pgRCsdlgt6RrTFZCRH8uglvXEu2ML0L9fx+tfrOWpQKucc1pHeWXFGxxMREfkbs9lMfX290TFCgsfjITw83OgYIiIA/LKsnLioMHpnxRodZa9UiLYgk8nEqP6pjOqfyryVFcz4eh0XPj6frhnRHNwriUN6JTGoWzxR4Rajo4qIiGA2m/F4PEbHCAlut5uIiNb5HJaItD8/LNnKwT0TMZtb7+odKkQNMqBrPAO69mLdlmrmLCrjhyXlvP/9EjweL31y4ujVOZaeDX+6ZUQTZtEoahERaXkWi4Wamhoslr9/Ser1enG73dTX12Mytd4PO8Hk8Xhwu91ERkYaHUVEBPD9f/MPS8q57Ogso6PskwpRg3VKieLMER05c0RHaus9zFtVwYKi7Sxdu4NvC9ayprSayHAzfbJjGdg1gQG58QzIjSchRv/TiYhI8IWHh/9ZcO7O4/FQWFhIfn7+HgvV9iA8PFw9oSLSqqwprWZ9WQ1DeyUaHWWfVM20IhFhZg7snsiB3f/3S7Oj2s3i4u3MXVXBvJUVvPvdBqpqPRw+IIXTh2cysGt8u/0WWkREWobJZCIs7O8fGUwm05/72mshKiLS2vy4pJxumdF0SGzdIzVUiLZysVEWDuieyAENxanH4+WPlRW8990Gxj+3iC7pvh7VEw/ugKUVjwEXEREREZHg8z0fmmR0jP1SIRpizGYTQ/ISGJKXQOm2Wmb9VMJzs1cz+7dN3HNODzKTW/c3HyIiIiIiEhx19R5+LSzn9EMzjY6yX5oBJ4SlJURw8ZFZvH7TQKIjLJz7yFw+n7vZ6FgiIiIiImKA+UXbqXd7Gdwt3ugo+6VCtA1IjY/gsUt7c7k9m4mvL+Nfby6npk7T7YuIiIiItCc/Lt3KoK4JREW0/uf2VYi2ESaTiTEjOzLt2v78vmIbd71eiNvjNTqWiIiIiIi0kB+XbOWQXq3/+VBQIdrm9OgUy9NX9GHRmu1M/6EWr1fFqIiIiIhIW7d1ex0FxTs4uJUv27KTCtE2qFNKFI9f2ovfV7uZ/Emx0XFERERERCTIfl5WTnJcON07xhgdxS8qRNuovMwYrjkikpnfbuTVL9caHUdERERERILoh6VbObhnIiZTaCzpqEK0DctLt3D/Bd2ZMnuNZtMVEREREWmjvF4vPy4pZ1iIPB8KKkTbvEN6JXH18V144J1CSsprjI4jIiIiIiIBtmJjFSXltRzUMzSeDwUVou3CmSMy6ZUVx6S3CjV5kYiIiIhIG/PtojL6ZMeSGh9hdBS/qRBtB8xmE3eNyWNB0Xbe/W6j0XFERERERCSA5hSUMaJPstExGiXMn4NsVvsEYCwwAPjB6XLYdtmXDzwBHATUAbOBq50uR3nA00qTZSRHcsupXfm/d1ZwUM9EuqRHGx1JRERERESaqbyyjnkrK7jhpFyjozSKvz2i64EHgMf2sO8NYAWQCfQCshqOlVbGPiSNEX2Smfj6curdGqIrIiIiIhLqvl+8ldT4cHp1jjU6SqP4VYg6XY73nC7He8CexnV2A6Y7XY4ap8tRBrwL9A9gRgkQk8nErad1ZePWGmZ8vc7oOCIiIiIi0kxzFpVxaJ/kkFm2ZSe/hubux0PAWJvV/jsQC5wBfLS/k9xuN2632++b7Dy2Mee0Z3trr/goM9eekMN976zEPiiF9MTQeaA5WPS75T+1VeOovfzX1LZS24qISHtW7/by/eKt3H1Wd6OjNFogCtHZwEvANsACfAo8sr+TCgoKiIgIb/TNCgoKGn1Oe7an9sowe8lKgvvfnM/Fh0YakKp10u+W/9RWjaP28l9j26q2ti5ISURERFq/easqqKnzcFCP0Fm2ZadmFaI2qz0Z+By4C3gWX4/oU8BrwJn7Ojc/P5/o6Ci/7+V2uykoKCA/Px+LxdL00O3E/trr9pQdXPLUQi4dnUuf7DgDErYe+t3yn9qqcdRe/mtqW1VVVQOzghdMRESkFZuzqIwDuycSHRl6nzOa2yOaB8QATzpdDi9Qa7PapwCf7O9Ei8XSpA9mTT2vvdpbe/XJSeD4gzrw2KzVvHh1v5AbUx4M+t3yn9qqcdRe/mtsW6ldRUSkPfu2oIwzDs00OkaT+Lt8S1jDsWGA2Wa1RwEeYDFQAYxvKEBjgHHA70FJKwF1xbHZnP7AHzh+K+WYA9KNjiMiIiIiIn4qLq1m5cYqDs0PrfVDd/J3+ZY7gCp8ExONbPj3p06XYztwAnA2UAqsBBLwrTkqrVxqfASXHJnFUx8WUVmjCT9ERERERELFtwVldO8YQ8eU0Jzzxa8eUafLMRGYuJd93wIjAhdJWtKYkZm8/8NGpn+1lnHH5BgdR0RERERE/DBnURkj+oRmbygEZtZcCWHhYWauOj6Hia8v5/RDM0mN13IuIiLBZrPaI4GngSOADsB64Bmny/F4w/4wfDPQn49v9NJMYILT5agJxH4REQltO6rd/Fq4jcuOzjY6SpP5OzRX2jBbvxS6ZcYw7fO1RkcREWkvwoANwNH4Hmk5A7jNZrWPadj/T+AwoB/QA+gL3L/L+c3dLyIiIey7gjLioiz07RK6q1+oEBVMJhNXHpvDe99vZP0WfVkuIhJsTpdjh9PluNPpcix3uhwep8vxB/ARcGjDIZcCk5wuxzqny7EJ3+MxF9msdnOA9ouISAj7788l2IekYTGH7soXGporAAztmcigrvFM/XQNd53V3eg4IiLtSsNQ2hHAv21WexKQzV9noP8NSAKybVZ7eXP2A0V7y+F2u3G7/Z+8buexjTmnPVN7+U9t5T+1VeO0hfYq2VrLj0vLufLYrKC+jqa2lb/HqxCVP40fncNlTy/gPFsnumXGGB1HRKQ9eRIoB14FMhq2le+yf2vD3/H4lk9rzv69KigoICIi3P/Uu5wn/lN7+U9t5T+1VeOEcnt9NL+OrCQTtVtWsmBL8O/X2Laqra3z6zgVovKnfl3iGdEnmSmz1/Dghb2MjiMi0i7YrPZH8PWGHu50OWptVntFw65EfEujga83E3xrdzd3/17l5+cTHR3ld3a3201BQQH5+flYLBa/z2uv1F7+U1v5T23VOKHeXl6vl3s+mscZI7Pp1y8zqPdqaltVVVUDs/Z7nApR+Ysrjs3h3Ifnsmj1dvrkhO7DzyIiocBmtT+Ob+bcw50uRymA0+XYarPa1wCDgMKGQwfj69Vc43Q5PM3Zv688FoulSR/Mmnpee6X28p/ayn9qq8YJ1fb6fcU2Nmyt5dgDOrRY/sa2lb/HqhCVv8jLjOGYA9J59pPVPDWuj9FxRETaLJvV/iRwODCqYUKhXb0A3G6z2r8H6vBNNjTN6XJ4ArRfRERC0Ic/lWDtm0xSXOMfo2htNHue/M1lR2fxy/Jt/Fa4zegoIiJtks1q7wJcDXQHVtqs9u0Nfz5pOOQ+YA6wEFgOLMK3JAsB2i8iIiGmssbN53M3c8LQDkZHCQj1iMrfdE6N4sSDO/Dc7NVMGd8Xkyl0p4UWEWmNnC5HEbDX/3N1uhz1wDUNfwK+X0REQs+XczcTG2Xh4J5J+z84BKhHVPbo4iM7s2j1dn5aWr7/g0VEREREJKj++3MJow9IJ8zSNjqJVIjKHmUkRXLqsAyem70Gr9drdBwRERERkXZrTWkVv6+oaDPDckGFqOzDBYd3pnBDJXMWlRkdRURERESk3Xrtq3UM6hZPlw7RRkcJGBWisldpCRGceWgmz81eg8ejXlERERERkZ28Xi9FJVXM/G4Dkz9ezS/Ly6l3B35y8vlFFcz6qYTrTsgN+LWNpMmKZJ/OG9WJd7/byJfzt3DkwFSj44iIiIiIGGrR6u28+c16flleTum2OrplRpOeEMEbX68jItzMsF5J2Aakcnj/FMzm5j3PWe/28sDMFZx8SAZ9cuIC9ApaBxWisk9JseGcbe3I87PXMKp/CpZm/sckIiIiIhKqPvyphPtnrmDUgBSuOzGXIXkJpCVEAFBd6+anpeV8s6iM+94u5O1cNQcAACAASURBVO0567njzDxy0ps+nPadbzeweVsd40fnBOoltBoamiv7dc5hHdmyvY5Pftl9vXURERERkbav3u3l0Q9W8sC7K/jnmXlMOq8nRw9O+7MIBYiKsGDtl8LtZ+bx9q2DSIoN59yH5zLDuQ53Ex5zKymvYcrs1Vx7Yhfio9te/6EKUdmvuOgwLjy8M89/uoba+sCPexcRERERaa227qjjmucX8eW8LTw/oR/HHZi+33PSEiJ4cGxP7jq7O69+tZbLnlrAig2Vjbrvox+sIj87jmOGpDU1equmQlT8cvqIDDweeO+7jUZHERERERFpEVU1bq6eUkB1nYeXr+vfqOc0TSYTRw1K462bB9EpNZLzH53Hi58V+zWh0VfzNuNaWMatp3bDZGqbj8apEBW/RIVbuPToLKZ9UcyOarfRcUREREREgsrj8TLxjeXU1nt48vL8vwzDbYykuHAmndeTB8b25L3vNnDh4/NZUrxjj8eWbqvl7teX8c/pS5lwXA65GW1nuZbdtb3BxhI0xx/UgelfreMN1zouPTrb6DgiIiIiIkEzxbGG3wq38fJ1/YmLan7ZNLJvCoO6JfDkf4u48In59M+NY0CXePrnxpOfHYvjt1Je/KyYnp1jeeX6AfTsFBuAV9F6qRAVv4VZTFxxTDb3vbOC04dnkhQXbnQkEREREWkD3B4vH/5cwqtfrqN7xxjGjMxkcLcEw4alzv51E699tY6nxvWhc2pUwK4bHx3G7WfmccqwDH5cspX5Rdv54McStlXWk54Qzm2n53H04NQ2Oxx3VypEpVGOGJjKq1+t4+Uv13LdiblGxxERERGREPfL8nIe/88qNpTVcP6ozqwsqeLqKQV06RDNmSMyGX1gOhFhLfdE4bxVFUx6u5BbT+vGkLyEoNyjT3YcfbJ9z5t6vV7Wbq4hNT6c6EhLUO7XGqkQlUYxm02MH53NzdOWMGZERzqmRBodSURERERCUF29hztnLOPrBWWcfmgGlx6dRWKMb8Td1cfn8MEPJUyZvYbP/9jMw5f0Iio8+EVa6bZabn15CWccmsmJB3cI+v3AN6lRVlrgel1DhSYrkkY7pFcSA7sm8Ownq42OIiIiIiIh6s1v1jN/VQVv3DSQG0/u+mcRCpAaH8ElR2Ux/YYBbNxaw60vL6WmLrjLCNa7vdz52jJyO0Qz4bguQb2XqBCVJjCZTFx7Qhc+/b2URWu2Gx1HREREREJM6bZaXvysmKuP77LPmWHTEiKYfGVfikur+ccrS6gL4pr2zzvWULSpin+d14MwS9t/RtNoKkSlSXp2jmX0gek8MWsVXq/X6DgiIiIiEkKe+Wg13TvGYh+Stt9j0xMjmHxFH1ZurOKf05f6tQ5nY32zcAvTv1rHpPN6NnmZFmkcFaLSZFccm82iNTtwLSwzOoqIiIiIhIgFRRXM/m0TN52S6/fssBnJkUy+sg9L1u7g7teX4/EEriNk7eZqJr6xnPGjs4M2OZH8nQpRabIOiZGcZ+vIUx8WBeWbKRERERFpWzweLw+/v5IThnagd1Zco87tlBLFM1f04Zfl5Tz6n8CMyttR7eYfryxlSF4C59k6Nft64j8VotIs59k6s73azXvfbzQ6ioiIiIi0ch//sonVm6q58ticJp2fnRbN45fm8+HPJUz7Ym2zstTUebh52mI8Xi93n9W9Xazd2ZqoEJVmiY2yMM6ezVRHMRVV9UbHEREREZFWake1m6c/Xs3l9myS48L3f8Je5GfH8e8Le/Pip8V88EPTOkPq3V7unLGMDWU1PHFZPnHRWtWypakQlWY7YWgH0hLCefGzYqOjiIiIiEgr5fi9lIgwE6cfmtHsaw3tmcjEc7rz4Lsr+WLu5kad6/V6+fd7q1hYVMFT4/pociKDqBCVZguzmLjh5K689c0GVm2sMjqOiIiIiLRCH/5cwvEHdiDMEpgS5KhBadx0Si53vLbM7zlLPB4vM3+tw7lgC09e3ofOqVEBySKNp0JUAuKgHolY+ybzmJZzEREREZHdrNpYxYKi7Yw+MD2g1z1teCaTr+yD47dSLnt6IWs3V+/12N8Kt3HxUwuZs7yeRy/uRV7HmIBmkcZRISoBc80JXfh1eTnfFmw1OoqIiIiItCIf/VLC4G7xZKUFvgdycLcEXrtxAKnx4Zz36DzenrOe3wrLKdpUxY5qN2s3V/OPV5Yw4bmF9O8Sx/+dHE2/Lo2bsVcCT0/lSsB0To3iXFsnHvvPKg7umUh4mL7nEBEREWnv3B4vH/+yiSuaOFOuP5Jiw3nool7M/HYjb7jWsam8lpr6/43SO6RXIjNuHEiX9EgWLFgQtBziPxWiElAXHt6Zj37exFvfbOC8UVqLSURERKS9+2lpORXVbg4fkBrU+5hMJs4YkckZIzLxer1sr3ZTuq2WereXHp1iAXC73UHNIP5Tl5UEVHSkhauOy+HFz4op3VZrdBwRERERMdhHP5dw+IBUYqMsLXZPk8lEfHQYXTNi/ixCpXVRISoBZx+SRl7HGCZ/vNroKCIiIiJioIqqepwLtnD8QYGdpEhCnwpRCTiTycRNp+Tyya+bmF9UYXQcERERETHIZ39sJjU+giHdEoyOIq2MClEJit5ZcZw4tAMPv78Sj0fLuYiIiIi0Rx/+XMJxB6ZjNpuMjiKtjApRCZorj81hbWk1//25xOgoIiIiItLCgrV2qLQNKkQlaJLiwhl3TA6TP15NRVW90XFEREREpAV99kcpA3KDs3aohD4VohJUpwzLIDU+gqmONUZHEREREZEW9NX8LRw+IMXoGNJKqRCVoAqz+CYueufbjRSurzQ6joiIiIi0gDWlVSxfX4mtvwpR2TMVohJ0Q/ISOWJACo/+ZxVeryYuEhEREWnrnPO30KtzLJ1SNCxX9kyFqLSIq47vwrxVFbgWlhkdRURERESC7Kv5W9QbKvukQlRaRGZyJBeM6sTjs1ZRW+8xOo6IiIiIBMmm8loWFG1nlApR2QcVotJizh/ViXq3lzdc642OIiIiIiJB8vWCLeSkR9E1I9roKNKKqRCVFhMVYeHq47sw7fNiSrfVGh1HRERERILgq/lbGNU/BZPJZHQUacVUiEqLOmpQKj06xjL549VGRxERERGRANu6o47fCssZ1T/V6CjSyqkQlRZlMpm44eRcPv5lE4tWbzc6joiIiIgE0JyFZaTGR5CfHWt0FGnlVIhKi8vPjuO4g9K1nIuIiIhIG7NztlwNy5X9USEqhrjy2ByWrtvBl/O2GB1FRERERAJgR7WbH5du1Wy54hcVomKItIQIzrd14umPiqjTci4iIiIiIe/7JVuJibQwsGuC0VEkBKgQFcOcZ+tEbZ2Hd77dYHQUEREREWmmL+duxto3hTCLhuXK/qkQFcNER1q44tgcXvqsmPLKOqPjiIiIiEgTVde6mVNQxhEDNVuu+EeFqBhq9IHpZCRH8tJna42OIiIiIiJN9G3BViLCzBzUQ8NyxT9h/hxks9onAGOBAcAPTpfDttv+i4GbgRxgM3Cb0+WYEdio0hZZzCauPSGX614o4PRDM8hOizY6koiIiIg00hdzN2Prl0KYRf1c4h9/f1PWAw8Aj+2+w2a1Xw7cCpwPxAMHAD8FKqC0fUN7JnJwz0Se+Wi10VFEREREpJE0LFeawq8eUafL8R6AzWrP2XW7zWq3APcCFzldjl8aNm9q+CPit6uP78K5j8xlflEF/bvEGx1HRERERPz03WINy5XG86sQ3YdeQAbQ1Wa1rwAigc+B650uxz4XiHS73bjdbr9vtPPYxpzTnoVae3VJj+S4A9N5ctYqnr0yv0UXQQ61tjKS2qpx1F7+a2pbqW1FRIz3+R8aliuN19xCdOdqtacBhwB1wKvAFOCMfZ1YUFBARER4o29YUFDQ6HPas1Bqr5E5Hmb/Ws2MT+YyKKe5v5qNF0ptZTS1VeOovfzX2LaqrdWM2yIiRto5LPfBsb2MjiIhprmf9isa/r7f6XKUANis9nuAb2xWu9npcnj2dmJ+fj7R0VF+38jtdlNQUEB+fj4Wi6VZoduDUG2vc7YU8+GCLZxl79tia1CFalsZQW3VOGov/zW1raqqqoFZwQsmIiL7pGG50lTNLUSXAtWAdw/79llFWCyWJn0wa+p57VWotdcFR3Tmgx9L+PjXzZwyLKNF7x1qbWUktVXjqL3819i2UruKiBhLs+VKU/m7fEtYw7FhgNlmtUcBHqfLUWWz2l8BbrNZ7b8D9cAdwIdOl0MP7kijxUWFcenRWTzvWMMxQ9KIjtSHTBEREZHWqLrOzZxFZTygYbnSBP72iN4B3L3Lz1XA14ANuB54ClgB1AKzgesCF1Ham1MOyeBN13ped63nkqOyjI4jIhKSGr5EfgTf8mpmYCYwwely1BgaTETajO8LthKuYbnSRP4u3zIRmLiXfVXApQ1/RJotPMzM+OO6MOmt5Zx0cAfSEiKMjiQiEor+CRwG9MM3meAs4H7gBiNDiUjbMfu3Ug3LlSbTb420SkcMSKFbZgxTPy02OoqISKi6FJjkdDnWOV2OTfi+UL7IZrXrvV9Emm1jWQ2uhVs4ZVgHo6NIiGr5NTJE/GAymbjuhFzGTV7AmBGZdMuMMTqSiEjIsFntSUA28Psum38Ddm4v2tN5WuM7uNRe/lNb+c+otpr53Xp6dY6ld+eYkPrfSb9b/gv2Gt8qRKXVGtA1nsP6pfDUh0U8dmm+0XFEREJJfMPf5bts27rbvr/RGt8tQ+3lP7WV/1qyrercXt79tooxB0WwYMGCFrtvIOl3y3/BWuNbhai0ahOOy2HMv+fy09JyhvZMNDqOiEio2LnOdyJQ2vDvpN32/Y3W+A4utZf/1Fb+M6KtPvplE+Hha7hg9AAiwkJrtL9+t/wX7DW+VYhKq5adFs3ph2by5H9X8er1AzCb97k8rYiIAE6XY6vNal8DDAIKGzYPxtcrumZv52mN75ah9vKf2sp/LdVWXq+Xmd+VcOqwDKIjGz+CorXQ75b/grXGd2h9hSHt0sVHdmZ9WQ0f/7rJ6CgiIqHkBeB2m9XeyWa1p+ObrGia0+XwGBtLRELZ/KLtLFtXyanDMo2OIiFOhai0ekmx4Vx8ZBbPfbKGqho9WC4i4qf7gDnAQmA5sAjfki4iIk329pz1jOqfQnqilteT5tHQXAkJZ4zI5L3vNzLti7WMH51jdBwRkVbP6XLUA9c0/BERabbSbbV8MXcLz17Zx+go0gaoR1RCQkSYmZtOyeU15zqKNlUZHUdERESk3Xnv+43kZUYzsOteJ98W8ZsKUQkZw3onM7JPMg+/txKv12t0HBEREZF2Y92Wat5wredcWydMJk0eKc2nQlRCyvUn5fLHygq+nLfF6CgiIiIi7UK928vdM5YzpFsCxwxJMzqOtBEqRCWkZCZHcslRWTz2n1VUauIiERERkaB75cu1rNlcze1n5qk3VAJGhaiEnHMO60hUhJmXPis2OoqIiIhIm7ZwdQUvfFrMXWPySIkP3XVDpfVRISohJyLMzM2ndOV113oK11caHUdERESk1dlcUcuKDZXU1Td96eDKGjd3zVjOacMzGJ6fHMB0Ilq+RULUwb2SsA9O484Zy3j5uv5EhOk7FREREWm/3B4vC9dU8P3iMr4t2Mri4h0AWMyQnRZN14xoemfFcvTgNDqnRu33etW1bh6YuYLwMBNXHa+l8yTwVIhKyLrxlFzOfWQez368mmtPzDU6joiIiLRBvxWWM9VRjNvrJSrcTFSEhdhIC2eN7EivrFij4wFQss3Dv59YyIqNVQzqFs/Rg9K4++zuZCRFUFRSxcoNVazYWIVzwRae/WQNA7vGc9yB6RwxMJX46L+WA5U1bt79bgMznOuJCDfx6CW9iQq3GPTKpC1TISohKy4qjHvP6c4VkxcxrHcyQ3smGh1JRERE2oiaOg9TZq/mDdcGTh2WQafUSKprPdTUeVhTWs1FT87ncns254/qhMVs3AQ+roVlTPqomkN6J/Ps+L4kxPz1433fnHj65vxv3c9VG6v4+NdNvPhZMffPXEFmUiQ56VHkpEcRFWFh1o8lxEZZuHJ0NqMPSCdco84kSFSISkgb2DWBsYd34t43lzPjpgEkxughehEREWmepet2cPeMZVTWeph8ZR8Gd0v42zHfLCrj/94uZM6iMiae3Z2stP0Pdw2kereX5z5Zzeuu9Zw2OJxrz+hOWNj+P9rnZkQzfnQOVxyTzfL1lRSVVLG6tJrVm6pYVVLNtSd24ZghaYRZVIBKcKkQlZB36dFZ/LBkK/e/s4L7L+ipacVFRESkyb5fXMZNLy3h2APSuf6kXGKj9jwsdWSfZN64aSD3z1zBuY/MZdJ5PRjZN6VFMtbUebh52mKWr6/k6ct7Y9lR1OjPP2aziZ6dY+nZuXUML5b2R191SMgLs5i559wefLd4K2/P2WB0HBEREQlRazdXc+dry7jMns0dY/L2WoTulBwXzoNjezLumGxue3UpPy0tD3rGereH26cvZe3mGl65bgADu8bv/ySRVkiFqLQJXdKjueec7jw+q4jP/ig1Oo6IiIiEmOo6N/94ZSlDuicy9vBOfp9nMpk457BOXHxUFjdPW8z8ooqgZfR4vPzrrUIWF+/g6XF9SE+MCNq9RIJNhai0GaP6p3LraV2Z+Ppyflq61eg4IiIiEiK8Xi//fnclVbVu7hqT16THfC46ojOnDc/kuqkFLF23IygZH/lgJd8v3srT4/rQMSUy4PcQaUkqRKVNOfmQDC6zZ3PztCUsWr3d6DgiIiISAt7/oYTP527mwQt7ERfdtClUTCYTVx+fw5EDU7lmSgFFm6oCmnHK7DV8/GspT16eT25GdECvLWIEFaLS5ow9vBMnH5LBdS8UsGJDpdFxRERE2iyPx2t0hGZbtGY7j7y/kjvPzCMvM6ZZ1zKZTNxyWjcO7JHAhGcXsXZzdUAyvvR5MTO+Xs+jl/Smd1ZcQK4pYjTNmittjslk4toTurC9up6Ln5zPP07vxjFD0o2OJSIi0mbUu728/EUxL32+ljCLiYToMBJiwkiJC+eCwzsxtGeS0RH9UlfvYdJbhZwwtANHDU4LyDUtZhMTz+7O7dOXMf65RTw/vi8ZyU0fRvvql2uZ9lkxj1ySv8dlZERClQpRaZPMZhN3nJlH35x4Jr1VyC/LtnHTKblERex79jsRERHZt6JNVUx8fTkbt9Zw77k9SIkLo7yynm2V9SxfX8m1Uxdz8iEduPr4LsREtu733enOdWyrrOeq43MCet0wi5lJ5/XglpeXMP65RUyZ0Je0hMZPLPT61+t43rGGhy7qzdCeiQHNKGI0Dc2VNstkMnHqsAymXdufuSu3Mfbx+Swu1nOjIiIiTeH1enn3uw2c/+g8MpIieP2mgRw5MJUheYmM6p/KSQdncOPJXXnx6n78VriNsx+ey6/Lg7+cSVMVlVTx0mfF3HJqV+KiAt83Ex5m5oGxveiUEsmE5xaxpaKuUee/M2cDz3y0mgfG9mJY79DoYRZpDBWi0ub16BTLK9cPoF9OHGMfn8+Vkxfy1bzN1LtD/7kWERGRlvKGaz1Pf7ia207vxv0X9CQpNnyPx/XJiePV6wdw5MBUrpqyiNe/XtfCSffP4/Fy3zuFjOybgrVfStDuExlu5qGLepEcF86FT8zjm4Vb9nvOloo67pqxjMdmreK+C3oyok9y0PKJGElDc6VdiIm0cOdZ3Rl7RGdmfruBe98sJD5mFccdmE7f7Fi8VSpKRURE9mblxkomf7yae87pwREDU/d7fGS4mauP78Lgbgnc+vISwi1mzhiR2QJJ/fOfH0tYvr6St87vGfR7RUVYePLyfF79ci23vbqUYb2TuPHkrmTu9tyo1+vlvz9t4sn/rqJLh2imXz+AvI7NmzxJpDVTISrtSk56NDec3JVxx+Tw8S+b+Gr+Zt76ZgPbq91kfvYHPTvHkpEUQXpiJB0SI+iQFEFOehTpCRFNWlNMREQk1NW7PUx8Yzm2/il+FaG7GtEnmfsu6Mltry4lzGLilGEZQUrpv03ltTz5YRHXn5jbpOc2myIizMylR2dz9OA0/v3uSsb8+w9OGNqBqHAzHq8XjxcWrt7O8vWVXHVcDqcckoHZrM8d0rapEJV2KTbKwhkjMjljRCb19fV89f183DGdWbGxmpLyWlYu3cqm8lo2bK2lps5DTKSZLh2iye0QzSG9khjZJ7nJ64yJiIiEkle+XOcr3i7Pb9L5h/VL4V/n9uDOGcsIt5g4fmiHACf0n9fr5cF3V9A7K5YThrb8jPo56dE8NS6fz/7YzBdzN2MygdlkwmyCXp1jue/8nqQntkxxLGI0fZKWds9kMtEhwUy/fqlYLH+d3c/j8bKpvJaiTVUUlVSzfP0OnvqwiElvFXJwryQOH5DCkQNTNRuviIi0SYuLt/PiZ8U8dFEvEmP2/EyoP44YmEqd28M9bxQSEW7m6AAtldJYn/6+mZ+WlfPGTQMNG+lkMpk4enCaYW0g0lqoEBXZB7PZREZyJBnJkQxteIzkVo+Xuasq+HLeZiZ/vJqpjmJuODkXa99kDd8VEZE2o6bONyT3+IPSOTS/+RPmHDMkndo6L/e8sZzkuHAO6tGyy5FsrqjlofdXMmF0Dp1To1r03iLyd5o1V6SRzGYTg7slcOPJXXn/n0M4YWg6d0xfyg0vLqa4tNroeCIiIgHx+tfrqKrxcO0JuQG75okHd+CiIztzy8tLWLZuR8Cuuz++Ibkr6ZYZzRmHtp5Jk0TaMxWiIs0QGe6bfODNWwZhNpk466E/eHvOeqNjiYiINEtljZvXv17PZfYsYqMC+/jJJUdlcdSgVK6bWsCGspqAXntvPv9jM98v3sqdY/I0CZBIK6FCVCQAOqdG8cglvbnnnB489d8ipsxejderJWFERCQ0vffdRmKjLBwzJPDPMZpMJm45tRs9O8dy7dQCtlXWB/weu9pSUce/31vJ+NE5ZKdFB/VeIuI/FaIiAXTEwFSeHNeHt77ZwIPvrsTtUTEqIiKhpbrOzWtfr2Ps4Z0JswTno2KYxcR95/ckJtLC9S8UsKPaHZT7uD1eJr1dSG6HaM5sReuYiogKUZGAG9wtgecm9MW5YAt3vraMunqP0ZFERET8NuvHEsLMJo47KLjLm0RHWnjs0t5U13m4bmpwitFnPipi0ert3HtuDywakivSqqgQFQmCnp1ieeGqfixas53bXl2qnlERaRcWF2/n87mbjY4hzVBX7+HVL9dx/qhORIQF/2NiUmw4z4zrw44aN9e/WEBlTeCK0Q9+2Mg7czbw0MW96JgSGbDrikhgqBAVCZKstCieG9+X+UUVTJm9xug4IiJBV7ihirtnLOP3FduMjiJN9NEvm6h3eznpkA4tds+kuHAmX9GH7VVurn+hgKoAFKM/LS3n3++t5K6zutO/S3wAUopIoKkQFQmizORI7r+gF6851/HZH6VGxxERCarjDkznXFsnbpm2hDWlVUbHkUaqd3t55cu1nGvrSFR4YGfK3Z+kuHCeuaIP2yrrufr5AjZubfpsuqs2VvGPV5b4ZucdHPjJlkQkMFSIigTZkLwEbjw5l3+9VcjSFlwzTUTECFcck82BPRK5/oXFQZ8NVQLr099Lqahyc+owYyb1SY4LZ/KVfYmJNHPB4wv4Y3Xjf39+WlrOVVMWMaJPMhcf2TkIKUUkUFSIirSA04ZncuwB6dz80hK2bq8zOo6ISNCYzSbuPjuPhOgwbn1liSZsCxFer5fXnOs4c0RmwNcNbYzkuHAevzSfC0Z14jlXLY98sIqauv3/DlXXuXn0g5VcO7WA0Qemc+eYPEwmTU4k0pqpEBVpITednEuHpAhuf20ZHk1eJCJtWFS4hYcu7sW6zTU8/MEqo+OIH34r3EZRSRWnDc8wOgpms4lzD+vIrcdE8v3ics59ZC4vf7GW1Zv+Ptzb6/Uyv6iCsY/NZ86iMqaM78v40TmEt8BESyLSPGFGBxBpL8LDzNx/QU/O+vcfzPxuA2eO6Gh0JBGRoEmNj+C+C3pw8ZMLOM/Wkey0aKMjyT68+c16jhqcRmp8hNFR/tQ1zcIr1+Xz/o+b+HLeZiZ/vJoenWI4uGcSmytqWbmxiqKSKqpqPZwyLINrT+hCTKRxvbki0jgqREVaUFpCBDed0pX7Zq5geO9kstKijI4kIhI0fXPiGZKXwJuuDdx8alej48herN1czTcLy3j5uv5GR/mb2CgLYw/vzNjDO7N2czVfzdvCbyu2kZEUwfEHpZPbIYZumdGkJbSeAlpE/KNCVKSF2Yek8cW8zdz71nKeu7IvZi2wLSJt2DmHdeL26UsZd0w2CTH62NEavfPtBvrnxtM7K87oKPvUOTWK80Z14rxRnYyOIiIBoAH0Ii3MZDJx62ndWLGhirfnbDA6johIUB3aO4kOiRG8/8NGo6PIHlTWuJn1YwlnjdTjIiLSslSIihggLSGCW07tyjMfr97j5AsiIm2F2WzibGtH3v5mvWbQbYU++nkTcdEWDuuXYnQUEWlnVIiKGOSoQakM753Ev94s1Cy6ItKmHXdgOrX1Xr6Yu9noKLILj8fLW3PWc/rwTMIsekxERFqWClERg5hMJm45rSsrNlby4c+bjI4jIhI0UREWThueweuu9Xi9+uKttfhhyVY2bq3lpEM6GB1FRNohFaIiBkqNj2DcMdk8/VER5ZV1RscREQma0w/NpHB9Jb+v2GZ0FGnwhms9ow9MJzEm3OgoItIOqRAVMdipwzLpkBjBlE/WGB1FRCRo0hIisA9J4/Wv1xsdRYDFxdv5eVk55xymSYpExBgqREUMFmYxccup3Xj/h40sLt5udBwRkaAZM6Ij3ywqo3RbrdFR2r1X/r+9Ow+Psyr/P/6eLfueNGnTJUnbtE2bpottaatMBwSnIouAyFJ+gH5R+AmCKCIiCoILsigCIooLKAgKFhFQR1mG0Mdi6gAAIABJREFUAVoo/UKXhKRtuqRLurfZ95n5/jEphkLTmWRmnpnM53Vdvdo855l57tzXdM7cc55zzsuNnFSVT8moVKNDEZEEpUJUJAZUlWXy6Y+N4s7lW7VwkYiMWFPGpjEuP4VXqw8ZHUpC276/k1fWHeQS7ccpIgZSISoSI67+TAnb9nbywmotXCQiI5PJZMIxMw/3ehWiRvrjK43ML8+mYnyG0aGISAJTISoSI/IybVz56fHc/3wDLR19RocjIhIRJ83MY3V9i97nDLKvuZsXVu/n0k+ONToUEUlwKkRFYsg5i0ZTkJXEb/6z0+hQREQiYvr4DPIzbbz+3mGjQ0lIT3p2M3VsOh+blGV0KCKS4IIqRB1251UOu3OVw+7sctid7mOcU+SwOw857M41YY1QJIFYLSauO6uUp17fw7a9nUaHIyISdmaziSWVubyy/qDRoSSclo4+lq/cy6Unj8VkMhkdjogkuGBHRHcDdwA/G+ScB4B1w45IJMHNL8/mxBm5/Py5bUaHIiISEY6Z+bxZ10Rnt9foUBLKU2/soTA7GfuMXKNDEREJrhB1e1zL3R7XcmDvR7U77M4zgQLgkfCFJpK4rj2jhFUbm1lZp1vXRGTkmTMxi+QkCys3NBkdSsI42NrDE682csnJxZjNGg0VEeNZh/sEDrszi8BI6WnAomAf5/V68XqD/yb0yLmhPCaRKV/Bi8Vcjc6xccGJo/nps9t4bGIGVktsTOeOxVzFMuUreEPNlXIbn6wWE0tm5OJef4iTq/KNDsdwfr+fF9ccpKvXx8KpOYzKTgr7Ne5+ZhtlRWmc9rFRYX9uEZGhGHYhCvwE+IPb49rgsDuDLkRra2tJSrKFfLHa2tqQH5PIlK/gxVquFozx8+ybXTywfC2nVIT+fyWSYi1XsU75Cl6ouerp6Y1QJBJpjqp8vvf4Jnr7fNissfFlmxHau7z8+KnNvF57mOx0G7f/eTPlxWksnpbDZ+YVUlqUOuxruNcf4rWaQzz+jVkaDRWRmDGsQtRhd34cWALMDvWxFRUVpKamBH2+1+ultraWiooKLBZLqJdLOMpX8GI5V1f37ef+57dz2WlTyUk3vhiN5VzFIuUreEPNVWdnF/D3yAUmEbOgPBu/38/q+mYWTUvMOYub93Rw4yMbsFpMPPq1KiaMSmH7/i5W1DXxWs0hnnxtDzedN5FPD2MUs7WzjzuXb+HyT42npHD4Ra2ISLgMd0T0VGACsN1hdwKkAmkOu3MPMN/tce041gMtFsuQPpgN9XGJSvkKXizm6swFRSxfuY+H/93IjZ+baHQ474vFXMUy5St4oeYqHvPqsDs/A9wAVAFeYCXwdbfHtWnAOYuAB4FpwBbgGrfH9VK42mNBss3M4mm5vLL+UEIWoq53DvDDpzbzyap8vnVuGSlJgddySWEqJYWpXGgfw7Nv7eWHf9nM+m2tXHdW6ZBGju9/voG8DBsXO8aE+1cQERmWYLdvsTrszhQChavZYXemOOzOJOBuYDKBEdHZwPeADf3/boxIxCIJxGw2cf3ZZTz71l427Gw3OhwRCY9sAv3nBGAcgULx/WFdh92ZAzwPPATkAD8G/uawO8eEoz2WnDQzj1erD+H1+Y0OJar2Nffw/Sfrueb0Er53waT3i9CjnXVCEQ9fXckbdU1c8Ysa9jZ1h3Sdtzc189yq/dx8/qSYWWtAROSIYEdEbwZuGfBzJ/Cq2+NyAG1HDjrszmagz+1x7QlbhCIJrqo0E+fcUdz9zFZ+ffUM7f0mEufcHtefBv7ssDvvAa5x2J15bo/rEHAOsNvtcf2q/5THHHbnVcAFBBYHHG77MUV7IcETpmTR0uGlelsLlSUZQ3qOeHIkT096djOlOI3PnlCAz+cb9DFTilP5/TUz+P4Tm1l2z1puOKeMT1blHfdaDfs6+cGf67nIPpryMalxt7CXFnsLnnIVGuUreJFeSDCoQtTtcd0K3BrEeY+gLVxEwu7qz0zgvDvW4HrnAEu14qHISHMSsKu/CIXALbvvHnXOO/3Hw9F+TEYsJFhWYOKFNzZBq/Hz4KOhvdvP8pV7+eInkqipqQn6cZct8PNypplb/1TPc29YuHBBEunJH/5i0u/383q9lyff7mHWOAsLi5uorm4O568QVVrsLXjKVWiUr+BFaiHBcKyaKyIRVpCVxP+cOo77nm/gxBl5pKfE37w4kUTgsDuTgcEqqk63x+UdcP404B7gigHnZAJHVw5NQFGY2o/JiIUEHbt38c7mFiorK4b0+Hji9Xq5+y/rKMpN5uKlVSGvYFs1E85a0sltT27mh//q5dozSpg6No2i7CRsVjPNHX3c8fRWVm1s5oZzyjjtYwVxexeNFnsLnnIVGuUreJFeSFCFqEicOP/E0Tz71l5+/+JOrj69xOhwROSj/RZYNkj7SYAbwGF3TgVeBG52e1x/HXBOK3D05po5/cfD0X5MRiwkuGBKDo++3Eivl2POlRwpunp9vFTby1fPKMNmG9pHsMnFGfzu2pn89j87+dFTW+jo9mEyQX6mjd4+P+MKUvjjN6oYXzAyVsjVYm/BU65Co3wFL1ILCaoQFYkTNquZb5xdxtd/W8cZJxRSMmpkfMgQGUncHtfFwMXHO89hd1YALwG3uz2uh45qXgdcd9SxOcDTYWqPKZUlGdisJtZsbWXh1Byjw4moF97ej8Vswjnn6O8JQmO1mLli6QS+7BxPc0cfew/3sPtwN129Xk6Zla+FiUQkLqgQFYkjC6fm8Inpudzx9BYevHJ63N5yJZLIHHbndAJF6A/cHtcvP+KUZ4C7HHbnl4BHgXOBmcDnwtQeU6wWM3MmZrFqY/OILkT7vH4ef3U3p063kjSEbVg+islkIifdRk66janj0sPynCIi0aKvzETizDfPLqNuZzvPvrXP6FBEZGi+SWC+5k8cdmfbgD8nArg9rsPAGcBVBOZ6fhc42+1xNYajPRbNL89mdX38LqgTjJfWHqSty4u9XGMAIiKgEVGRuDMqO4lrTi/hvuca+HhFLqOyk4wOSURC4Pa4vgB84TjnrCCwJ3dE2mPN/PJsfv5cA03tveSkj7zVc/1+P394ZRfnLi4ixdZkdDgiIjFBI6IiceisEwqZMjadu5/ZanQoIiLDNml0GrnpNv63vsXoUCJiy95ONjV2cO6iQqNDERGJGSpEReKQ2WzipvMmsqL2MK+sO2h0OCIiw2I2m5g3OYtVm0bm7bkra5uYNi6dgizdwSIicoQKUZE4NWFUKpc7x3Pn8q20dPQZHY6IyLDMn5LN6hFaiK6oO8ziaSN3ISYRkaFQISoSx5YtKaYgK4m7lm/F7/cbHY6IyJDNL89mx4Eudh/qNjqUsGrv8rJmayuLp+UaHYqISExRISoSx6wWE7ctm4yn5hDPvKlVdEUkfhXnpTAuP5m3R9io6Or6ZtKSzEyfkGF0KCIiMUWFqEicKytK46bzJvHTv22lbmeb0eGIiAzZ/PJsVm0aWavKrqxrYsGUHKwW7fssIjKQClGREcA5t4AzFhRy46Mbae3UfFERiU/zp+SwelPLiJlq4Pf7WVnXpPmhIiIfQYWoyAhx3VmlZKVZ+f4T9SPmQ5yIJJZ5k7I43N7L5j0dRocSFtv2dbL7cDcLVYiKiHyIClGRESLJaubHl0zh3S0t/OHlRqPDEREJWU6GjUmj03hn88jYT3RFXRNTx2rbFhGRj6JCVGQEGZufwu3Lyvm1awd/eX230eGIiIRszsRM1m5tNTqMsFhZ28QijYaKiHwkFaIiI8ziilx+dMkU7v17g4pREYk7s8uyWLMl/ueJdnZ7eXdLiwpREZFjUCEqMgItqczjx5dM4ed/b+Cp1/cYHY6ISNBmTcxkf0svjXG+n+jqzS0k28zMLMk0OhQRkZikQlRkhFpSmdc/MrqNP73aGPejCyKSGAqzkynOS2ZNnN+eu7L2MAumZGvbFhGRY1AhKjKCHRkZfdi1k2sfrmXP4fgeYRCRxDCrLJO1W+N3wSK/38+KuiYWT8s1OhQRkZilQlRkhLNX5vHEN2dhMpm44K41/HXFHnw+jY6KSOyaMzGLNVvid0R0x4EuGg91s3BattGhiIjELBWiIglgdG4y914+jevPLuPBf2zny7+o4blV+2hq7zU6NBGRD5lVlsm2fZ0cbovP96h121oZl59MYXay0aGIiMQsq9EBiEh0mEwmTp9fyMKpOfzhlUZ+7drBj57azLzJ2dhn5DEmL5msNCtZqVYyUi109/po7eyjpaOPlk4vbZ19tHX2sXVHLy9v3U5ykoVRWUmMyk6iMDuJCYUpZKToLUVEhq+0MJXsNCtrt7bimJlndDghq25oY2apFikSERmMPjWKJJiCrCS+flYp151Zwns72nhp7SGeeXMvh1p7ae7ow3vUbbvpKRayUq2kp1hITzbj6/VSRA89fX7e2tjE/uYeDrb2YrOYWFKZx+nzR7FgSg4WsxboEJGhMZlM788Tjc9CtJWzTigyOgwRkZimQlQkQZlMJmZMyGTGhEyuOaMECCyw0dnjo7Wjj+QkMxkp1g+s+Oj1eqmurqaycjIWi+X9431eH2u2tvL82/v51qMbyUyxcPaiIi45eSxJVs0AEJHQzZmYxYtrDxodRsg6ur3U7+6gsiTD6FBERGKaPiGKyPtMJhNpyRaKcpPJSbcFve2A1WJm3uRsbr1wMv+8ZR5XLJ3Ac6v2c9m966nf3R7hqEVkJJpVlkndznY6u71GhxKS2h1t2KxmyovTjA5FRCSmqRAVkbBKT7Fw5gmFPH59FVPHpnPZvet5/NVGrdQrIiGZNi4dq8VE9fY2o0MJSXVDGxXj07Fa9BFLRGQwepcUkYjISLFyy4WTuW1ZOY+8uIuv/rqWtq4+o8MSkThhtZipLMmIu/1E1ze0UjlBCxWJiByPClERiaiTq/J54puzaO3s41oVoyISgjll8bWfqN/vp3p7GzM1P1RE5LhUiIpIxBVkJXH/FRV09/r42sN1tHfF15wvETHGrLJM1je00ueNj1v7Gw91c6i1l8oSjYiKiByPClERiYrsNBsPXDmdzh4vX3u4VsWoiBxXZUkmPX0+NjbGx6Jn1Q1tFOUE9lcWEZHBqRAVkajJSbfxiyum097t5brf1tLVq2JURI4tPcVCeXE6a7fGx+251dtbmanRUBGRoKgQFZGoysmw8eCV0znc2suPn9qC3x8ft9yJiDFmlWWyblucFKINbdo/VEQkSCpERSTqcjJs3PWFaXiqD/Pn1/YYHY6IxLCqkkzWx0Eh2t3rY8Ouds0PFREJkgpRETFEaVEqt140mfueb+B/65uNDkdEYtTM0kz2Nfew93C30aEMasPOwDzWqWPTDY5ERCQ+qBAVEcMsqczjspPH8u0/bGRPjH/IFBFjFOUkUZidFPO3567f3sq0sekk2/TRSkQkGHq3FBFDXf6pcVSWZHLDIxu0eJGIfIjJZGJmaSbrGmK7ENX8UBGR0KgQFRFDmc0mbrtoMu1dXu5/rsHocEQkBlWVxv6CRdUNrZofKiISAhWiImK4jFQrty8r55k39/Hae4eNDkdEYkxVaQYbd3XQ1RObd03sa+5mb1OPRkRFREKgQlREYsL0CRlc4RzPD/5cz4GWHqPDEZEYMqU4HasZ3tvRbnQoH6mmoY28TBtjcpONDkVEJG6oEBWRmHHxScWUFaVx+5834/Npf1ERCbBZzUwbnxGz27jU7mxn+vgMTCaT0aGIiMQNFaIiEjMsZhPfv3Ay1Q2tPPWG9hcVkf+K5XmitTvamDZO27aIiIRChaiIxJSi3GS+fd4k7n++gfrdsXkbnohEX1VpJusbWvH7Y+tuCb/fT+3OdirGa36oiEgoVIiKSMw5ZVY+p84u4JY/1dPb5zM6HBGJATNLMmlq72PHgS6jQ/mAxkPdtHT0UaERURGRkKgQFZGY9I3PltLa2cfD/95pdCgiEgPyMm2ML0iJudtz63a2MyrLRkFWktGhiIjEFRWiIhKTMlKtfPf8yfzxlUbWx/hG9iISHZUlsbdgUWB+qG7LFREJlQpREYlZ88uzOXdxEbf+qT5m9w8UkeiJxQWL6na2UzFet+WKiIRKhaiIxLSrPzMBkwnuf3670aGIiMGqSjPZsreT1s4+o0MBAgsV1e1s00JFIiJDoEJURGJaSpKFWy6czPKVe1m1scnocETEQBNHp5GWZKG6oc3oUID+hYo6vdq6RURkCFSIikjMm1mSySUnF3Pbk5tjZiRERKLPYjbF1DzR93a0UZidRH6mFioSEQmVClERiQuXnzqO3Awbdy3fanQoImKgmaWZrI2RQrRuZ7tGQ0VEhkiFqIjEBZvVzPcvmszL6w7y4tqDRocjIgaZVZpJdUMrfV6/0aFQu0PzQ0VEhkqFqIjEjYmj0/jKaRO44+ktHGjpMTocETFAZUkm3b0+6ne3GxpHYKGidio0IioiMiQqREUkrlxw4hjKi9P4wZ834/cbPyIiItGVnmJh8pg01m419vbcnQe7aOvyag9REZEhUiEqInHFbDZxywWTWbutlb+u3Gt0OCJigFllxu8nWrejnaKcJPIybYbGISISr1SIikjcGZ2bzLfOKePnf29gy54Oo8MRkSibVZbF2q2tht4VUav9Q0VEhkWFqIjEpaUfG8XJVXnc/Ngmunt9RocjIlE0qzSTfc097Dls3Fzx2h1aMVdEZDhUiIpI3PrmOWV09Xi577kGo0MRkSgqyk2mKCeJtdtaDLm+z+enblc7FZofKiIyZNZgTnLYnVcBlwJVwJtuj8vRf7wQ+BlgB3KABuBHbo/rTxGJVkRkgIwUKz+4eAqXP1DNwqnZnDgjz+iQRCRKZpVmsm5rK0vnjor6tXcc7KK9y6sVc0VEhiHYEdHdwB0Eis6BMoA1wGIgC7gG+LXD7lwUtghFRAYxfUIGVy4dz21PbmZfc7fR4YhIlFSVZbLWoAWL6na0MyY3mZwMLVQkIjJUQY2Iuj2u5QAOu3PCUce3AHcNOPSyw+5cBSwCVoYrSBGRwVzsKGbVxmZuebyeB66cjsVsMjokEYmwWaVZ3PO3bbR19pGRGtTHmbCp2dFGxXiNhoqIDEdY37kddmc2MAf40fHO9Xq9eL3eoJ/7yLmhPCaRKV/BU66CF8u5+t4FZVx6bw0P/bOBK5eONzocILbzFWuGmivlNnFNGpNGWpKF9Q1tLJqWE9Vr1zS0ctLM/KheU0RkpAlbIeqwO63AHwGP2+N68Xjn19bWkpQU+i0ttbW1Q4gucSlfwVOugherufriYjM//fduMv0HmTU+uiMkg4nVfMWiUHPV09MboUiiw2F3XgE8BFzn9rjuHXB8EfAgMA3YAlzj9rheClf7SGC1mKgsyWDdtpaoFqI9fT427Grnq6eXRO2aIiIjUVg+qfUXoY8B6cDpwTymoqKC1NSUoK/h9Xqpra2loqICi8UytEATiPIVPOUqeLGeq8pK6LLu5vcvNfLIgmmMzQ/+PSYSYj1fsWSouers7AL+HrnAIshhd44BbgCqjzqeAzwP3AQ8ApwH/M1hd05xe1y7h9sejd8tWqpKM3l3S3RXzt3U2I7X59fWLSIiwzTsQrS/CH0CyAdOd3tcncE8zmKxDOmD2VAfl6iUr+ApV8GL5VxdfNJYqre3c9Mf6/nNNZWk2IyPM5bzFWtCzVWc5/UXwO3AZUcdPwfY7fa4ftX/82P9q9dfQGDRwOG2H1O8TZupLEnnj6800t3Ti9USnR3p1m9rYfKYNGyWod9KrlvKj0+5Cp5yFRrlK3iRnjYT7PYt1v5zrYDZYXemAD7ADzxJoAg9ze1xdYQUpYhImJlMJr57wSQuu3c9d/11KzefPwmTSYsXSWxx2J3nArluj+sRh9152VHNVcC7Rx17p/94ONqPKd6mzZh7/XT3+fjXa+spLYjOlxKvr+tmTIaJ6urq4598DLpdP3jKVfCUq9AoX8GL1LSZYEdEbwZuGfBzJ/Bq/7FzgC5gn8PuPNL+mNvjujLI5xYRCauMFCs/uXQqX7xvPVPGpnP+iWOMDkkShMPuTAYGq+Q6gUzgbmDpMc7JBJqPOtYEFIWp/ZjicdpM+avVtFsLqKwcHZXr7XphLV88ZSyVlQUhPzYW8hUvlKvgKVehUb6CF+lpM8Fu33IrcOsxmjXUICIxZ9KYNG5bVs6Nj26kpDCVhVOju6qmJKzfAssGaT8JuAh4xO1xbTjGOa0E7jQaKKf/eDjajykep83MnpjFum1tXLQk8tc/3NbLroPdVJVlDev31e36wVOugqdchUb5Cl6kps3EzrKSIiJhtqQyjyuWjuemP2zkd9fMpLQo1eiQZIRze1wXAxcPdo7D7nwESHfYnf+//1AeMNdhd9rdHtc5wDrguqMeNgd4uv/fw20fUeZOyuInf92Cz+fHHOE9hKu3t5GVamFCgbELoYmIjAQqREVkRLv05GK27OngG7+r43fXVpKdFvr8N5Ewmw8M/Lp4OfBPAosXATwD3OWwO78EPAqcC8wEPhem9hFl7qQsmtr72Lyng/LiyK5kW9PQyoySTM07FxEJg+gsMSciYhCTycR3Pj+J7DQr3350I31en9EhSYJze1z73R7XniN/gB6g1e1xHepvPwycAVxFYK7nd4Gz3R5XYzjaR5qcdBtTitNZXR/5bVyqG9qonJAR8euIiCQCjYiKyIiXbDNz5xem8oWfr+eHf9nC9y7QSroSO9wel+Mjjq0AZg/ymGG1jzTzy7N4e1MzF9ojtzCZz+enZkcbyxxa/ExEJBw0IioiCaEgK4l7v1SBp+YQD/1rh9HhiEgYzZuczbubW+jz+iN2jW37Omnv8jJdI6IiImGhQlREEsak0Wnc/cVpPO5u5OkVe4wOR0TCZPbELLp6fdTubIvYNaob2pgwKkXzzEVEwkSFqIgklDkTs7h9WTn3PLMN9/pDRocjImGQlmyhckIGb286evvU8KnZ3kplSWbEnl9EJNGoEBWRhHNSVT7XnVXCdx/byLtbIr/AiYhE3vzybFZHsBBd39BGZYluyxURCRcVoiKSkD7/iTEscxTz9d/UUbO91ehwRGSY5pVns25bK1293rA/d0e3ly17OrRirohIGKkQFZGEdcXS8Zx5QiHX/rqWTY3tRocjIsNQWZKByWRi/bbwzxOt3dGGzWpm8pi0sD+3iEiiUiEqIgnLZDLxtTNL+OTsAq7+1Xts29tpdEgiMkRJVjOzyzIjMk+0uqGNaePSsVr0sUlEJFz0jioiCc1kMvGtc8pYODWHqx6qYeeBLqNDEpEhmleezer68Bei72xpYVapFioSEQknFaIikvDMZhPfPX8yVWVZXPlgDQ37NDIqEo/ml2dTu6ONtq6+sD1nV6+Xd+qbWTQtJ2zPKSIiKkRFRACwWkzcvqycuZOyuOLBGjbv7jA6JBEJ0dSx6aQlW3h3S/gWIHunvgWrxUyVRkRFRMJKhaiISD+rxcQtF07m4xU5XPlgDRt2agEjkXhiMZuYOykrrPNEV25oYl55FjarPjKJiIST3lVFRAawmE1857xJnDI7n688VEN1g7Z2EYkn88uzWbWxKWzPt6K2icXTcsP2fCIiEqBCVETkKGaziRvOKePMBYV85aH3eO29w0aHJCJBOnF6Hlv2dIZlrvfOA13sONDFYs0PFREJOxWiIiIfwWQyce2ZpVy5dDw3/H4Dz6zca3RIIhKEMXnJVJZk8J81B4b9XCvrmpg4OpWi3OQwRCYiIgOpEBURGcRFS4q5bdlk7vnbVh7653b8fr/RIYnIcXxqdgH/WXNw2P9fV9QdZtFUjYaKiESCClERkeM4dXYB9315Ok+9sYdbn6inq9drdEgiMohPzspn275ONu8Z+urX3b0+Vte3sEjzQ0VEIkKFqIhIEOZOyuI3X62kuqGNLz9Qw+5D3UaHJCLHMCo7iTkTs/j3uweH/BzvbmnBbILZE7Vti4hIJKgQFREJUllRGo9+bSaF2Ulc8rN1YV2ZU0TC65TZ+by45sCQb89dWdfEvMnZJGnbFhGRiNC7q4hICDJSrdx52VQusI/h2ofrePSlXXh9mjcqEmtOnpnP7sPd1A1xP+CVdU0sqtD8UBGRSFEhKiISIrPZxP+cOo67vziVx9yNfOWXNew62GV0WCIyQF6mjXmTs4e0em7joS627evUti0iIhGkQlREZIg+XpHLE9+cRUaKlWX3rOXZt/ZqVV2RGHLK7HxeXBv66rkr6pooLUylOC8lQpGJiIgKURGRYSjISuLuL07l62eVce+zDXz9t3UaHRWJEY6ZeRxo6WV9Q1tIj3u95jALNRoqIhJRKkRFRIbJZDJx5gmFPH59FT4/nH/nGh78x3Y6urXNi4iRstNsnDAltNtz63a28eaGJs5cMCqCkYmIiApREZEwKc5L4edfquAnl03l5XUHOf+udazc3IdPixmJGOZTcwp4cc1Bunt9QZ3/q3/t4JTZ+Uwekx7hyEREEpsKURGRMPt4RS5PXD+LC08czROrerjwnvX8/a199PQF90FYRMLnpJl5JFnNPPLSzuOeu25rK29uaOJLzvFRiExEJLGpEBURiQCb1cxFS8Zwx7mpnPaxAh78x3bO/uE7PO5upKWjz+jwRBJGSpKFGz9XxqMvN7J5d8eg5/7yX9s5bd4oSkalRik6EZHEpUJURCSC0pJMXHpyMX+7eQ5fOHUcT6/Yw6dvXc2Nj27g1epD9GqUVCTiFk3L5dTZ+fzwL5uPue/v25uaWbu1lctP1WioiEg0WI0OQEQkEaTYLHxu8WjOWVjEmq2t/GP1fm59oh6r2cQnpudywtRsFpTnkJdpMzpUkRHpujNL+fyda3j6jT2cf+KYD7T5/X4e+ud2zl5YyJi8ZIMiFBFJLCpERUSiyGw2MXdSFnMnZXH9OaW8/l4Tb7x3mPufa2B/Sz3lxWnMmZjFlOJ0yovTmDg6jWRb9G9e8fu7Fpb4AAAIp0lEQVT9+Pzg8/nx+v3gD6wObDGD2WTCZAr8LBIvcjJsXHdWKXc8vYUllXmMzv1vwflGbRMbd7Xzk8umGhihiEhiUSEqImKQFJuFU2blc8qsfPx+P1v2drJqYzM1Da085m5k+/5OTCYYm59CUU4yhdlJFOYkUZCZRFqKhbQkM6nJFlL6C1U//y0ge3p9dPX66Orx0dnjpaPbS3uXl/ZuL22d//25oztwrKvHR3evj+6+wN993uOv9Gu1mLD1/7Faze//23bk31YzVrMJqyXwx2IOFLBH/p47KZsL7WOOex2RcFk6t4B//u9+fviXzZw2bxS7Dnax62A3qzY2cd4nRlOQlWR0iCIiCUOFqIhIDDCZTEwancak0WlAoDjr6vGyeU8HDfu62Nfcw96mbuobO3irtYnOHh8d3V46u7109vgCI5SAyWzCBCTbzKQmWUhJMpNsM5OebCE9pf9PsoWCLBtpyRbS+o+nJllItplJsppJsZmxWf9bOJr7n9PvB6/fj98HXp+fPq+fPp+P3j4/PV4/fX0+er1+evoCxwLtfvq8gcLW5w88zu/34/VBgW5DligzmUzc+LmJfOWX7/Gbf+9kbH4yY/NTWOYo5rMLi4wOT0QkoagQFRGJUSlJFmZMyGTGhEyjQxEZMYrzUvjbd+YaHYaISMLTqrkiIiIiIiISVSpERUREREREJKpUiIqIiIiIiEhUqRAVERERERGRqFIhKiIiIiIiIlGlQlRERERERESiSoWoiIiIiIiIRJUKUREREREREYkqFaIiIiIiIiISVSpERUREREREJKpUiIqIiIiIiEhUqRAVERERERGRqLIadeHOrq6Qzvd6vfT09NLZ2YXFYolQVCOH8hU85Sp4ylVolK/gDTVXofYlMjj1zZGlfAVPuQqechUa5St4ke6bTX6/f6ixDclV139vPLA9qhcVEZGRbsIv7r5th9FBxCv1zSIiEgGD9s1GjIjuBCYALQZcW0RERp4sAn2LDJ36ZhERCafj9s1RHxEVERERERGRxKbFikRERERERCSqVIiKiIiIiIhIVKkQFRERERERkahSISoiIiIiIiJRZdg+oqFw2J1W4B7g/xEonp8GrnJ7XN2GBmYAh92ZDDwAfBIoBHYDv3B7XPf2tw+aq0TNpcPuTAXWAwVujyun/5hydRSH3XkG8H1gCtAK/NTtcd2lXH2Yw+4sBu4HlgAm4HUCv/PORM+Xw+68CrgUqALedHtcjgFtGcBDwJlAN/Bb4Ntuj8sfjnaJnpH+Og6F+uahUd8cHPXNwVPffGyx2DfHy4joTQReUJVAOTAD+LGhERnHCuwBPkVgWeTzgG877M7z+9uPl6tEzeVtfHgJaeVqAIfduZTAm8g3gRxgKvDP/mbl6sMeBGxAGTAeaAce7m9L9HztBu4AfvYRbfcBo4ASYAHwOeCrYWyX6Bnpr+NQqG8eGvXNx6G+OWTqm48t5vrmeClELwd+4Pa4Gt0e137gVuALDrszXuIPG7fH1e72uL7r9rjq3R6Xz+1xrQFeAD7ef8rxcpVwuXTYnXOB0/jwm4ly9UG3E/h9X3J7XH1uj6vF7XFV97cpVx82Efiz2+NqdXtcHcDjwMz+toTOl9vjWu72uJYDewced9idacBFwHfcHtdht8e1FbgL+J9wtEvUjejXcSjUN4dOfXPQ1DeHRn3zMcRi3xzzt+Y67M4cAt9ovDvg8DsEvhUaDzQYEVes6L+N4BPAncfLlcPubB6snRGYy/78PAxcddRx5WoAh92ZDnwMeNJhd74H5AMrgGsJbHCvXH3YT4HPO+zOFwAvgVt5XtBra1BTgCRgzYBj7wAzHHanZbjtbo/LG8ng5b/UNw9OffPg1DcHR33zkKhvDp1hfXM8VPiZ/X83DzjWdFRbIruPQG7+wPFzlYi5/Aawzu1xuY86rlx9UC6BuRSXEviGugzYB/wV5epY3gDygEMEft8pwI0oX4PJBDrcHlffgGNNgAVIDUO7RE8iv46Dob55cOqbg6O+OXTqm0NnWN8cD4Voa//f2QOO5RzVlpAcduc9BL5x/bTb4+rh+LlKqFw67M5JBL5tvf4jmpWrDzryO93n9ri29d/O8h1gHoFvFEG5el//bTr/AVYCGQTehF8kMG9Hr61jawXS+kdDjsgh8BrrDEO7RE8iv44Hpb55cOqbQ6K+OQTqm4fMsL455gtRt8fVBOwAZg84PIdApb3DkKBigMPuvJfAogifdHtcB+D4uUrAXJ5IYOJ0jcPu3AMsB7L6/z0d5ep9bo+rmcBtJwNXNxv4b+Xqg/IITMi/z+1xdfR/OLgfOIHAlAfl66NtBHqAWQOOzQHe6791Z7jtEiUJ/jo+JvXNQVHfHCT1zSFT3zw0hvXNJr8/9le8d9id3wM+C5wO9ALPElh2+OuGBmYQh915H3AycFL/ZOqBbYPmKpFy2b8s/MBvthYDvyew4twhArdqKFf9HHbnjQQmm38G2E9gnsV8t8c1X6+rD3PYnZsILO3+/f5DtwEXuj2u8Ymer/5vRa3A1QSWcv8U4HN7XD0Ou/N3wBjgQgLfmP4beMDtcd3X/9hhtUv0jPTXcajUNwdHfXNo1DeHRn3zscVi3xzzixX1+xFQANQQGMV9isASywnHYXeWEFgOuRvY6rA7jzS95va4Ps3xc5UwuXR7XJ0MuCXAYXceAvxuj2tP/8/K1QfdSWA+yjsEft/XgXP725SrDzuLwBLouwjM4VlD4I0dlK+bgVsG/NwJvAo4CCyy8UtgO4FvUH9D4BvrI4bbLtEz0l/HQVPfHDz1zSFT3xwa9c3HFnN9c1yMiIqIiIiIiMjIEfNzREVERERERGRkUSEqIiIiIiIiUaVCVERERERERKJKhaiIiIiIiIhElQpRERERERERiSoVoiIiIiIiIhJVKkRFREREREQkqlSIioiIiIiISFSpEBUREREREZGo+j92FyywrOcErAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 1000 | value: 10.312933921813965 | policy 1737.69384765625\n" + ] + }, + { + "ename": "AssertionError", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0mplotter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreinforce\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1000\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0;32massert\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mAssertionError\u001b[0m: " + ] + } + ], + "source": [ + "for epoch in range(n_epochs):\n", + " for batch in tqdm(env.train_dataloader):\n", + " loss = reinforce.update(batch)\n", + " reinforce._step += 1\n", + " if loss:\n", + " plotter.log_losses(loss)\n", + " if reinforce._step % plot_every == 0:\n", + " clear_output(True)\n", + " print('step', reinforce._step)\n", + " plotter.plot_loss()\n", + " if reinforce._step > 1000:\n", + " assert False\n", + " \n", + " " + ] } ], "metadata": { diff --git a/recnn/data/dataset_functions.py b/recnn/data/dataset_functions.py index 8df88ed..ba88b7e 100644 --- a/recnn/data/dataset_functions.py +++ b/recnn/data/dataset_functions.py @@ -4,6 +4,9 @@ What? +++++ + Chain of responsibility pattern. + https://refactoring.guru/design-patterns/chain-of-responsibility/python/example + RecNN is designed to work with your dataflow. Function that contain 'dataset' are needed to interact with environment. The environment is provided via env.argument. @@ -46,6 +49,8 @@ def prepare_dataset(df, key_to_id, frame_size, env, sort_users=False, **kwargs): [1, 34, 123, 2000], recnn makes it look like [0,1,2,3] for you. """ + key_to_id = env.key_to_id + df['rating'] = df['rating'].progress_apply(lambda i: 2 * (i - 2.5)) df['movieId'] = df['movieId'].progress_apply(key_to_id.get) users = df[['userId', 'movieId']].groupby(['userId']).size() @@ -70,8 +75,7 @@ def app(x): env.users = users return {'df': df, 'key_to_id': key_to_id, - 'frame_size': frame_size, 'env': env, 'sort_users': sort_users, - **kwargs} + 'frame_size': frame_size, 'env': env, 'sort_users': sort_users, **kwargs} def truncate_dataset(df, key_to_id, frame_size, env, reduce_items_to, sort_users=False, **kwargs): @@ -79,12 +83,11 @@ def truncate_dataset(df, key_to_id, frame_size, env, reduce_items_to, sort_users Truncate #items to num_items provided in the arguments """ + # here n items to keep are adjusted num_items = reduce_items_to - value_counts = df['movieId'].value_counts().sort_values() - - to_remove = value_counts[:-num_items].index - to_keep = value_counts[-num_items:].index + to_remove = df['movieId'].value_counts().sort_values()[:-num_items].index + to_keep = df['movieId'].value_counts().sort_values()[-num_items:].index to_remove_indices = df[df['movieId'].isin(to_remove)].index num_removed = len(to_remove) @@ -95,17 +98,17 @@ def truncate_dataset(df, key_to_id, frame_size, env, reduce_items_to, sort_users del env.movie_embeddings_key_dict[i] env.embeddings, env.key_to_id, env.id_to_key = make_items_tensor(env.movie_embeddings_key_dict) + print('action space is reduced to {} - {} = {}'.format(num_items + num_removed, num_removed, num_items)) - return {'df': df, 'key_to_id': key_to_id, - 'frame_size': frame_size, 'env': env, 'sort_users': sort_users, - 'reduce_items_to': reduce_items_to, **kwargs} + return {'df': df, 'key_to_id': env.key_to_id, 'env': env, + 'frame_size': frame_size, 'sort_users': sort_users, **kwargs} def build_data_pipeline(chain, **kwargs): """ - curry function chain + Chain of responsibility pattern :param chain: array of callable :param **kwargs: any kwargs you like @@ -113,6 +116,6 @@ def build_data_pipeline(chain, **kwargs): kwargdict = kwargs for call in chain: - kwargdict = call(**kwargs) + kwargdict = call(**kwargdict) return kwargdict diff --git a/recnn/nn/algo.py b/recnn/nn/algo.py index e714e54..389437b 100644 --- a/recnn/nn/algo.py +++ b/recnn/nn/algo.py @@ -172,7 +172,7 @@ def __init__(self, policy_net, value_net1, value_net2): class Reinforce(Algo): def __init__(self, policy_net, value_net): - super(Algo, self).__init__() + super(Reinforce, self).__init__() self.algorithm = update.reinforce_update @@ -203,7 +203,7 @@ def __init__(self, policy_net, value_net): 'value_optimizer': value_optimizer } - params = { + self.params = { 'reinforce': ChooseREINFORCE(ChooseREINFORCE.basic_reinforce), 'gamma': 0.99, 'min_value': -10, diff --git a/recnn/nn/models.py b/recnn/nn/models.py index 9d20c11..bfc3da1 100755 --- a/recnn/nn/models.py +++ b/recnn/nn/models.py @@ -73,7 +73,7 @@ def forward(self, state, tanh=False): class DiscreteActor(nn.Module): - def __init__(self, input_dim, action_dim, hidden_size, init_w=2e-1): + def __init__(self, input_dim, action_dim, hidden_size, init_w=0): super(DiscreteActor, self).__init__() self.linear1 = nn.Linear(input_dim, hidden_size) @@ -82,6 +82,11 @@ def __init__(self, input_dim, action_dim, hidden_size, init_w=2e-1): self.saved_log_probs = [] self.rewards = [] + # with large action spaces it can be overflowed + # in order to prevent this, I set a max limit + + self.save_limit = 15 + def forward(self, inputs): x = inputs x = F.relu(self.linear1(x)) @@ -89,6 +94,11 @@ def forward(self, inputs): return F.softmax(action_scores) def select_action(self, state): + + if len(self.saved_log_probs) > self.save_limit: + del self.saved_log_probs[:] + del self.rewards[:] + probs = self.forward(state) m = Categorical(probs) action = m.sample() diff --git a/recnn/nn/update/reinforce.py b/recnn/nn/update/reinforce.py index 6ced8bd..01baa21 100644 --- a/recnn/nn/update/reinforce.py +++ b/recnn/nn/update/reinforce.py @@ -3,8 +3,8 @@ from recnn import utils from recnn import data from recnn.utils import soft_update - from recnn.nn.update import value_update +import gc class ChooseREINFORCE: @@ -25,8 +25,7 @@ def basic_reinforce(policy, returns, *args, **kwargs): def reinforce_with_correction(): raise NotImplemented - def __call__(self, policy, optimizer): - + def __call__(self, policy, optimizer, learn=True): R = 0 returns = [] @@ -39,9 +38,10 @@ def __call__(self, policy, optimizer): policy_loss = self.method(policy, returns) - optimizer.zero_grad() - policy_loss.backward() - optimizer.step() + if learn: + optimizer.zero_grad() + policy_loss.backward() + optimizer.step() del policy.rewards[:] del policy.saved_log_probs[:] @@ -51,7 +51,7 @@ def __call__(self, policy, optimizer): def reinforce_update(batch, params, nets, optimizer, device=torch.device('cpu'), - debug=None, writer= utils.DummyWriter(), + debug=None, writer=utils.DummyWriter(), learn=False, step=-1): state, action, reward, next_state, done = data.get_base_batch(batch) @@ -60,17 +60,22 @@ def reinforce_update(batch, params, nets, optimizer, nets['policy_net'].rewards.append(reward.mean()) value_loss = value_update(batch, params, nets, optimizer, - writer=writer, device=device, - debug=debug, learn=learn, step=step) + writer=writer, + device=device, + debug=debug, learn=learn, step=step) + + if len(nets['policy_net'].saved_log_probs) > params['policy_step'] and learn: + policy_loss = params['reinforce'](nets['policy_net'], optimizer['policy_optimizer'], learn=learn) + + print('step: ', step, '| value:', value_loss.item(), '| policy', policy_loss.item()) + + utils.soft_update(nets['value_net'], nets['target_value_net'], soft_tau=params['soft_tau']) + utils.soft_update(nets['policy_net'], nets['target_policy_net'], soft_tau=params['soft_tau']) - if step % params['policy_step'] == 0 and step > 0: - policy_loss = params['reinforce'](nets['policy_net'], optimizer['policy_optimizer']) del nets['policy_net'].rewards[:] del nets['policy_net'].saved_log_probs[:] - print('step: ', step, '| value:', value_loss.item(), '| policy', policy_loss.item()) - soft_update(nets['value_net'], nets['target_value_net'], soft_tau=params['soft_tau']) - soft_update(nets['policy_net'], nets['target_policy_net'], soft_tau=params['soft_tau']) + gc.collect() losses = {'value': value_loss.item(), 'policy': policy_loss.item(),