linear_DM_run.py

# Copyright (C) H.R. Oosterhuis 2022.
# Distributed under the MIT License (see the accompanying README.md and LICENSE files).

import argparse
import json
import numpy as np
import tensorflow as tf
import time

import utils.click_generation as clkgn
import utils.clicks as clk
import utils.dataset as dataset
import utils.estimators as est
import utils.evaluation as evl
import utils.nnmodel as nn
import utils.optimization as opt
import utils.bias_estimation as EM

parser = argparse.ArgumentParser()
parser.add_argument("n_queries_sampled", type=int,
                    help="Number of sampled queries for training and validation.")
parser.add_argument("output_path", type=str,
                    help="Path to output model.")
parser.add_argument("--fold_id", type=int,
                    help="Fold number to select, modulo operator is applied to stay in range.",
                    default=1)
parser.add_argument("--click_model", type=str,
                    help="Name of click model to use.",
                    default='default')
parser.add_argument("--dataset", type=str,
                    default="Webscope_C14_Set1",
                    help="Name of dataset to sample from.")
parser.add_argument("--dataset_info_path", type=str,
                    default="local_dataset_info.txt",
                    help="Path to dataset info file.")
parser.add_argument("--cutoff", type=int,
                    help="Maximum number of items that can be displayed.",
                    default=5)
parser.add_argument("--pretrained_model", type=str,
                    default=None,
                    help="Path to pretrianed model file.")
parser.add_argument("--estimate_bias", action='store_true',
                    help="Flag to make bias estimated instead of given.")
parser.add_argument('--quick_run', action='store_true',
                    help="Quick run for testing setting.")

args = parser.parse_args()

bias_given = not args.estimate_bias

click_model_name = args.click_model
cutoff = args.cutoff
n_queries_sampled = args.n_queries_sampled

data = dataset.get_dataset_from_json_info(
                  args.dataset,
                  args.dataset_info_path,
                  shared_resource = False,
                )
fold_id = (args.fold_id-1)%data.num_folds()
data = data.get_data_folds()[fold_id]

start = time.time()
data.read_data()
print('Time past for reading data: %d seconds' % (time.time() - start))

cutoff = min(cutoff, data.max_query_size())

alpha, beta = clkgn.get_alpha_beta(click_model_name, cutoff)

print('cutoff', cutoff)

print('True alpha', alpha)
print('True beta', beta)
print('alpha + beta', alpha + beta)

true_train_doc_weights = data.train.label_vector*0.25
true_vali_doc_weights = data.validation.label_vector*0.25
true_test_doc_weights = data.test.label_vector*0.25

model_params = {'hidden units': [32, 32],}
model = nn.init_model(model_params)
logging_model = nn.init_model(model_params)
optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
model.build(input_shape=data.train.feature_matrix.shape)
logging_model.build(input_shape=data.train.feature_matrix.shape)
if args.pretrained_model:
  model.load_weights(args.pretrained_model)
  logging_model.load_weights(args.pretrained_model)
init_weights = model.get_weights()
init_opt_weights = optimizer.get_weights()

regression_model_params = {'hidden units': [32, 32], 'final activation': True}
regression_model = nn.init_model(regression_model_params)
regression_optimizer = tf.keras.optimizers.Adam()
regression_model.build(input_shape=data.train.feature_matrix.shape)
regression_init_weights = regression_model.get_weights()
regression_init_opt_weights = regression_optimizer.get_weights()

train_clicks = np.zeros((data.train.num_docs(), cutoff))
train_displays = np.zeros((data.train.num_docs(), cutoff))
train_query_freq = np.zeros(data.train.num_queries())
train_exp_alpha = np.zeros(data.train.num_docs())
train_exp_beta = np.zeros(data.train.num_docs())
train_policy_scores = logging_model(data.train.feature_matrix)[:, 0].numpy()

vali_clicks = np.zeros((data.validation.num_docs(), cutoff))
vali_displays = np.zeros((data.validation.num_docs(), cutoff))
vali_query_freq = np.zeros(data.validation.num_queries())
vali_exp_alpha = np.zeros(data.validation.num_docs())
vali_exp_beta = np.zeros(data.validation.num_docs())
vali_policy_scores = logging_model(data.validation.feature_matrix)[:, 0].numpy()

alpha_clip = min(10./np.sqrt(n_queries_sampled), 1.)
beta_clip = 0.

output = {
  'dataset': args.dataset,
  'fold number': args.fold_id,
  'click model': click_model_name,
  'initial model': 'random initialization',
  'model hyperparameters': model_params,
  'regression_model_type': 'uniform',
  'regression_model_params': regression_model_params,
  'bias estimated': not bias_given,
  'deterministic ranking': False,
  'alpha clip': alpha_clip,
  'run name': 'linear_direct_method',
}

if args.pretrained_model:
  output['initial model'] = args.pretrained_model

logging_policy_metrics = evl.evaluate_policy(
                                model,
                                data.test,
                                true_test_doc_weights,
                                alpha,
                                beta,
                              )

cur_train_exp_alpha = None
cur_train_exp_beta = None
cur_vali_exp_alpha = None
cur_vali_exp_beta = None

print('Alpha clip:', alpha_clip)

print('sampling up to %s queries' % n_queries_sampled)

(new_train_clicks, new_train_displays,
 new_train_query_freq,
 new_vali_clicks, new_vali_displays,
 new_vali_query_freq,) = clkgn.simulate_on_dataset(
                         data.train,
                         data.validation,
                         n_queries_sampled,
                         true_train_doc_weights,
                         true_vali_doc_weights,
                         alpha,
                         beta,
                         model=logging_model,
                         train_policy_scores=train_policy_scores,
                         vali_policy_scores=vali_policy_scores,
                         store_per_rank=True,
                         return_display=True,
                         )
print('done')

if bias_given:
  # values precomputed by fitting the linear model to 10^9 generated clicks
  if args.dataset == 'Webscope_C14_Set1':
    est_alpha = np.array([0.9999999990533238, 0.6819924179935528, 0.49188888261201313, 0.3703074524329633, 0.28742160369597813, 0.2287699606546821, 0.18612153018438995, 0.15396199757475773, 0.12931657262527246, 0.10966387223986278, 0.09421320741452674, 0.08157024978578292, 0.07142017688530568, 0.06283397741985136, 0.055643993028170344, 0.04975113981887318, 0.044686002034113034, 0.040119824233784504, 0.03633317673265203, 0.03310189781498406, 0.030159364612554945, 0.027578591504572907, 0.025493473491523155, 0.023480323838067096, 0.021561129958659928, 0.020067776301843736, 0.01862840476113399, 0.017383782329399505, 0.0161329750395027, 0.015139883137293567, 0.014125078692581516, 0.01328773656016265, 0.012441650088646639, 0.011702988035655705, 0.011087199696339926, 0.010451019959630922, 0.009844531924085293, 0.009277604830188432, 0.008893502309719841, 0.008387672341550561, 0.00795350696174466, 0.007528189189775156, 0.007262390749193331, 0.006852454618695177, 0.0065473633580887345, 0.006248818915163488, 0.005957972244802261, 0.005677554505622566, 0.005407985407850893, 0.0051122962944940495, 0.004986249223042524, 0.004825171911954642, 0.004622643725924385, 0.0044863882041033245, 0.004291943327691986, 0.004089371218674426, 0.003977449297162089, 0.0037301934555844588, 0.003636766969629071, 0.0035589952865197395, 0.0033989224912267155, 0.003241246908044615, 0.0030541691411637897, 0.0029875212132952607, 0.0028513549309975494, 0.002745470308224811, 0.0027700291621262254, 0.002724381023473112, 0.0026037879398477137, 0.002476009496967786, 0.0023718845877777695, 0.0023314746782068857, 0.002192598698526551, 0.0022359131550585645, 0.0021483630586968828, 0.002050047797581479, 0.002037979773064702, 0.0020406503628309313, 0.0018463501430111395, 0.0018400352945298851, 0.0018197714295417316, 0.0018313173609930396, 0.0017157455173136907, 0.0016487182955245861, 0.001591801498488583, 0.0015955705270483552, 0.0015505880106477703, 0.0015320826624567537, 0.0014327234631018337, 0.0014549387127490566, 0.0012802689138823693, 0.0013337310933652985, 0.0012731503436457547, 0.0012625348267810434, 0.0011306266560276094, 0.0012671962490340793, 0.0012227068832032632, 0.0012294782467551135, 0.0012544837193431012, 0.0010908339479169394, 0.0011036226765059002, 0.0010467364272476487, 0.0011121023663046052, 0.001018148404971188, 0.0009712415571613729, 0.0009977889208741442, 0.000919425009811178, 0.0009205857715083351, 0.0009641221940641525, 0.0009256996084591794, 0.0007990529752721556, 0.0008070126355175792, 0.0008012959228366676, 0.0005505239792370875, 0.000777449093362805, 0.0007056422673930474, 0.0007367080720057175, 0.0009566237039774824, 0.0006612333694511319, 0.0007156931266094976, 0.0007735597654410186, 0.0008575433525335487, 0.0009506310002062675, 0.0005926125731340783, 0.0006671436580934628, 0.0005393645400030682, 0.0007366042840012884, 0.0006309157641801052, 0.0005140714736513752, 0.0005355181708788744, 0.0004997005489404246, 0.0006161213996759603, 0.0006253101302008331, 0.00037090252971047296, 0.0007836251027642157, 0.0007937217620887255, 0.000933596026486591, 0.0005740620288960721, 0.0007571767545608486],
                          dtype=np.float64)
  elif args.dataset == 'istella':
    est_alpha = np.array([1.0, 0.6753726339216596, 0.48128028233135833, 0.3586641693551303, 0.2750325485312247, 0.21701599977970512, 0.17401301825540647, 0.1423156642697086, 0.11782568144165428, 0.09896004640653991, 0.08386262569049988, 0.07181194774421715, 0.0616356005707708, 0.05392079086618834, 0.04694649797087253, 0.04125904271137441, 0.03652617414711406, 0.032625472423207486, 0.029115232558075955, 0.026189791853562717, 0.02365059545303408, 0.02148834276646426, 0.019434482434414632, 0.01770520479582067, 0.016240616369316566, 0.01497035397291004, 0.013732786783036166, 0.012713303013632982, 0.011752636480946923, 0.01090067906883099, 0.010169938661165221, 0.00940949259048193, 0.00883858775014741, 0.00823142695545798, 0.0077201997242020105, 0.0072325180891862125, 0.006802137256002709, 0.006394474067313969, 0.006037592581616878, 0.0057035876762652924, 0.005391780233281735, 0.005080027612890667, 0.0048109204884704, 0.0045821427973475526, 0.00434024796526138, 0.0041407697106984, 0.003940486596400989, 0.003740179126677478, 0.0035739697031833906, 0.003402379413535882, 0.0032478882421017935, 0.003103741075166001, 0.0029778457818305714, 0.0028491075066154354, 0.00273886782708396, 0.002614637204379773, 0.0025033163585563467, 0.0024162147010669674, 0.00232662471457846, 0.0022274691693596376, 0.0021477659766174343, 0.002066117864109548, 0.0019877914859366434, 0.001913553577690286, 0.0018401580431082303, 0.0017782481077498649, 0.00172055521996986, 0.0016598570153392265, 0.0016042346867643875, 0.0015550025774929079, 0.0014956344381324633, 0.0014468638502437212, 0.0013993501111749253, 0.0013600476299118148, 0.0013207331750921482, 0.0012765882635797656, 0.001239804990681535, 0.0012035817543431385, 0.0011653889230407684, 0.0011325649887750427, 0.0011002403260610592, 0.001068923512042099, 0.0010374804087076481, 0.0010069619581960512, 0.0009833941783478734, 0.0009509914508843479, 0.000927773850392527, 0.0009079927307011541, 0.0008835066667413688, 0.0008564455488678397, 0.0008351635029881947, 0.0008129073567628507, 0.0007927294087397287, 0.0007725128352507384, 0.0007564892214670377, 0.0007368504960621884, 0.0007178361382798382, 0.0006994536559333662, 0.000685127463429555, 0.0006696499290887964, 0.0006563145620173638, 0.0006389747776485077, 0.0006248771121167439, 0.00060886650803406, 0.0005972626873272998, 0.0005834988383378008, 0.0005711199941177472, 0.0005582466433951534, 0.0005465147686297256, 0.0005350734292796718, 0.0005239474960357586, 0.0005135257007725042, 0.0005025142855135967, 0.0004921310271486413, 0.0004830948691257637, 0.0004738750609371339, 0.000462997919678231, 0.0004544035439445225, 0.0004448944480707241, 0.00043683963537833074, 0.0004290286803346412, 0.00042050668073672265, 0.0004120712744228286, 0.00040461942442186066, 0.0003982554241802167, 0.0003889750676112496, 0.0003804823620818946, 0.0003755697232516303, 0.0003694591458158522, 0.00036243782775080395, 0.0003554375940981435, 0.000349256127764725, 0.00034291803734228015, 0.00033742677785741904, 0.00033170987022377354, 0.000326147309099735, 0.00032159297276683203, 0.0003144829669880485, 0.000310157894548361, 0.00030378998151171625, 0.00029970319657695313, 0.00029481215386894366, 0.000290076280005012, 0.00028631635851828494, 0.0002811565979307336, 0.0002769317279022126, 0.00027268801168061957, 0.0002682279304010954, 0.000263499111101156, 0.00026023111212209127, 0.0002561249161463604, 0.0002518619686572079, 0.00024913269743916713, 0.0002450017385216466, 0.00024011177966373455, 0.00023679032016488081, 0.00023378887794926265, 0.0002298846715549522, 0.00022637695285511575, 0.00022398110369607586, 0.00022036272465427486, 0.0002173375439239171, 0.0002135277771329051, 0.0002114880739358335, 0.00020805184633770828, 0.00020563628703222075, 0.00020192628874782654, 0.00019957511430238774, 0.0001960875869608822, 0.0001946083302696591, 0.00019108154763107724, 0.00018850520671592975, 0.00018607108127835238, 0.00018372349532499627, 0.00018113519976011223, 0.0001788635799483773, 0.00017670716622881667, 0.00017407661603270475, 0.00017198072610633754, 0.0001699996535155192, 0.0001680042295556872, 0.0001657450406132212, 0.00016344058090480052, 0.00016167896879852231, 0.00015962822270054587, 0.00015758609187126307, 0.0001558102173207971, 0.00015392905263670933, 0.00015202407401666092, 0.00015014825055247467, 0.00014833968556095804, 0.0001468153638417618, 0.00014473446904827303, 0.00014295032383577957, 0.00014132344639312177, 0.0001396678884444832, 0.00013792406225252303, 0.00013613754347495353, 0.00013461551284570388, 0.0001333316251448492, 0.00013187826198358772, 0.00013027325709359517, 0.00012937185393146537, 0.00012778402690638712, 0.00012618932408863488, 0.0001245587081327989, 0.0001230563004599986, 0.00012191527205044763, 0.00012064099898973929, 0.00011942114887782181, 0.00011812481832759234, 0.00011655875495482468, 0.0001160126331906471, 0.00011420820176076948, 0.00011319354289821663, 0.00011184855549510145, 0.00011057569475165507, 0.00010981709578666417, 0.00010877195448946264, 0.00010756262695363461, 0.00010625134466662118, 0.00010492848809143447, 0.00010404621586438027, 0.00010338590041135098, 0.00010242223669773825, 0.00010089700678369477, 0.00010005168862085642, 9.92706053602802e-05, 9.778226350975965e-05, 9.714340379390774e-05, 9.630814595492475e-05, 9.553773394226501e-05, 9.431288455277223e-05, 9.34870569123685e-05, 9.247593090156437e-05, 9.1338622191703e-05, 9.10736506538185e-05, 9.003649279184306e-05, 8.894199967019153e-05, 8.832536906737988e-05, 8.743811093334065e-05, 8.649430196539331e-05, 8.585574496940952e-05, 8.479062218442543e-05, 8.435662114437125e-05, 8.327445570446263e-05, 8.264045835071335e-05, 8.212070786476572e-05, 8.143246631311828e-05, 8.015908949034237e-05, 7.992539553313781e-05, 7.899018554396566e-05, 7.827848766972303e-05, 7.762228126112628e-05, 7.70526508936297e-05, 7.591580864695073e-05, 7.561038482470402e-05, 7.497873897319424e-05, 7.44613284176417e-05, 7.386970163159013e-05, 7.319488579139019e-05, 7.257617850994218e-05, 7.17322966350785e-05, 7.121486865357222e-05, 7.066838822850918e-05, 7.015785864793091e-05, 6.968223818462362e-05, 6.914204995350936e-05, 6.856400000929477e-05, 6.777081486026163e-05, 6.750619873374032e-05, 6.66978589956869e-05, 6.619952017541254e-05, 6.577098296013892e-05, 6.48769698086425e-05, 6.441208398602776e-05, 6.394549452841611e-05, 6.344792320492712e-05, 6.290850752608916e-05, 6.255474247620244e-05, 6.193810870893944e-05, 6.155505671715479e-05, 6.1023033532802926e-05, 6.057207067080768e-05, 6.0037400845486925e-05, 5.9824458052420024e-05, 5.916553031833938e-05, 5.8624354008936795e-05, 5.8308880800734335e-05, 5.7793536533384406e-05, 5.731421174365943e-05, 5.690874505252184e-05, 5.656944689162017e-05, 5.59733357824481e-05, 5.564798162955259e-05, 5.515520694669353e-05, 5.491554083629077e-05, 5.4392365435550456e-05, 5.405742043226035e-05, 5.366594229136577e-05, 5.338750115142921e-05, 5.2927342168110486e-05, 5.242234561163567e-05, 5.2241489725834575e-05, 5.1778169569512394e-05, 5.140973985565067e-05, 5.0996087384112774e-05, 5.055268977285187e-05, 5.042624980757858e-05, 4.9879617467596145e-05, 4.969416904182873e-05, 4.927888589516815e-05, 4.893253065047804e-05, 4.867259987173268e-05, 4.815843420418713e-05, 4.778282957920345e-05, 4.7729938617042776e-05, 4.7075599404200413e-05, 4.697428245486609e-05, 4.643151195730922e-05, 4.607334446540172e-05, 4.583909705999995e-05, 4.539525230051056e-05, 4.510289126105696e-05, 4.504948710229771e-05, 4.4713989639576495e-05, 4.437264573885556e-05, 4.399377567587512e-05, 4.408326018279429e-05, 4.3449288718917e-05, 4.316136188458024e-05, 4.27846573572042e-05, 4.2753880151935975e-05, 4.246705772572982e-05, 4.221777724147477e-05, 4.178088790480484e-05, 4.137785595231621e-05, 4.1280054399879076e-05, 4.093153076438183e-05, 4.070171135243122e-05, 4.046062260493749e-05, 4.010586739963179e-05, 3.9892086783056e-05, 3.967846333500617e-05, 3.9404553173834705e-05, 3.920761299868703e-05, 3.881275965776537e-05, 3.8745076905780093e-05, 3.842482413624094e-05, 3.8143730968274046e-05, 3.7992960221753094e-05, 3.782114432659131e-05, 3.7384467673950036e-05, 3.726065408989307e-05, 3.7035904280040605e-05, 3.690150454121448e-05, 3.659706948136415e-05, 3.627328578802433e-05, 3.609283438311187e-05, 3.6012543543168204e-05, 3.582712837965831e-05, 3.548963846906337e-05, 3.532719865696507e-05, 3.514364641960694e-05, 3.478655523889605e-05, 3.471831115013818e-05, 3.4467592421330946e-05, 3.423355293796592e-05, 3.4135916306180737e-05, 3.391041656754546e-05, 3.367794724625464e-05, 3.3509884065632956e-05, 3.33023962734174e-05, 3.3070056995890294e-05, 3.284133090616379e-05, 3.276482167479682e-05, 3.2532587271685606e-05, 3.2343867631485685e-05, 3.2149031770449525e-05, 3.187823024358042e-05, 3.17830115816442e-05, 3.1588231729180817e-05, 3.140784692827475e-05, 3.123632431252772e-05, 3.119846365165971e-05, 3.082541904431718e-05, 3.065516664589771e-05, 3.067547047623057e-05, 3.0438146441342467e-05, 3.0334420227654098e-05, 3.0086608799138936e-05, 2.9921635928386685e-05, 2.9774561777872908e-05, 2.949829104520255e-05, 2.9368391710666762e-05, 2.9305502549482644e-05, 2.9071565996089352e-05, 2.893915898807437e-05, 2.8728383662485577e-05, 2.852174814229768e-05, 2.844697531786719e-05, 2.8306163553966282e-05, 2.8179247734750407e-05, 2.8070857343828486e-05, 2.7868506015908907e-05, 2.779056803503501e-05, 2.7607240480454847e-05, 2.7418795572174268e-05, 2.728196782095019e-05, 2.718481059845498e-05, 2.696371786362416e-05, 2.7034333639424674e-05, 2.6726941177934607e-05, 2.6597550512881357e-05, 2.6505048942237244e-05, 2.638009680209356e-05, 2.627101040649671e-05, 2.6063929129001213e-05, 2.592213313683991e-05, 2.5808411099850885e-05, 2.5705836354915362e-05, 2.571865330322733e-05, 2.54447425508134e-05, 2.5358382915525046e-05, 2.5129240078879248e-05, 2.5020632347848283e-05, 2.4946073526013614e-05, 2.4714015022326035e-05, 2.4808359824366463e-05, 2.464082866152122e-05, 2.452117223489347e-05, 2.440891850909323e-05, 2.430886247262858e-05, 2.408139057579192e-05, 2.3878258524743328e-05, 2.3621284342194958e-05, 2.324527099749152e-05, 2.2704856903731333e-05, 2.4151054348426282e-05],
                          dtype=np.float64)
  elif args.dataset == 'MSLR-WEB30k':
    est_alpha = np.array([0.9999999947998217, 0.6801982357534745, 0.4888685029408647, 0.3663149283792338, 0.2834684623588152, 0.22500291898660318, 0.1823119301452679, 0.15009935986269393, 0.12590870356272502, 0.10664683115449163, 0.09137908964441968, 0.07908010014312207, 0.06897522499370908, 0.06048552309309556, 0.053628966491296184, 0.04779513553785575, 0.04277886100561383, 0.03848958782801827, 0.034687882968688036, 0.03158820485180733, 0.028724085434524115, 0.02621992572429662, 0.024117218844368654, 0.022184667468578168, 0.020483383444409968, 0.01900172548249955, 0.017590760124274605, 0.016357469255071762, 0.015243188646777803, 0.014241517875759568, 0.013359425989149349, 0.012480416034146218, 0.011727791422721795, 0.01104518492590207, 0.010409729692549006, 0.009830497903743376, 0.009291242457157043, 0.008773486374243947, 0.008359752202997183, 0.007915071898910747, 0.0075147471273531, 0.007126772204882605, 0.006795302782998924, 0.0064897980484042855, 0.006160291434631967, 0.0058965435765801835, 0.005649922933622672, 0.005418120022507181, 0.005155861816428549, 0.00496357274829107, 0.004761002072293329, 0.004567283328345533, 0.004382552615421688, 0.0042252283022080174, 0.0040649156693940685, 0.0039192767208687095, 0.0037712877501480625, 0.003631999037053945, 0.0034970490088767486, 0.003376899986873123, 0.0032698225368225425, 0.003154792861785106, 0.0030510169464676067, 0.002946494649439281, 0.002855894218149371, 0.0027629374133268008, 0.002679441531593843, 0.0025929056051318547, 0.0025215462048333295, 0.0024393141234651727, 0.002361969895476872, 0.002296526395969417, 0.0022296934038471467, 0.0021788699440205925, 0.002095268271207541, 0.0020491694191759804, 0.0019888201918103, 0.0019429633678176867, 0.0018967579175918087, 0.00183989223128134, 0.0017890449360259838, 0.0017511490556005528, 0.0017027747107526523, 0.0016575123396630853, 0.0016172701659095707, 0.0015753849456778858, 0.0015495862994485614, 0.001497401305186228, 0.0014707009328521349, 0.0014364059861473992, 0.0014086453538419295, 0.0013752169060209096, 0.0013440284178278185, 0.001297942303254015, 0.0012778792229063032, 0.0012554007374982587, 0.0012271629047915836, 0.0012011764993268143, 0.0011724311042787426, 0.0011501870106316316, 0.001128452218640042, 0.0011054027611713028, 0.0010759953147956758, 0.0010561276604649325, 0.0010349739909796057, 0.0010177053379379643, 0.00099452890146499, 0.000976958588457635, 0.0009554438674467961, 0.0009394775591295096, 0.0009241098246855308, 0.0008999319315634201, 0.0008859345189281848, 0.0008653439760753806, 0.0008489302535313527, 0.0008399305236555067, 0.0008218840758018725, 0.0008088055261357641, 0.0007928533524858075, 0.0007763102685362448, 0.0007645197694296417, 0.0007513669541542455, 0.0007403192988064545, 0.0007229456493588868, 0.0007123245976325351, 0.0007043458577375035, 0.0006887387630177149, 0.0006833032147178088, 0.0006709226943082377, 0.0006556039782355725, 0.0006475161899970213, 0.0006376843550346294, 0.0006284356812200563, 0.0006170862517733134, 0.0006029688646527004, 0.0005972265221226834, 0.0005904316520880653, 0.0005766016216845943, 0.0005683466332478566, 0.0005575847517821441, 0.000555677022195826, 0.0005472664412241203, 0.0005380673374031162, 0.0005265889430370016, 0.0005162786063408336, 0.0005090875948196131, 0.0004998584103471169, 0.0004959166071710164, 0.0004922230323520515, 0.0004843605991338572, 0.00047592668688611276, 0.0004705924183361483, 0.00046829898147307476, 0.0004555080255581168, 0.0004507402567624403, 0.0004468508056654135, 0.00043989605560251527, 0.0004333764934565538, 0.00042662575052651835, 0.0004188841114193178, 0.0004165548594797638, 0.00041276422918941374, 0.000401331366171795, 0.00039840039612745883, 0.0003966640403991979, 0.00039346808390554224, 0.00038359502896328146, 0.0003833525250507698, 0.000368006045254422, 0.0003701308818763521, 0.0003654010183253172, 0.00035956251468731665, 0.00035068779476990833, 0.00035122210412108864, 0.00034907747971635544, 0.0003405836539228905, 0.00033978034046962784, 0.0003342949019958812, 0.0003288639629743671, 0.0003305315254497305, 0.0003255284697786837, 0.0003214877502653982, 0.0003184317157858659, 0.00031113879766556037, 0.00030202065114553475, 0.00030612503299413935, 0.0002958423995650087, 0.0003017351322026348, 0.0002943515106246934, 0.00028900357334305275, 0.00029122814687915926, 0.0002858977652805402, 0.0002785946540024075, 0.0002735390717257124, 0.00027525944775341836, 0.00027286800004391025, 0.00026877452768943604, 0.0002655153098509198, 0.00026003513120378215, 0.0002561705210459024, 0.0002530568501401038, 0.0002521464133556728, 0.0002504048051275134, 0.00024541447753281847, 0.0002469880759630317, 0.00024512854925227485, 0.00024001095344806573, 0.00023752598265117477, 0.00023638399977539385, 0.0002345772035885749, 0.0002275085625112853, 0.0002293602512756219, 0.0002269628942440451, 0.00021948528093161147, 0.00022123237584367473, 0.00021725650559160296, 0.0002183625000729109, 0.00021308708272235809, 0.0002089073044554117, 0.0002101189081935585, 0.00020748764079494414, 0.0002092286063731743, 0.00020703738227157158, 0.0002077431658862847, 0.0001994342740698273, 0.00019925166428991746, 0.00019348365750584623, 0.00019398452482670436, 0.0001938631850908608, 0.0001901165440717641, 0.00018947806341417943, 0.0001846655042751228, 0.00018355670596476777, 0.00018454120279489704, 0.00018371478421855907, 0.00018017588991766926, 0.00018444854624302348, 0.00017718160071876452, 0.000175202783698937, 0.0001793709512671617, 0.00017571982599396435, 0.00017393077356031325, 0.00016809806078767095, 0.00016992518683430302, 0.00016529163159308306, 0.00017015207434135368, 0.0001593261857046331, 0.00016397359414869987, 0.00016128045578608005, 0.0001591033114076517, 0.00015701106370124776, 0.00016150614231116855, 0.0001539530302716713, 0.0001557633183063599, 0.00015370972113565897, 0.00015374092524647303, 0.0001558724086681476, 0.00014739322927534058, 0.00014482240822106938, 0.00014589179162148314, 0.0001464425892335445, 0.00014248648807123544, 0.00014229605516592284, 0.0001429419174428836, 0.00014461311214735388, 0.0001442580979262174, 0.00014034418174595074, 0.00013350835310756493, 0.0001379682122906125, 0.00013646917012353064, 0.0001366445618745741, 0.00012829617209973772, 0.00012627149437029135, 0.0001324866669632914, 0.00012959381273261683, 0.00012744159404781333, 0.00013087336548686142, 0.000123954797990296, 0.00012551116996315032, 0.00012708928434740214, 0.0001252246054570699, 0.00012528761646653794, 0.0001229551810281565, 0.00012228047447474525, 0.00012182874055286267, 0.00012150602732612555, 0.00011825470161904315, 0.00012095254454426076, 0.00011196075218705288, 0.00011711029323758003, 0.0001111960336649424, 0.00011265516819883516, 0.00011179479498837413, 0.00011168836071945663, 0.00011476693255451525, 0.000111939195569719, 0.00010838320261541234, 0.0001061159548682344, 0.00010744452229810897, 0.00010730799594963329, 0.00010425911330336617, 0.00010511023259664831, 0.00010284603562214667, 0.00010445818704418203, 0.00010625754370346891, 0.00010551204264325925, 0.000108122710269203, 0.00010579628337299628, 0.00010246654888808587, 0.00010143142916518486, 9.874569907269659e-05, 0.00010172265012716151, 9.704269443348921e-05, 9.315374914517406e-05, 9.791632324174799e-05, 9.469473867423517e-05, 9.813236807686909e-05, 9.338636608596547e-05, 9.737731229390145e-05, 9.61829362193846e-05, 9.494341505843363e-05, 9.194161692845042e-05, 8.81036529968601e-05, 9.240444500323922e-05, 9.195975992815186e-05, 8.748340015412147e-05, 8.489748219990691e-05, 9.179782162037959e-05, 8.57336927429565e-05, 8.695855616348704e-05, 8.676328401096772e-05, 8.669730621558837e-05, 8.712818783838106e-05, 8.362971575555534e-05, 8.455120642011624e-05, 8.122244712299091e-05, 8.253803817275378e-05, 8.145445574930606e-05, 8.33074531040522e-05, 8.258009738332663e-05, 7.774196150089358e-05, 8.01588833194633e-05, 7.940193281215398e-05, 8.027094627275891e-05, 8.195037275155282e-05, 7.850837017017797e-05, 7.466112718821327e-05, 8.139930894109638e-05, 7.755809415275775e-05, 7.757203657911618e-05, 7.156697659161255e-05, 7.709203111090502e-05, 7.656238185653221e-05, 7.63952829361962e-05, 7.480481996269903e-05, 7.602709514577073e-05, 7.397339617707767e-05, 7.091198708697918e-05, 7.010310095015145e-05, 7.252014531757845e-05, 7.36876900648274e-05, 6.970063310184722e-05, 7.240486426999766e-05, 6.91143987910657e-05, 7.317677104967732e-05, 6.811392472536038e-05, 6.643020633556484e-05, 6.695852855890197e-05, 6.53559462197429e-05, 6.156305497899273e-05, 6.546016673736091e-05, 6.533263471114206e-05, 6.917323489130978e-05, 6.610910080238077e-05, 6.328571909824951e-05, 6.650959638067525e-05, 6.343733548144065e-05, 6.657506402948734e-05, 6.211474619421901e-05, 6.40074312836029e-05, 6.343314686608448e-05, 6.765986806334699e-05, 6.029696499857207e-05, 6.320475472509389e-05, 5.9748159437422775e-05, 6.287268120970792e-05, 6.40322453946796e-05, 6.061650944287327e-05, 6.166027116724497e-05, 5.845952337345567e-05, 6.00473029287006e-05, 6.1980372413521e-05, 6.0722951242719304e-05, 6.029327501291455e-05, 5.655810346267573e-05, 6.0367520783026505e-05, 5.4876352296291526e-05, 5.900098406915723e-05, 5.400194700932354e-05, 6.058079025796809e-05, 5.328420135487607e-05, 5.237999679925504e-05, 5.405156101754026e-05, 5.865146064707021e-05, 6.02628646282908e-05, 5.684561310441331e-05, 5.390821396962532e-05, 5.374061508672853e-05, 5.037537276561608e-05, 5.580370008612501e-05, 5.223267962544643e-05, 5.533095723966775e-05, 5.22734379557545e-05, 4.89893430229883e-05, 5.409130640286732e-05, 5.24441035138344e-05, 5.3016321339741154e-05, 5.157408509487383e-05, 5.117311242098592e-05, 5.058654806387761e-05, 4.898812366765315e-05, 4.842125710475957e-05, 5.208139677530426e-05, 4.896929487631226e-05, 4.796471679590289e-05, 4.584931684858998e-05, 4.828303089938404e-05, 4.926801726471685e-05, 4.8103901135716655e-05, 4.6039944390346047e-05, 4.9111444437444845e-05, 4.466252975841603e-05, 4.3212690315737447e-05, 4.2376021069720826e-05, 4.603357509010886e-05, 4.348853296597385e-05, 4.438479822721514e-05, 4.565705613703407e-05, 4.589862260000228e-05,
                          4.57238071346794e-05, 4.426889173430167e-05, 4.941942398398039e-05, 4.740589749246356e-05, 4.380125995834497e-05, 3.848327487921976e-05, 4.541734757003822e-05, 4.40622689880557e-05, 4.341951815307616e-05, 4.047652189665724e-05, 4.2725497298844304e-05, 4.6224619878208276e-05, 5.024052309771648e-05, 4.1944924558333915e-05, 3.89823242610872e-05, 3.905452209823607e-05, 4.141018022690381e-05, 4.0810617852986415e-05, 4.125999725495265e-05, 4.1065435718327345e-05, 3.491603193222965e-05, 4.002757091471849e-05, 4.0713380715364875e-05, 4.3885805261876986e-05, 4.1586378470406244e-05, 3.6189536722540424e-05, 3.932096692883428e-05, 3.790785183903173e-05, 3.7252612189748224e-05, 3.966573644761713e-05, 3.773294071514861e-05, 3.7116976234237066e-05, 4.058055858050602e-05, 3.2382611404410745e-05, 4.0725899630431945e-05, 3.836540116531427e-05, 3.885457239654392e-05, 3.815674356699468e-05, 3.8160001693559315e-05, 3.859583422091716e-05, 3.5514178295804965e-05, 3.624961031556023e-05, 3.411240520546774e-05, 3.4772604181238614e-05, 3.671169974723994e-05, 3.594881933907039e-05, 3.644401795235294e-05, 3.386391728388256e-05, 3.4472969684449305e-05, 3.245429497144931e-05, 3.668016267372688e-05, 3.49831173804528e-05, 3.4716608685184684e-05, 3.605291334670684e-05, 3.290595517034957e-05, 3.304897567811376e-05, 4.0355368575845e-05, 3.464376216178751e-05, 3.2786296154590266e-05, 3.509163354689334e-05, 3.218430298145968e-05, 3.491588262121674e-05, 3.329641355212755e-05, 3.275453365917782e-05, 2.9959423021957153e-05, 3.549624276021498e-05, 3.523907348208581e-05, 3.39799362407444e-05, 2.9414397705703643e-05, 2.9085132036178435e-05, 2.9956172960679154e-05, 2.8932116872360236e-05, 2.752294740079113e-05, 2.9622832099234414e-05, 3.93289294964269e-05, 3.492648342830103e-05, 3.369244763921736e-05, 3.342319260392507e-05, 2.918484784366518e-05, 2.597762229470165e-05, 3.2017554315767226e-05, 3.4305777198222504e-05, 3.272300080703567e-05, 3.0237407266145692e-05, 3.0930996044921784e-05, 2.6207068180740013e-05, 3.054162156853391e-05, 3.227929125235541e-05, 2.5258462552074965e-05, 2.6286124572628897e-05, 2.8946550129693993e-05, 2.6216496212383264e-05, 2.9400649849306274e-05, 2.5477074668588512e-05, 2.7256948646993633e-05, 3.0649730278875055e-05, 2.687942310824648e-05, 2.6124009311987107e-05, 2.7794458116838252e-05, 2.487876046079519e-05, 2.553882033341336e-05, 2.7646913084545106e-05, 2.639738371872169e-05, 2.4402692590986286e-05, 3.065539752894119e-05, 2.605143186471384e-05, 3.0919180568733997e-05, 2.4688144694808386e-05, 2.5603422709693937e-05, 3.6167628744558236e-05, 2.4958833086609372e-05, 2.9732702697459932e-05, 2.803561663600837e-05, 2.7210552519355828e-05, 2.8416812810595387e-05, 2.2126713571283484e-05, 2.6317896403076796e-05, 2.468920740475716e-05, 2.9240550008927282e-05, 2.2700218617865282e-05, 2.719372016211446e-05, 2.4540174861886688e-05, 2.709765587703112e-05, 2.7370668793990156e-05, 2.5511666512042125e-05, 2.169653892069248e-05, 2.7384217604045674e-05, 2.6936133439177694e-05, 3.058438003245711e-05, 2.409497677445291e-05, 2.3674433334798004e-05, 2.6533914333646483e-05, 2.9028369523112264e-05, 2.209778624001085e-05, 2.5789008298429113e-05, 2.3124596959825594e-05, 2.8202511489589878e-05, 2.8422767571821234e-05, 2.5913911131738583e-05, 2.1380653757370967e-05, 2.031179389986275e-05, 2.7446956515855807e-05, 2.4194856538101675e-05, 2.5571600591087164e-05, 2.2182979122466626e-05, 1.8880363673974313e-05, 2.2492218482854103e-05, 2.226819906211762e-05, 2.3709465283327122e-05, 2.3990395526241654e-05, 1.9149931644722983e-05, 2.7171040295654713e-05, 1.7728110078008297e-05, 2.4948174930514514e-05, 2.3789154102602744e-05, 2.5679356952763848e-05, 2.1024117351515323e-05, 2.796477777349622e-05, 2.456961594246344e-05, 1.9220811182805903e-05, 2.0836901150300186e-05, 1.863137972216071e-05, 2.1356011565869712e-05, 2.1659383844592677e-05, 2.1720468490723973e-05, 2.3521992738725346e-05, 2.2144403949671866e-05, 2.2464691803738034e-05, 2.1816474522034905e-05, 2.7790886341933997e-05, 2.0895814736098393e-05, 2.008377521648654e-05, 2.272692901311659e-05, 1.761834422144475e-05, 2.2712203146270183e-05, 2.2047153411194305e-05, 2.7534322868202464e-05, 2.016956276356121e-05, 1.969610667760441e-05, 1.6587423782686263e-05, 1.8779868964085527e-05, 2.552176120031493e-05, 1.6286177095129048e-05, 2.1918646742285136e-05, 2.3745385194582862e-05, 2.131476029228959e-05, 2.2413148854943204e-05, 1.9292476559485482e-05, 2.360657651426109e-05, 2.0311460187944093e-05, 2.0192127997203774e-05, 1.7343569450529106e-05, 2.315942462114889e-05, 2.1651081853037782e-05, 2.1153083807173138e-05, 2.2016993346867958e-05, 2.308180987901788e-05, 2.5683100204265646e-05, 1.4152933304168383e-05, 1.5017026857616351e-05, 1.9168340013181706e-05, 1.7045009869372813e-05, 1.4205756177205527e-05, 1.7935259581522168e-05, 2.0829159389691692e-05, 1.9614844584989435e-05, 1.699435697321537e-05, 2.529215110662177e-05, 1.6595396457722977e-05, 1.8799835641883138e-05, 1.5545620065235737e-05, 1.9442130367314506e-05, 2.0194707588068793e-05, 2.1639998335843977e-05, 1.3413369525378573e-05, 1.6709637654009783e-05, 1.1917803912662272e-05, 1.9246893965442855e-05, 1.5356005960272972e-05, 2.2782788031578305e-05, 1.982599755522856e-05, 1.8676645316703345e-05, 1.1999397270580044e-05, 2.4228170722835465e-05, 1.2147698443548967e-05, 1.8676229752612347e-05, 1.6835882292994053e-05, 1.835559008963697e-05, 1.61113695497263e-05, 1.529488801145316e-05, 1.9761374665572428e-05, 1.586394558693167e-05, 1.4542767562090775e-05, 1.468282909555161e-05, 1.984817759651848e-05, 1.4817038774868132e-05, 1.3766652226230488e-05, 1.167097268680827e-05, 1.7769299270364992e-05, 1.4450755331280312e-05, 2.5517505710381625e-05, 1.3741499146600306e-05, 1.8569362265676365e-05, 1.409052487217276e-05, 1.8919615202997424e-05, 1.3740707788274188e-05, 1.5445605745383027e-05, 2.168584665346631e-05, 1.6814135926317177e-05, 1.5309220382984287e-05, 1.0171646692774978e-05, 1.7110949109695645e-05, 1.3287468756336898e-05, 1.2220911499448422e-05, 1.1348052185283876e-05, 1.3097249274441709e-05, 1.5377316613822554e-05, 1.5387174851446266e-05, 1.2254682573859676e-05, 1.6537226487482893e-05, 1.139024135650978e-05, 1.6014612374903745e-05, 1.750148584544129e-05, 1.4147400729102474e-05, 1.5805753514016447e-05, 1.941552826982851e-05, 1.4055272673831319e-05, 1.50878273473861e-05, 1.2003343946234861e-05, 1.595978579469481e-05, 1.779907456322871e-05, 1.629356722925086e-05, 1.3666716301899618e-05, 1.0828345463309483e-05, 1.57024591031449e-05, 1.9030704094693017e-05, 1.9116917773601984e-05, 1.0410646487077325e-05, 1.3021308158527605e-05, 1.6560947798938316e-05, 1.2766417527314278e-05, 1.3350647931303399e-05, 1.4517516436959976e-05, 1.8367315991045364e-05, 1.094407871679444e-05, 1.8435168890609687e-05, 1.853477349634918e-05, 1.390287952923884e-05, 1.5666931569067575e-05, 1.573861461662e-05, 1.6273753411461814e-05, 1.1835054066103177e-05, 9.733824492665395e-06, 1.632516649344282e-05, 1.0482717933158614e-05, 1.2141369814043884e-05, 1.216262319456654e-05, 1.1208209686109847e-05, 1.8292094097655896e-05, 1.3046700452753278e-05, 1.5369861382708188e-05, 1.2575788958397915e-05, 1.6208774977827034e-05, 9.305581785732716e-06, 1.4240753009594799e-05, 1.3679452756549336e-05, 1.9442438816961017e-05, 1.3266768576784436e-05, 1.1474330630057848e-05, 1.2148307682995858e-05, 1.2027076983833821e-05, 1.4496910794914838e-05, 1.4060681738082452e-05, 1.6229550147948217e-05, 1.2624002568752282e-05, 1.5176571028484653e-05, 1.2093973577118208e-05, 1.430240757574747e-05, 1.2437924894002646e-05, 9.267074403502532e-06, 1.0530582089129038e-05, 1.2343167182795233e-05, 1.2896907360883602e-05, 1.6857326774790845e-05, 1.0818536065476169e-05, 1.0484484810819506e-05, 1.3492433272749115e-05, 1.2806820527956866e-05, 1.1171945858346075e-05, 1.1891739634336e-05, 8.538853907361121e-06, 7.95112301149989e-06, 1.9764164790954783e-05, 1.2563944457609765e-05, 8.782210386374046e-06, 1.5388182763367936e-05, 6.459260920046964e-06, 1.0738508585696846e-05, 1.2862836862199243e-05, 1.2515590426235092e-05, 1.5438408854520752e-05, 1.219080660636279e-05, 1.7108215304421054e-05, 1.1392315674775577e-05, 1.1323139935972816e-05, 1.1960330437028777e-05, 1.2151440918812989e-05, 2.2834905940989763e-05, 1.2182353457868268e-05, 1.1605697736199726e-05, 9.094308697658552e-06, 9.729029054580833e-06, 1.79548079160945e-05, 7.177774885661322e-06, 5.478302707380149e-06, 1.2472980597780712e-05, 1.9491966102966622e-05, 1.2744075907763865e-05, 1.3441424037080912e-05, 4.515125240901846e-06, 1.2734919446940793e-05, 1.2439926061315633e-05, 1.8812134851099202e-05, 1.5378184392219434e-05, 9.408813031858508e-06, 1.618843275225016e-05, 1.26903059129938e-05, 9.877917988078308e-06, 1.2336206079056348e-05, 1.4166227281336898e-05, 8.14429000712598e-06, 1.1407999557443137e-05, 1.485660380085444e-05, 1.4189235567735204e-05, 1.0266846013484604e-05, 1.4974079324969825e-05, 1.0347215394134702e-05, 1.2247542833182428e-05, 1.0089455296879052e-05, 1.182830155843136e-05, 4.269298647079227e-06, 3.822171855107113e-06, 7.625815614304807e-06, 1.4522589859094199e-05, 1.1942284321033006e-05, 8.37239899141913e-06, 1.0699152629636613e-05, 9.586160800286358e-06, 5.72806147270144e-06, 1.7841369823445018e-05, 9.550585160308592e-06, 1.240080338544896e-05, 1.4809989364621136e-05, 5.551239955448931e-06, 9.884753208467111e-06, 1.2826760292916534e-05, 1.088607832650477e-05, 6.203486124251855e-06, 8.978800519007605e-06, 6.298547660123914e-06, 1.1029992292654597e-05, 7.44483512363817e-06, 5.860938569739404e-06, 8.83965650465493e-06, 9.086854622271821e-06, 1.655881299217367e-05, 1.4640968470565319e-05, 8.090161781229013e-06, 1.346790709883622e-05, 8.503534869422082e-06, 1.2070370623213514e-05, 7.932889616763916e-06, 1.0830214834019821e-05, 5.854254648724809e-06, 6.2566615670518365e-06, 5.466409557137577e-06, 1.030659802700475e-05, 5.6748490617334605e-06, 6.326220785916012e-06, 6.514269552670756e-06, 1.0612031123771354e-05, 7.52339834440359e-06, 9.171681970535e-06, 3.079341040487534e-06, 6.451114303449139e-06, 3.2119614400957865e-06, 1.297030280040653e-05, 3.2370392108979653e-06,
                          8.631722390181935e-06, 1.2829360574673923e-05, 5.970422415552129e-06, 4.030235140422993e-06, 8.275535369664514e-06, 8.307803343959246e-06, 1.0266917608217234e-05, 1.095404595412316e-05, 5.643273859424181e-06, 7.70676783444235e-06, 6.230034933983901e-06, 7.246690048494944e-06, 5.343193620522134e-06, 4.407223826943333e-06, 8.55068795134342e-06, 8.604689726647342e-06, 1.0596471095311595e-05, 1.0338513591744013e-05, 5.861280398934539e-06, 5.569673702065592e-06, 8.623833702035676e-06, 1.379925056448798e-05, 7.887073189923514e-06, 3.939638261750644e-06, 7.967948073606472e-06, 1.1368911499992889e-05, 1.2017970588209061e-05, 7.892965736497768e-06, 6.805698467354649e-06, 1.067664694307703e-05, 8.711468547502899e-06, 7.995152130310193e-06, 7.301763008192394e-06, 7.758586441715848e-06, 3.0520288862450506e-06, 8.613430148581034e-06, 8.309860086036742e-06, 6.816122197393062e-06, 8.406158692616369e-06, 1.700080835943933e-05, 9.020312708678183e-06, 4.90620037954828e-06, 3.9784392244652674e-06, 9.92190455803261e-06, 1.3024003436355985e-05, 7.820543504088485e-06, 9.524551262274697e-06, 5.617364633277382e-06, 8.897392309339023e-06, 5.530442367418767e-06, 1.7574153245564426e-06, 1.42937718241875e-05, 1.4208867145776725e-05, 6.7948276807522425e-06, 7.600173988278475e-06, 3.2499329494390095e-06, 5.962482022803111e-06, 5.764317397077847e-06, 1.9790980823137057e-06, 5.076907030975028e-06, 4.317548211696292e-06, 3.0319777421641362e-06, 3.0962616224835828e-06, 1.4902188919127025e-05, 6.448164410071669e-06, 7.207497250356707e-06, 4.8662209149983685e-06, 8.933065832963622e-06, 7.066170188439993e-06, 9.43895737700116e-06, 6.380851424217051e-06, 1.1562323108816358e-05, 1.3782624830015178e-05, 4.621140432903966e-06, 3.418862896493128e-06, 2.407930883812436e-05, 5.215226060941602e-07, 8.93943404243625e-06, 5.928515005400393e-06, 8.637757860751723e-06, 1.0251850324107732e-05, 1.4596931704445404e-05, 5.558171631718025e-06, 1.5236075055874016e-05, 7.082679809764775e-06, 5.102721721777591e-06, 8.099221702638537e-06, 9.179351194230769e-06, 7.816901964637145e-06, 1.6464960431054565e-05, 1.6821294849660922e-05, 9.893978571572981e-06, 4.471452392640445e-06, 5.473830329397336e-06, 2.506207504300374e-06, 2.5999878919257303e-06, 5.9772527501346124e-06, 4.0406129926129685e-06, 5.039854431677397e-06, 1.0015529894376399e-05, 1.1318349639583483e-05, 1.7053542867410283e-05, 7.1993383952504756e-06, 7.035092250512735e-06, 2.0863906317275145e-06, 1.3975554110693564e-06, 6.575130118000616e-06, 4.5542375179642055e-06, 3.4848656601789618e-06, 1.7459030069583057e-05, 1.6262267440032598e-05, 9.603967827634468e-06, 8.089667956719705e-06, 1.2996860145747177e-05, 3.609986481477005e-06, 1.529338454723547e-05, 6.770306205547517e-06, 7.043105926155398e-06, 1.0485059117821023e-05, 5.015557382659856e-06, 2.974536565915381e-06, 6.989899561004773e-06, 4.8845008167223836e-06, 7.734065397667803e-06, 3.983623856712735e-06, 2.949255859332365e-06, 2.011670531852967e-06, 1.3808083785036421e-05, 6.100591869763763e-06, 1.2477529107783912e-05, 6.730860694536441e-06, 2.9764776016213734e-06, 7.69936782294758e-06, 5.076105064732552e-06, 8.324923519343263e-06, 0.0, 7.860430651628473e-06, 3.876462261572419e-06, 1.0768341424248414e-05, 1.942762247009844e-06, 2.964393430297393e-06, 3.906694895093769e-06, 1.9418105490899687e-06, 9.852854920393347e-07, 3.89171308979506e-06, 7.930545019868108e-06, 5.023770101217069e-06, 8.868817064595416e-06, 0.0, 1.0045719415383744e-05, 4.117306696407358e-06, 4.9868444816757884e-06, 3.7871243327678975e-06, 1.8443402670379438e-05, 3.900693094574607e-06, 1.800309437089253e-06, 5.04012490572974e-06, 9.812937743801561e-07, 3.0471802346651414e-06, 2.0353144024841235e-06, 6.940276166190568e-06, 5.995823216333222e-06, 3.811013143762771e-06, 6.033124293343502e-06, 2.352540016313096e-06, 2.7288165836861252e-06, 1.0011794564538232e-05, 4.61876961550957e-06, 2.5269411873808276e-06, 0.0, 4.966856135840691e-06, 0.0, 1.0570358555839852e-05, 0.0, 2.4544045101739376e-06, 0.0, 0.0, 4.951744597607246e-06, 0.0, 2.4945146768303115e-06, 4.9803183256478784e-06, 0.0, 2.598063648195982e-06, 2.4648149318676533e-06, 1.785215892070383e-05, 0.0, 9.977956379029773e-06, 1.3040148142139547e-05, 2.4757646555321767e-06, 1.0041622095561062e-05, 4.981651233824452e-06, 0.0, 0.0, 0.0, 0.0, 2.4795816166273217e-06, 7.60582279191786e-06, 0.0, 0.0, 2.528713889616729e-06, 2.629394705790841e-06, 7.581444162284089e-06, 2.3448059427848507e-06, 2.597388850590813e-06, 7.890217579653805e-06, 2.5731447611208505e-06, 0.0, 7.41484171525477e-06, 7.425463027005656e-06, 2.3679867766563022e-06, 5.018277503024622e-06, 2.3790607106638682e-06, 0.0, 8.022429646407445e-06, 0.0, 2.409627316126026e-06, 2.177631358074911e-05, 2.4641231401318173e-06, 1.0650164795929222e-05, 8.387429156118399e-06, 5.512755927554253e-06, 5.733176047485473e-06, 5.228182435282994e-06, 2.6481152515879633e-06, 0.0, 0.0, 2.800169064318294e-06, 0.0, 0.0, 5.462444029020928e-06, 1.0450022100463332e-05, 5.146663906473237e-06, 0.0, 0.0, 8.318426871960103e-06, 0.0, 2.653779726071351e-06, 2.893913578737629e-06, 0.0, 0.0, 2.748078986539283e-06, 2.7083473419926007e-06, 8.252332085526248e-06, 2.167165432785918e-05, 0.0, 0.0, 2.9234590561942485e-06, 5.39395375502985e-06, 5.415064436517042e-06, 8.481429262998513e-06, 5.341136832947439e-06, 2.94580943519948e-06, 2.7236964027235564e-06, 0.0, 2.6270757570559917e-06, 0.0, 1.0728853315329109e-05, 0.0, 2.6804187431874786e-06, 2.6330392474506205e-06, 2.690945296140037e-06, 8.119556367147997e-06, 5.595850257955641e-06, 0.0, 5.643995569168122e-06, 0.0, 0.0, 0.0, 1.6490502954158796e-05, 2.912695279365117e-06, 1.146810979270093e-05, 5.617269362763474e-06, 8.623736978819909e-06, 5.488886027679365e-06, 8.40408867226096e-06, 5.730799094843533e-06, 8.33447970929639e-06, 0.0, 2.7112110426830732e-06, 2.7148733710060534e-06, 8.54722338596273e-06, 2.705040880839462e-06, 5.970462686444872e-06, 2.792942777725155e-06, 2.7600748964733792e-06, 2.8364554367049944e-06, 5.945510871058227e-06, 1.3718267871807375e-05, 2.839357305042346e-06, 2.6717196295079065e-06, 0.0, 2.702499497022514e-06, 2.874456627287572e-06, 0.0, 0.0, 0.0, 2.670877919373051e-06, 2.98998114806261e-06, 0.0, 0.0, 2.733441118870291e-06, 5.703480715009922e-06, 5.799678507210019e-06, 2.694841056882254e-06, 0.0, 0.0, 5.856557148184497e-06, 0.0, 0.0, 2.965087572913647e-06, 5.6787702546745224e-06, 5.44367759853847e-06, 5.718928024649903e-06, 0.0, 2.8638831601275997e-06, 2.973233443193339e-06, 0.0, 2.9328183443947683e-06, 5.741355155282111e-06, 2.7249214948322935e-06, 0.0, 5.716719902573141e-06, 5.634243029410695e-06, 2.8659560879627828e-06, 2.9700121867957852e-06, 2.7511628941741435e-06, 2.947591941255763e-06, 5.541781553406196e-06, 2.868393304728949e-06, 3.0441164019270537e-06, 3.166987902843399e-06, 1.1438589874766916e-05, 2.891387743096599e-06, 0.0, 5.677926467164814e-06, 2.572883436797675e-06, 0.0, 5.293362742056658e-06, 5.843182377069789e-06, 8.63302234456088e-06, 3.0807437502210274e-06, 0.0, 0.0, 5.860199280129031e-06, 6.17375184455502e-06, 5.807247180595579e-06, 2.8827219079725093e-06, 1.3499482468332966e-05, 0.0, 2.903460013472516e-06, 6.207227689629514e-06, 1.1831565639858488e-05, 1.4942230956715943e-05, 5.914185404998476e-06, 2.8152732943856345e-06, 0.0, 0.0, 2.8633459153401756e-06, 0.0, 2.9631811935817386e-06, 2.86979249695706e-06, 0.0, 2.855831948988687e-06, 0.0, 0.0, 2.925892552845099e-06, 0.0, 2.9599685832387596e-06, 0.0, 0.0, 1.1784608302487435e-05, 2.8987238477583987e-06, 2.918211004855451e-06, 0.0, 0.0, 5.795784253375294e-06, 6.203811572018715e-06, 8.972346647842334e-06, 0.0, 2.9727418021151827e-06, 5.755693892183312e-06, 9.222875877299889e-06, 2.9391466511269104e-06, 3.127583225970499e-06],
                          dtype=np.float64)
  est_alpha = est_alpha[:cutoff]

  est.update_statistics(
                    True,
                    data.train,
                    train_clicks,
                    train_displays,
                    train_query_freq,
                    train_exp_alpha,
                    train_exp_beta,
                    new_train_clicks,
                    new_train_displays,
                    new_train_query_freq,
                    cur_train_exp_alpha,
                    cur_train_exp_beta,
                    est_alpha,
                    np.zeros_like(est_alpha),
                   )

  est.update_statistics(
                    True,
                    data.validation,
                    vali_clicks,
                    vali_displays,
                    vali_query_freq,
                    vali_exp_alpha,
                    vali_exp_beta,
                    new_vali_clicks,
                    new_vali_displays,
                    new_vali_query_freq,
                    cur_vali_exp_alpha,
                    cur_vali_exp_beta,
                    est_alpha,
                    np.zeros_like(est_alpha),
                   )

else:
  est.update_observations(
                    train_clicks,
                    train_displays,
                    train_query_freq,
                    new_train_clicks,
                    new_train_displays,
                    new_train_query_freq,
                   )

  est.update_observations(
                    vali_clicks,
                    vali_displays,
                    vali_query_freq,
                    new_vali_clicks,
                    new_vali_displays,
                    new_vali_query_freq,
                   )

  print('Bias Estimation: %s' % bias_estimation)

  est_alpha = EM.EM_estimate_linear_bias(
                      bias_estimation,
                      data.train,
                      data.validation,
                      train_clicks,
                      train_displays,
                      vali_clicks,
                      vali_displays,
                     )
  print('done')

  est.update_frequency_estimated_bias(
                    data.train,
                    train_displays,
                    train_query_freq,
                    train_exp_alpha,
                    train_exp_beta,
                    est_alpha,
                    np.zeros_like(est_alpha),
                    )

  est.update_frequency_estimated_bias(
                    data.validation,
                    vali_displays,
                    vali_query_freq,
                    vali_exp_alpha,
                    vali_exp_beta,
                    est_alpha,
                    np.zeros_like(est_alpha),
                    )

print('optimize regression with an unbiased estimate of uniform loss')
regression_model.set_weights(regression_init_weights)
opt.optimize_regressor_counterfactual_linear(
                  regression_model, regression_optimizer,
                  data.train, train_query_freq, train_clicks, train_exp_alpha,
                  data.validation, vali_query_freq, vali_clicks, vali_exp_alpha,
                  alpha_clip=alpha_clip
                  )
  
train_doc_weights = est.compute_direct_method_weights(
                        data.train,
                        train_query_freq,
                        regression_model,
                        )
vali_doc_weights = est.compute_direct_method_weights(
                        data.validation,
                        vali_query_freq,
                        regression_model,
                        )

if args.quick_run:
  max_epochs = 3
else:
  max_epochs = 500
model.set_weights(init_weights)
optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
model, vali_reward = opt.optimize_policy(model, optimizer,
                    data_train=data.train,
                    train_doc_weights=train_doc_weights,
                    train_alpha=est_alpha,
                    train_beta=np.zeros_like(beta),
                    data_vali=data.validation,
                    vali_doc_weights=vali_doc_weights,
                    vali_alpha=est_alpha,
                    vali_beta=np.zeros_like(beta),
                    max_epochs=max_epochs,
                    early_stop_per_epochs=3,
                    print_updates=True,
                    )

cur_metrics = evl.evaluate_policy(
                      model,
                      data.test,
                      true_test_doc_weights,
                      alpha,
                      beta,
                    )

output['results'] = {
  'iteration': int(n_queries_sampled),
  'metrics': cur_metrics,
  'logging policy metrics': logging_policy_metrics,
}

print('No. query %09d, RCTR %0.5f, DCG %0.5f' % (
      n_queries_sampled, cur_metrics['RCTR'], cur_metrics['DCG']))

print(output)

print('Writing results to %s' % args.output_path)
with open(args.output_path, 'w') as f:
  json.dump(output, f)