jupyter_notebooks/Examples/BootstrappedErrorBars.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# An example showing how to generate bootstrapped error bars."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import json\n",
    "\n",
    "import pygsti\n",
    "from pygsti.construction import std1Q_XYI\n",
    "\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- Std Practice:  Iter 1 of 1  (TP) --: \n",
      "  --- Circuit Creation ---\n",
      "  --- LGST ---\n",
      "  --- Iterative MLGST: [##################################################] 100.0%  1282 operation sequences ---\n",
      "  Iterative MLGST Total Time: 2.3s\n",
      "  --- Re-optimizing logl after robust data scaling ---\n",
      "  -- Performing 'single' gauge optimization on TP estimate --\n",
      "  -- Conveying 'single' gauge optimization to TP.Robust+ estimate --\n"
     ]
    }
   ],
   "source": [
    "#Get a GST estimate (similar to Tutorial 0)\n",
    "\n",
    "# 1) get the target Model\n",
    "target_model = std1Q_XYI.target_model()\n",
    "\n",
    "# 2) get the building blocks needed to specify which operation sequences are needed\n",
    "prep_fiducials, meas_fiducials = std1Q_XYI.prepStrs, std1Q_XYI.effectStrs\n",
    "germs = std1Q_XYI.germs\n",
    "maxLengths = [1,2,4,8,16]\n",
    "\n",
    "# 3) generate \"fake\" data from a depolarized version of target_model\n",
    "mdl_datagen = target_model.depolarize(op_noise=0.1, spam_noise=0.001)\n",
    "listOfExperiments = pygsti.construction.make_lsgst_experiment_list(\n",
    "    target_model, prep_fiducials, meas_fiducials, germs, maxLengths)\n",
    "ds = pygsti.construction.generate_fake_data(mdl_datagen, listOfExperiments, nSamples=1000,\n",
    "                                            sampleError=\"binomial\", seed=1234)\n",
    "\n",
    "\n",
    "results = pygsti.do_stdpractice_gst(ds, target_model, prep_fiducials, meas_fiducials,\n",
    "                                    germs, maxLengths, modes=\"TP\")\n",
    "estimated_model = results.estimates['TP'].models['single']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parametric Bootstrapping\n",
    "Here we do parametric bootstrapping, as indicated by the 'parametric' argument below.\n",
    "The output is eventually stored in the \"mean\" and \"std\" Models, which hold the mean and standard deviation values of the set of bootstrapped models (after gauge optimization).  It is this latter \"standard deviation Model\"\n",
    "which holds the collection of error bars.  Note: due to print setting issues, the outputs that are printed here will not necessarily reflect the true accuracy of the estimates made.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating DataSets: \n",
      "0 Generating parametric dataset.\n",
      "1 Generating parametric dataset.\n",
      "2 Generating parametric dataset.\n",
      "3 Generating parametric dataset.\n",
      "4 Generating parametric dataset.\n",
      "5 Generating parametric dataset.\n",
      "6 Generating parametric dataset.\n",
      "7 Generating parametric dataset.\n",
      "8 Generating parametric dataset.\n",
      "9 Generating parametric dataset.\n",
      "Creating Models: \n",
      "Running MLGST Iteration 0 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.243740981718673\n",
      "  1.196825270321894\n",
      "  0.9803977850297132\n",
      "  0.9189829897008691\n",
      "  0.04573617221601241\n",
      "  0.02684031856476036\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 54.6619 (92 data params - 31 model params = expected mean of 61; p-value = 0.703349)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 54.714\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 129.189 (168 data params - 31 model params = expected mean of 137; p-value = 0.670269)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 129.277\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 390.627 (450 data params - 31 model params = expected mean of 419; p-value = 0.83646)\n",
      "  Completed in 0.4s\n",
      "  2*Delta(log(L)) = 390.768\n",
      "  Iteration 3 took 0.4s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 834.331 (862 data params - 31 model params = expected mean of 831; p-value = 0.46099)\n",
      "  Completed in 0.7s\n",
      "  2*Delta(log(L)) = 834.617\n",
      "  Iteration 4 took 0.7s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1249.69 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.505154)\n",
      "  Completed in 1.0s\n",
      "  2*Delta(log(L)) = 1250.2\n",
      "  Iteration 5 took 1.0s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 625.055 below upper bound of -2.13561e+06\n",
      "      2*Delta(log(L)) = 1250.11 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.501789)\n",
      "    Completed in 0.4s\n",
      "  2*Delta(log(L)) = 1250.11\n",
      "  Final MLGST took 0.4s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.0s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 1 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244955776409719\n",
      "  1.1654832900216188\n",
      "  0.956272084807351\n",
      "  0.9195971979216845\n",
      "  0.04819500202308558\n",
      "  0.019674658291966154\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 69.4303 (92 data params - 31 model params = expected mean of 61; p-value = 0.214644)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 69.5064\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 139.522 (168 data params - 31 model params = expected mean of 137; p-value = 0.424013)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 139.487\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 425.208 (450 data params - 31 model params = expected mean of 419; p-value = 0.406563)\n",
      "  Completed in 0.4s\n",
      "  2*Delta(log(L)) = 424.889\n",
      "  Iteration 3 took 0.4s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 860.208 (862 data params - 31 model params = expected mean of 831; p-value = 0.234373)\n",
      "  Completed in 0.6s\n",
      "  2*Delta(log(L)) = 860.294\n",
      "  Iteration 4 took 0.7s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1236.14 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.612136)\n",
      "  Completed in 1.0s\n",
      "  2*Delta(log(L)) = 1236.41\n",
      "  Iteration 5 took 1.1s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 618.162 below upper bound of -2.136e+06\n",
      "      2*Delta(log(L)) = 1236.32 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.610735)\n",
      "    Completed in 0.4s\n",
      "  2*Delta(log(L)) = 1236.32\n",
      "  Final MLGST took 0.4s\n",
      "  \n",
      "Iterative MLGST Total Time: 2.9s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 2 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244105950849598\n",
      "  1.1579831871684374\n",
      "  0.9724007421046054\n",
      "  0.9008232174607133\n",
      "  0.04747934275633835\n",
      "  0.02335927755344803\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 53.7231 (92 data params - 31 model params = expected mean of 61; p-value = 0.734336)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 53.767\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 118.871 (168 data params - 31 model params = expected mean of 137; p-value = 0.865831)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 118.848\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 410.566 (450 data params - 31 model params = expected mean of 419; p-value = 0.606523)\n",
      "  Completed in 0.4s\n",
      "  2*Delta(log(L)) = 411.16\n",
      "  Iteration 3 took 0.4s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 846.655 (862 data params - 31 model params = expected mean of 831; p-value = 0.345336)\n",
      "  Completed in 0.6s\n",
      "  2*Delta(log(L)) = 847.325\n",
      "  Iteration 4 took 0.7s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1285.78 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.241266)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 1286.8\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 643.359 below upper bound of -2.13608e+06\n",
      "      2*Delta(log(L)) = 1286.72 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.235548)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 1286.72\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.2s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 3 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.245026706871426\n",
      "  1.1581551644431194\n",
      "  0.9751123814360809\n",
      "  0.917116445045877\n",
      "  0.04948557817173936\n",
      "  0.017700857933000646\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 54.7477 (92 data params - 31 model params = expected mean of 61; p-value = 0.700454)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 54.7464\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 123.896 (168 data params - 31 model params = expected mean of 137; p-value = 0.781645)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 123.873\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 375.888 (450 data params - 31 model params = expected mean of 419; p-value = 0.9358)\n",
      "  Completed in 0.4s\n",
      "  2*Delta(log(L)) = 376.466\n",
      "  Iteration 3 took 0.4s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 787.336 (862 data params - 31 model params = expected mean of 831; p-value = 0.858584)\n",
      "  Completed in 0.7s\n",
      "  2*Delta(log(L)) = 787.672\n",
      "  Iteration 4 took 0.7s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1173.37 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.942023)\n",
      "  Completed in 1.1s\n",
      "  2*Delta(log(L)) = 1173.91\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 586.913 below upper bound of -2.1359e+06\n",
      "      2*Delta(log(L)) = 1173.83 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.940902)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 1173.83\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.2s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 4 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.243957692250352\n",
      "  1.1690547567407383\n",
      "  0.965190202696205\n",
      "  0.9062510088586136\n",
      "  0.058565422676061636\n",
      "  0.020128461433915477\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 73.1954 (92 data params - 31 model params = expected mean of 61; p-value = 0.136202)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 73.3169\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 145.055 (168 data params - 31 model params = expected mean of 137; p-value = 0.302405)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 145.31\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 450.681 (450 data params - 31 model params = expected mean of 419; p-value = 0.137667)\n",
      "  Completed in 0.4s\n",
      "  2*Delta(log(L)) = 451.235\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 863.504 (862 data params - 31 model params = expected mean of 831; p-value = 0.21086)\n",
      "  Completed in 0.8s\n",
      "  2*Delta(log(L)) = 864.535\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1293.67 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.195791)\n",
      "  Completed in 1.1s\n",
      "  2*Delta(log(L)) = 1294.9\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 647.394 below upper bound of -2.13607e+06\n",
      "      2*Delta(log(L)) = 1294.79 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.189784)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 1294.79\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.5s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 5 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244076886752041\n",
      "  1.1641097446903883\n",
      "  0.9656589990825546\n",
      "  0.9256958692572328\n",
      "  0.05988709636348685\n",
      "  0.0018089980940076643\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 46.7157 (92 data params - 31 model params = expected mean of 61; p-value = 0.911338)\n",
      "  Completed in 0.3s\n",
      "  2*Delta(log(L)) = 46.8907\n",
      "  Iteration 1 took 0.3s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 110.427 (168 data params - 31 model params = expected mean of 137; p-value = 0.953691)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 110.699\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 421.079 (450 data params - 31 model params = expected mean of 419; p-value = 0.462264)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 421.678\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 836.041 (862 data params - 31 model params = expected mean of 831; p-value = 0.444428)\n",
      "  Completed in 0.8s\n",
      "  2*Delta(log(L)) = 837.086\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1227.62 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.676188)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 1228.84\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 614.366 below upper bound of -2.13643e+06\n",
      "      2*Delta(log(L)) = 1228.73 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.668034)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 1228.73\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.6s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 6 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244075532131559\n",
      "  1.167915782338968\n",
      "  0.9557843550642068\n",
      "  0.9176164826710592\n",
      "  0.07089776735122219\n",
      "  0.012139475083936863\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 63.587 (92 data params - 31 model params = expected mean of 61; p-value = 0.385456)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 63.6718\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 134.64 (168 data params - 31 model params = expected mean of 137; p-value = 0.541046)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 134.834\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 422.608 (450 data params - 31 model params = expected mean of 419; p-value = 0.441453)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 422.757\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 844.495 (862 data params - 31 model params = expected mean of 831; p-value = 0.364826)\n",
      "  Completed in 0.8s\n",
      "  2*Delta(log(L)) = 844.732\n",
      "  Iteration 4 took 0.9s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1269.48 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.351608)\n",
      "  Completed in 1.1s\n",
      "  2*Delta(log(L)) = 1269.93\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 634.909 below upper bound of -2.13574e+06\n",
      "      2*Delta(log(L)) = 1269.82 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.349133)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 1269.82\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.6s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 7 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244480338214524\n",
      "  1.165763527245094\n",
      "  0.9338843507514768\n",
      "  0.894817674590485\n",
      "  0.03560561553467106\n",
      "  0.011817184934139907\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 50.8873 (92 data params - 31 model params = expected mean of 61; p-value = 0.818698)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 51.09\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 118.749 (168 data params - 31 model params = expected mean of 137; p-value = 0.867575)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 118.844\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 404.585 (450 data params - 31 model params = expected mean of 419; p-value = 0.684631)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 404.878\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 825.414 (862 data params - 31 model params = expected mean of 831; p-value = 0.548141)\n",
      "  Completed in 0.9s\n",
      "  2*Delta(log(L)) = 826.085\n",
      "  Iteration 4 took 0.9s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1179.53 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.925572)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 1180.31\n",
      "  Iteration 5 took 1.3s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 590.119 below upper bound of -2.13605e+06\n",
      "      2*Delta(log(L)) = 1180.24 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.923479)\n",
      "    Completed in 0.6s\n",
      "  2*Delta(log(L)) = 1180.24\n",
      "  Final MLGST took 0.6s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.8s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 8 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244787150940367\n",
      "  1.1501560865775227\n",
      "  0.9566739054733346\n",
      "  0.9046650260494242\n",
      "  0.030204804282964466\n",
      "  0.005737486195909679\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 60.9529 (92 data params - 31 model params = expected mean of 61; p-value = 0.477613)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 60.9843\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 131.827 (168 data params - 31 model params = expected mean of 137; p-value = 0.608762)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 131.864\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 400.15 (450 data params - 31 model params = expected mean of 419; p-value = 0.738292)\n",
      "  Completed in 0.4s\n",
      "  2*Delta(log(L)) = 400.548\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 785.227 (862 data params - 31 model params = expected mean of 831; p-value = 0.870266)\n",
      "  Completed in 0.7s\n",
      "  2*Delta(log(L)) = 786.042\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1167.95 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.954012)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 1168.9\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 584.416 below upper bound of -2.13586e+06\n",
      "      2*Delta(log(L)) = 1168.83 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.952212)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 1168.83\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.4s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 9 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244086871227867\n",
      "  1.1742535252889312\n",
      "  0.9742543570194043\n",
      "  0.9330064213489485\n",
      "  0.048119822313218055\n",
      "  0.014464994594877137\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 47.5036 (92 data params - 31 model params = expected mean of 61; p-value = 0.896923)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 47.5543\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 115.65 (168 data params - 31 model params = expected mean of 137; p-value = 0.90705)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 115.791\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 398.455 (450 data params - 31 model params = expected mean of 419; p-value = 0.757579)\n",
      "  Completed in 0.4s\n",
      "  2*Delta(log(L)) = 398.689\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 806.048 (862 data params - 31 model params = expected mean of 831; p-value = 0.726382)\n",
      "  Completed in 0.7s\n",
      "  2*Delta(log(L)) = 806.647\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1182.69 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.915882)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 1183.65\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 591.796 below upper bound of -2.13645e+06\n",
      "      2*Delta(log(L)) = 1183.59 (1282 data params - 31 model params = expected mean of 1251; p-value = 0.912964)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 1183.59\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.4s\n",
      "  -- Adding Gauge Optimized (go0) --\n"
     ]
    }
   ],
   "source": [
    "#The number of simulated datasets & models made for bootstrapping purposes.  \n",
    "# For good statistics, should probably be greater than 10.\n",
    "numGatesets=10\n",
    "\n",
    "param_boot_models = pygsti.drivers.make_bootstrap_models(\n",
    "                        numGatesets, ds, 'parametric', prep_fiducials, meas_fiducials, germs, maxLengths,\n",
    "                        inputModel=estimated_model, startSeed=0, returnData=False,\n",
    "                        verbosity=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spam weight 0\n",
      "Spam weight 1\n",
      "Spam weight 2\n",
      "Spam weight 3\n",
      "Spam weight 4\n",
      "Spam weight 5\n",
      "Spam weight 6\n",
      "Spam weight 7\n",
      "Spam weight 8\n",
      "Spam weight 9\n",
      "Spam weight 10\n",
      "Spam weight 11\n",
      "Spam weight 12\n",
      "Best SPAM weight is 1.0\n"
     ]
    }
   ],
   "source": [
    "gauge_opt_pboot_models = pygsti.drivers.gauge_optimize_model_list(param_boot_models, estimated_model,\n",
    "                                                                 plot=False) #plotting support removed w/matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parametric bootstrapped error bars, with 10 resamples\n",
      "\n",
      "Error in rho vec:\n",
      "TPSPAMVec with dimension 4\n",
      " 0.71   0   0   0\n",
      "\n",
      "\n",
      "Error in effect vecs:\n",
      "TPPOVM with effect vectors:\n",
      "0: FullSPAMVec with dimension 4\n",
      "   0   0   0   0\n",
      "\n",
      "1: ComplementSPAMVec with dimension 4\n",
      " 1.41   0   0   0\n",
      "\n",
      "\n",
      "\n",
      "Error in Gi:\n",
      "TPDenseOp with shape (4, 4)\n",
      " 1.00   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "\n",
      "\n",
      "Error in Gx:\n",
      "TPDenseOp with shape (4, 4)\n",
      " 1.00   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "\n",
      "\n",
      "Error in Gy:\n",
      "TPDenseOp with shape (4, 4)\n",
      " 1.00   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pboot_mean = pygsti.drivers.to_mean_model(gauge_opt_pboot_models, estimated_model)\n",
    "pboot_std  = pygsti.drivers.to_std_model(gauge_opt_pboot_models, estimated_model)\n",
    "\n",
    "#Summary of the error bars\n",
    "print(\"Parametric bootstrapped error bars, with\", numGatesets, \"resamples\\n\")\n",
    "print(\"Error in rho vec:\") \n",
    "print(pboot_std['rho0'], end='\\n\\n')\n",
    "print(\"Error in effect vecs:\")\n",
    "print(pboot_std['Mdefault'], end='\\n\\n')\n",
    "print(\"Error in Gi:\")\n",
    "print(pboot_std['Gi'], end='\\n\\n')\n",
    "print(\"Error in Gx:\")\n",
    "print(pboot_std['Gx'], end='\\n\\n')\n",
    "print(\"Error in Gy:\")\n",
    "print(pboot_std['Gy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Non-parametric Bootstrapping\n",
    "Here we do non-parametric bootstrapping, as indicated by the 'nonparametric' argument below.\n",
    "The output is again eventually stored in the \"mean\" and \"std\" Models, which hold the mean and standard deviation values of the set of bootstrapped models (after gauge optimization).  It is this latter \"standard deviation Model\"\n",
    "which holds the collection of error bars.  Note: due to print setting issues, the outputs that are printed here will not necessarily reflect the true accuracy of the estimates made.\n",
    "\n",
    "(Technical note: ddof = 1 is by default used when computing the standard deviation -- see numpy.std -- meaning that we are computing a standard deviation of the sample, not of the population.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating DataSets: \n",
      "0 Generating non-parametric dataset.\n",
      "1 Generating non-parametric dataset.\n",
      "2 Generating non-parametric dataset.\n",
      "3 Generating non-parametric dataset.\n",
      "4 Generating non-parametric dataset.\n",
      "5 Generating non-parametric dataset.\n",
      "6 Generating non-parametric dataset.\n",
      "7 Generating non-parametric dataset.\n",
      "8 Generating non-parametric dataset.\n",
      "9 Generating non-parametric dataset.\n",
      "Creating Models: \n",
      "Running MLGST Iteration 0 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244212085138186\n",
      "  1.1843719301091324\n",
      "  0.9857947532159174\n",
      "  0.9013030041875327\n",
      "  0.09547108379285918\n",
      "  0.04886490428154151\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 170.59 (92 data params - 31 model params = expected mean of 61; p-value = 2.60802e-12)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 171.166\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 344.819 (168 data params - 31 model params = expected mean of 137; p-value = 0)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 345.262\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 991.904 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 993.352\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1815.39 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.9s\n",
      "  2*Delta(log(L)) = 1817.16\n",
      "  Iteration 4 took 0.9s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2701.38 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.1s\n",
      "  2*Delta(log(L)) = 2703.31\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1351.47 below upper bound of -2.13485e+06\n",
      "      2*Delta(log(L)) = 2702.94 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 2702.94\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.6s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1351.47 below upper bound of -2.13485e+06\n",
      "      2*Delta(log(L)) = 2702.94 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.3s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 1 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244167171173532\n",
      "  1.153514315008981\n",
      "  0.9636315553213499\n",
      "  0.908474545441188\n",
      "  0.046056950903486917\n",
      "  0.014789568961852528\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 147.684 (92 data params - 31 model params = expected mean of 61; p-value = 3.78334e-09)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 148.749\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 274.913 (168 data params - 31 model params = expected mean of 137; p-value = 2.75325e-11)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 276.308\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 866.238 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 868.748\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1658.63 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.8s\n",
      "  2*Delta(log(L)) = 1661.19\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2467.46 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 2470.98\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1235.33 below upper bound of -2.13563e+06\n",
      "      2*Delta(log(L)) = 2470.66 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 2470.66\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.5s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1235.33 below upper bound of -2.13563e+06\n",
      "      2*Delta(log(L)) = 2470.66 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.3s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 2 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244617505701195\n",
      "  1.1425181984723756\n",
      "  0.9821383251044142\n",
      "  0.8825511904531211\n",
      "  0.03538596563546793\n",
      "  0.019645266130539504\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 152.339 (92 data params - 31 model params = expected mean of 61; p-value = 9.04838e-10)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 152.658\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 287.202 (168 data params - 31 model params = expected mean of 137; p-value = 1.08791e-12)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 287.882\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 932.984 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 934.913\n",
      "  Iteration 3 took 0.6s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1720.38 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.8s\n",
      "  2*Delta(log(L)) = 1723.17\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2634.67 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.3s\n",
      "  2*Delta(log(L)) = 2638.42\n",
      "  Iteration 5 took 1.3s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1318.98 below upper bound of -2.13523e+06\n",
      "      2*Delta(log(L)) = 2637.96 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 2637.96\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.7s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1318.98 below upper bound of -2.13523e+06\n",
      "      2*Delta(log(L)) = 2637.96 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.4s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 3 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.243988002748227\n",
      "  1.2013993194238075\n",
      "  0.9736701526820217\n",
      "  0.9363076178129426\n",
      "  0.027172808326342654\n",
      "  0.005255940106641545\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 159.939 (92 data params - 31 model params = expected mean of 61; p-value = 8.27658e-11)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 160.22\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 289.265 (168 data params - 31 model params = expected mean of 137; p-value = 6.24945e-13)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 289.212\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 849.321 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 849.302\n",
      "  Iteration 3 took 0.6s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1686.29 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.7s\n",
      "  2*Delta(log(L)) = 1687.79\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2496.17 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 2498.46\n",
      "  Iteration 5 took 1.3s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1249.03 below upper bound of -2.13547e+06\n",
      "      2*Delta(log(L)) = 2498.05 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 2498.05\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.6s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1249.03 below upper bound of -2.13547e+06\n",
      "      2*Delta(log(L)) = 2498.05 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.3s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 4 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.246180502104294\n",
      "  1.1521822289971684\n",
      "  0.9764404175045424\n",
      "  0.906610245735585\n",
      "  0.07369893163659891\n",
      "  0.020240938685679086\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 160.269 (92 data params - 31 model params = expected mean of 61; p-value = 7.44789e-11)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 161.452\n",
      "  Iteration 1 took 0.3s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 268.207 (168 data params - 31 model params = expected mean of 137; p-value = 1.52095e-10)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 269.635\n",
      "  Iteration 2 took 0.3s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 906.473 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 908.592\n",
      "  Iteration 3 took 0.6s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1752.98 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.8s\n",
      "  2*Delta(log(L)) = 1756.15\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2618.09 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.1s\n",
      "  2*Delta(log(L)) = 2621.8\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1310.73 below upper bound of -2.13509e+06\n",
      "      2*Delta(log(L)) = 2621.45 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 2621.45\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.6s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1310.73 below upper bound of -2.13509e+06\n",
      "      2*Delta(log(L)) = 2621.45 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.4s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 5 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244326073733323\n",
      "  1.1441327581502785\n",
      "  0.9622582316708493\n",
      "  0.9502766969909989\n",
      "  0.04152115972837583\n",
      "  0.029558409756115962\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 123.979 (92 data params - 31 model params = expected mean of 61; p-value = 3.45234e-06)\n",
      "  Completed in 0.3s\n",
      "  2*Delta(log(L)) = 125.106\n",
      "  Iteration 1 took 0.3s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 247.859 (168 data params - 31 model params = expected mean of 137; p-value = 2.10431e-08)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 248.561\n",
      "  Iteration 2 took 0.3s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 872.89 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.6s\n",
      "  2*Delta(log(L)) = 876.046\n",
      "  Iteration 3 took 0.6s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1682.54 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.8s\n",
      "  2*Delta(log(L)) = 1686.15\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2484.67 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 2489.05\n",
      "  Iteration 5 took 1.3s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1244.35 below upper bound of -2.13534e+06\n",
      "      2*Delta(log(L)) = 2488.7 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 2488.7\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.7s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1244.35 below upper bound of -2.13534e+06\n",
      "      2*Delta(log(L)) = 2488.7 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.4s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 6 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.243826627679486\n",
      "  1.181000471428712\n",
      "  0.9620835176798799\n",
      "  0.9235835321348868\n",
      "  0.029932975327267443\n",
      "  0.01795053898160025\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 167.863 (92 data params - 31 model params = expected mean of 61; p-value = 6.39e-12)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 168.443\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 305.858 (168 data params - 31 model params = expected mean of 137; p-value = 6.43929e-15)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 306.043\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 897.008 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 897.009\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1832.6 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.7s\n",
      "  2*Delta(log(L)) = 1834.65\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2774.87 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.1s\n",
      "  2*Delta(log(L)) = 2778.26\n",
      "  Iteration 5 took 1.1s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1388.92 below upper bound of -2.13458e+06\n",
      "      2*Delta(log(L)) = 2777.85 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 2777.85\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.4s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1388.92 below upper bound of -2.13458e+06\n",
      "      2*Delta(log(L)) = 2777.85 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.3s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 7 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244526056080279\n",
      "  1.151251520412904\n",
      "  0.9439701878450454\n",
      "  0.8888666392207678\n",
      "  0.06087224198184396\n",
      "  0.030748143014395605\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 181.945 (92 data params - 31 model params = expected mean of 61; p-value = 5.79536e-14)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 182.096\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 329.174 (168 data params - 31 model params = expected mean of 137; p-value = 0)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 329.668\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 926.384 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 928.499\n",
      "  Iteration 3 took 0.5s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1717.46 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.8s\n",
      "  2*Delta(log(L)) = 1720.98\n",
      "  Iteration 4 took 0.8s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2590.11 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 2594.67\n",
      "  Iteration 5 took 1.3s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1297.17 below upper bound of -2.13547e+06\n",
      "      2*Delta(log(L)) = 2594.34 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.5s\n",
      "  2*Delta(log(L)) = 2594.34\n",
      "  Final MLGST took 0.5s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.6s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1297.17 below upper bound of -2.13547e+06\n",
      "      2*Delta(log(L)) = 2594.34 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.3s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 8 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.243854590804392\n",
      "  1.1359961161784795\n",
      "  0.9430730695955586\n",
      "  0.9111704896479723\n",
      "  0.09720933554376518\n",
      "  0.03639925023022892\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 115.208 (92 data params - 31 model params = expected mean of 61; p-value = 3.39228e-05)\n",
      "  Completed in 0.3s\n",
      "  2*Delta(log(L)) = 115.295\n",
      "  Iteration 1 took 0.3s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 256.019 (168 data params - 31 model params = expected mean of 137; p-value = 3.0592e-09)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 255.846\n",
      "  Iteration 2 took 0.3s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 837.178 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 837.916\n",
      "  Iteration 3 took 0.6s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1599.73 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.9s\n",
      "  2*Delta(log(L)) = 1600.86\n",
      "  Iteration 4 took 0.9s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2488.63 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 2490.51\n",
      "  Iteration 5 took 1.2s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1245.07 below upper bound of -2.13539e+06\n",
      "      2*Delta(log(L)) = 2490.15 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.6s\n",
      "  2*Delta(log(L)) = 2490.15\n",
      "  Final MLGST took 0.6s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.8s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1245.07 below upper bound of -2.13539e+06\n",
      "      2*Delta(log(L)) = 2490.15 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.3s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "Running MLGST Iteration 9 \n",
      "--- Circuit Creation ---\n",
      "   1282 sequences created\n",
      "   Dataset has 1282 entries: 1282 utilized, 0 requested sequences were missing\n",
      "--- LGST ---\n",
      "  Singular values of I_tilde (truncating to first 4 of 6) = \n",
      "  4.244088493382835\n",
      "  1.158491440342675\n",
      "  0.9765988971692087\n",
      "  0.9291379020211377\n",
      "  0.06900998568202234\n",
      "  0.015665820756577954\n",
      "  \n",
      "  Singular values of target I_tilde (truncating to first 4 of 6) = \n",
      "  4.2440765871547805\n",
      "  1.1775300369228388\n",
      "  0.9531813852473716\n",
      "  0.9453339713427231\n",
      "  3.092369762442044e-16\n",
      "  2.0221726661332703e-16\n",
      "  \n",
      "--- Iterative MLGST: Iter 1 of 5  92 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 164.287 (92 data params - 31 model params = expected mean of 61; p-value = 2.04635e-11)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 165.06\n",
      "  Iteration 1 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 2 of 5  168 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 296.859 (168 data params - 31 model params = expected mean of 137; p-value = 7.89369e-14)\n",
      "  Completed in 0.2s\n",
      "  2*Delta(log(L)) = 298.794\n",
      "  Iteration 2 took 0.2s\n",
      "  \n",
      "--- Iterative MLGST: Iter 3 of 5  450 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 869.949 (450 data params - 31 model params = expected mean of 419; p-value = 0)\n",
      "  Completed in 0.5s\n",
      "  2*Delta(log(L)) = 873.417\n",
      "  Iteration 3 took 0.6s\n",
      "  \n",
      "--- Iterative MLGST: Iter 4 of 5  862 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 1691.71 (862 data params - 31 model params = expected mean of 831; p-value = 0)\n",
      "  Completed in 0.9s\n",
      "  2*Delta(log(L)) = 1696.13\n",
      "  Iteration 4 took 0.9s\n",
      "  \n",
      "--- Iterative MLGST: Iter 5 of 5  1282 operation sequences ---: \n",
      "  --- Minimum Chi^2 GST ---\n",
      "  Sum of Chi^2 = 2508.01 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "  Completed in 1.2s\n",
      "  2*Delta(log(L)) = 2513.11\n",
      "  Iteration 5 took 1.3s\n",
      "  \n",
      "  Switching to ML objective (last iteration)\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1256.4 below upper bound of -2.13552e+06\n",
      "      2*Delta(log(L)) = 2512.81 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.6s\n",
      "  2*Delta(log(L)) = 2512.81\n",
      "  Final MLGST took 0.6s\n",
      "  \n",
      "Iterative MLGST Total Time: 3.8s\n",
      "  -- Adding Gauge Optimized (go0) --\n",
      "--- Re-optimizing logl after robust data scaling ---\n",
      "  --- MLGST ---\n",
      "    Maximum log(L) = 1256.4 below upper bound of -2.13552e+06\n",
      "      2*Delta(log(L)) = 2512.81 (1282 data params - 31 model params = expected mean of 1251; p-value = 0)\n",
      "    Completed in 0.3s\n",
      "  -- Adding Gauge Optimized (go0) --\n"
     ]
    }
   ],
   "source": [
    "#The number of simulated datasets & models made for bootstrapping purposes.  \n",
    "# For good statistics, should probably be greater than 10.\n",
    "numModels=10\n",
    "\n",
    "nonparam_boot_models = pygsti.drivers.make_bootstrap_models(\n",
    "                          numModels, ds, 'nonparametric', prep_fiducials, meas_fiducials, germs, maxLengths,\n",
    "                          targetModel=estimated_model, startSeed=0, returnData=False, verbosity=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spam weight 0\n",
      "Spam weight 1\n",
      "Spam weight 2\n",
      "Spam weight 3\n",
      "Spam weight 4\n",
      "Spam weight 5\n",
      "Spam weight 6\n",
      "Spam weight 7\n",
      "Spam weight 8\n",
      "Spam weight 9\n",
      "Spam weight 10\n",
      "Spam weight 11\n",
      "Spam weight 12\n",
      "Best SPAM weight is 1.0\n"
     ]
    }
   ],
   "source": [
    "gauge_opt_npboot_models = pygsti.drivers.gauge_optimize_model_list(nonparam_boot_models, estimated_model,\n",
    "                                                                 plot=False) #plotting removed w/matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-parametric bootstrapped error bars, with 10 resamples\n",
      "\n",
      "Error in rho vec:\n",
      "TPSPAMVec with dimension 4\n",
      " 0.71   0   0   0\n",
      "\n",
      "\n",
      "Error in effect vecs:\n",
      "TPPOVM with effect vectors:\n",
      "0: FullSPAMVec with dimension 4\n",
      "   0   0   0   0\n",
      "\n",
      "1: ComplementSPAMVec with dimension 4\n",
      " 1.41   0   0   0\n",
      "\n",
      "\n",
      "\n",
      "Error in Gi:\n",
      "TPDenseOp with shape (4, 4)\n",
      " 1.00   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "\n",
      "\n",
      "Error in Gx:\n",
      "TPDenseOp with shape (4, 4)\n",
      " 1.00   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "\n",
      "\n",
      "Error in Gy:\n",
      "TPDenseOp with shape (4, 4)\n",
      " 1.00   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "   0   0   0   0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "npboot_mean = pygsti.drivers.to_mean_model(gauge_opt_npboot_models, estimated_model)\n",
    "npboot_std  = pygsti.drivers.to_std_model(gauge_opt_npboot_models, estimated_model)\n",
    "\n",
    "#Summary of the error bars\n",
    "print(\"Non-parametric bootstrapped error bars, with\", numGatesets, \"resamples\\n\")\n",
    "print(\"Error in rho vec:\")\n",
    "print(npboot_std['rho0'], end='\\n\\n')\n",
    "print(\"Error in effect vecs:\")\n",
    "print(npboot_std['Mdefault'], end='\\n\\n')\n",
    "print(\"Error in Gi:\")\n",
    "print(npboot_std['Gi'], end='\\n\\n')\n",
    "print(\"Error in Gx:\")\n",
    "print(npboot_std['Gx'], end='\\n\\n')\n",
    "print(\"Error in Gy:\")\n",
    "print(npboot_std['Gy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Scatter plot comparing param vs. non-param bootstrapping error bars.')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbsAAAEaCAYAAACIKflVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XecVNX5x/HPwyJVWJoNaSIoCqgBRMCGERUVNHaj0RALMbGXX6xJjNHYY4kVxSCJaOwCFqwIFqRZwI4gTRFpS697fn+cuzKMM7szuzNzZ+58368XL/aWufeZ2545955zrjnnEBERibJaYQcgIiKSbUp2IiISeUp2IiISeUp2IiISeUp2IiISeUp2IiISeUWd7MzMmVmHsOMIi5m1MbOVZlYSdixSHMysXXDe1Q47ljCZ2alm9mrYcWSLmQ0ys3fCjiNWlcnOzPYzs/fMrMzMlpjZu2a2d01WmmhDmNkwM7u+JsvNlnzccZngnJvjnNvaObcp7FhEaiqdH69m9q2Z9ct2TMk45x5zzh0a1vqLUaXJzswaA6OBfwHNgB2BvwHrsh9aeor9l2K6Cml7FVKshU7b2iuW7ZDoe6b73bO9rTK2fOdc0n9AD2BZFfOcDXwOrAA+A7oF468AvokZf0wwfjdgLbAJWAksAwYDG4D1wbhRwbwtgWeAH4FZwAUx670WeBr4L7AcOCtBbMOAB4DXgjjeBtrGTHdAh+DvUmB4sK7ZwDX4HwM/izfJdmgG/Bv4DlgKPB+3jWYAS4CRQMu4GP4IfB3E+HdgZ+C94Hs9CdQJ5u0LzAOuAhYB3wKnxizrSODD4HNzgWtjprUL1nUmMAcYFzOudjDP2GD97waxvAq0iFnG6cG2WQz8OVh/vyTbo6ptf1cQ43JgCrB/ZfsW6Am8jz9evgfuqdgu6W7HuDjrBsvsEjNuG2ANsC3QAv+Db1mw/8YDtSo7J+L21aXAwiDm38VMT3i8BdMGAe8At+GPpVnA4ZWsa1Cwz+4ByoAvgINjpv+OzefoTOD3CeK8HFgA/AdoGnznH4P1jwZaxXxmLHB9sG1XAqOA5sBjwbaeBLRLEmu7YF8Nxp8r3wOXxe2PO4Np3wV/163qXMIfzw5YFcR0UrJ9F3zH8mAfrwT+RILzI1juU8F2KQvW0TnN68sFwTZfBNwav4/j5j0Hf/wuA+4FLJhWAtweLGMWcB4x522CbZzWdTPJuKT7gQTHTA6OyWqdh1vEVMUJ2xh/YXsUOBxoGjf9BGA+sDdgQIeKnR1MaxkcXCfhD8IdEu3omAPn+pjhWviL4F+AOkD7YKMcFrPTNgC/Cuatn+SCuwI4INh5dyU4wCqS3XDgBaAR/sD/CjgzWbwJ1vUi8D/8hWIr4MBg/C/xB2m3IIZ/EZxIMTG8EGzrzvhS8xvB9y3F/1D4bcxBsBH4Z7CsA4PtumvM9K7B9tgD+AH4VdxFZjjQEKhP4mT3DbBLMH0scFMwbXf8hWG/YH/cFmz/ypJdZdv+N/gLZG18QlgA1Eu2b4HuQK9g/nb4E+Wi6mzHBLE+AtwQM3wu8Erw9434C9pWwb/9CS5CVRwPFfvquuBzRwCrCc4hqj7eNuAv7CXAH/AXnITrDebfCFwcrOsk/AWmWTD9SHziN/wxs5rNP0or4rw52E/1g/1yHNAgiO8ptvzxNhafcHaO2bZfAf2C/TMc+HeSWNsF++px/HHYFX9R7hdMvw6YgP+hsQ0+of49jXOpQ8xw0n1H3A81Epwfwfgzgm1QcfH/KM3ry1v4H8Jtgm10VqJrSjDvaKBJMO+PQP9g2jnBNm6Fv768TpJkRzWum0nGVbYf+hJ3zOTgmKzWebhFTCmctLsFO3VeEMBIYLtg2hjgwpRWBB8BRydLHvw82e0DzImb50qCkyjYQeOqWOcw4ImY4a3xJbTWsScH/oKyHtg9Zt7fA2OTxRu3nh3wvxSbJpg2FLglLoYNBL98gxj2jZk+Bbg8Zvh24M64g6BhzPQngT8nietO4I64k7l9ghM8NtldEzP9j2y+6P8FeDxmWoNgm1WW7JJu+wTzLwX2TGPfXgQ8F3exSGk7JlhWP+CbmOF3gdODv6/DJ6UOlcWTYJl98SWH2jHjFuITdirH24y4be2A7ZOsaxBxyRCYCJyWZP7nCc7bIM71BD80ksy/F7A0ZngscHXctn05ZnggMUkhblkVx1ynmHG3AEODv78BjoiZdhjwbRrnUmyyS7rvSJ7s2ieKO5inSTBPaSrHeDBv/7jz6Y2YfRaf7PaLGX4SuCL4+022LPn0I3myS/u6mWRcZfshlWMmo8dkZfsy1X9VVlBxzn3unBvknGsFdMGX1u4MJrcONsrPmNnpZvaRmS0zs2XBZ1tUtb4YbYGWFZ8PlnEVsF3MPHNTWM5P8zjnVuKLwC3j5mmB/7UwO2bcbPwzylS0BpY455YmmNYydrlBDIvjlv1DzN9rEgxvHTO81Dm3Ki7OlgBmto+ZvWVmP5pZGf4XYfw2r2qbLYj5e3XMuluy5bZcHXyPyiTd9mZ2mZl9HlR8WoYvIbRI9Nlg/l3MbLSZLTCz5cA/Eny3dLZjrLeABsH2a4e/uD8XTLsVX4p51cxmmtkVVXznWIudcxtjhiu2ZyrH20/7IdjWAFub2f5BDdqVZvZpzPzzXXBViFlexbY+3MwmBBXMluFLmbHb7kfn3NqKATNrYGYPmtnsYFuPA5rE1dqt7rauELt/f4qVuPOlsmlJzqVY1dl3P8VlZiVmdpOZfRNsh2+DSQmP0yTXl2TfM5GUzj0qP4ere92MH1fZfoC4YyaJjB2T1Ow8BNJseuCc+wL/a6ZLMGouvii6BTNrCzyEv7fc3DnXBJiOL7KC/1Xys8XHDc8FZjnnmsT8a+ScO6KSzyTSOiaurfG3FL6Lm2cR/hdi25hxbfC3aFNZz1ygmZk1STDtu9jlmllD/G2i+QnmTUXTYBmxcVZ8nxH4kndr51wpvthvcZ9PZZsl8j3+NgoAZlZxu6syCbe9me2Pf05yIr403AR/iyM21vg478ff9+/onGuMP4Hjv1u1OF8b9Ung18G/0c65FcG0Fc65S51z7YGjgEvM7OAarrKq462yWMc7X4N2a+dc55hJO5pZ7PZog9/WdfHPb27D35FpArxE5dv6UmBXYJ9gWx8QjM/I9g60jvk79hje4nypbFpV51IV+y7ZeRA7/hTgaHxJqhRf+oMtt0NV15dk3zMdW5x7ccuMV93rZvy4yvZDsmXEy9gxmYnzsKramJ3M7FIzaxUMt8ZfDCYEszwMXGZm3c3rECS6hkGwPwaf+x2bEyT4X4GtzKxO3Lj2McMTgRVmdrmZ1Q9+ZXWpRrOHI4LmE3XwlRYmOOe2+BUTc7G7wcwaBd/hEvwD22Txxn7+e+Bl4D4za2pmW5lZxQXiceB3ZrZXsJP/AXzgnPs2ze8R629mVidIGgPwz1TAP1tY4pxba2Y98SdrpjwNDDSzPsF2uJaqL37Jtn0j/O3YH4HaZvYX/LO2yjTCPzxfaWad8M+xMmkE/rnCqcHfAJjZgOC4NnxC3oS/ZV1tKRxv1bEtcEFw7J2Af/zwEv65TV38tt5oZocDVVV5b4QvnS0zs2bAX2sQVzJ/DkqQnfGVFf4XjH8cuMbMtjGzFvjb5/+NmVbZubTFNaSKfRd/vUmkEf7Z72L8reR/JJinquvL/wXXhNbAhTHfMx1PAhea2Y7BD+rLK5k3U9fNyvZDqjJ2TGbiPKyqZLcCfw/4AzNbhU9y0/G//HDOPQXcgL84rMDfd23mnPsMfx//ffxB1RX/HKTCm8CnwAIzWxSMGwrsHhS9nw8uCAPwt5Rm4X8NP4z/hZWOEfiTdQm+ksNvksx3Pr6yx0x8TbgR+IoLyeKNdxr+1/oX+GczFwE4517H11x8Bv8LbWfg5DS/Q6wF+Odb3+Frv50TlLjBPxO4zsxW4A/OJ2uwni045z7Fb6Mn8N9jJf57VtYMJdm2HwO8gn9gPxtf27Wq26uX4ZP3Cvxdg+pcNJJyzn2A3/8t8T9cKnTEVwhYiT+e73POvQVgZi+b2VXVXGVlx1t1fBDEugh/Th7vnFsclFAvwB8LS/HbcGQVy7oTXylgEf6cf6UGcSXzNv621BvAbc65igbW1wOTgU+AacDUYFwq59K1wKPBNeREKtl3+AoP1wTzXpYkxuH443M+voLIhATzVHV9eQH//PgjfCW2oUm3SHIP4WtGf4Kvbf0S/sfiz9rHZvC6mXQ/pCGTx2SNz8OKmkmRZGbDgHnOuWvCjiUTzKwv8N/g+WnYsWyNrwbc0Tk3K8H0YURo2+czMxuEr+W3X9ixFJOqjnEzc/jzY0aG13s48IBzrm2VM8tPirq7MEmPmQ0Mbj01xN9vn8bmh/YikgXB7cgjzKy2me2IL0k+V9XnZEsFk+zM7Fdm9pCZ/c/M1M1OOI5mcyPTjsDJLsq3BkTyg+F7rlqKv435Of4xhaQhJ7cxzewR/H3khc65LjHj++MbYpYADzvnbkphWU3x9/jPzFa8IiISLblKdgfgHywOr0h25tvsfAUcgm+wPglf07ME//A41hnOuYXB524HHnPOTc164CIiEgk56ezUOTfOfGPdWD3xvUTMBDCzJ/A9rNyILwVuIahyehO+pwYlOhERSVmYPXvvyJbVzefhmzkkcz5B404z6+CceyB+BjMbjO9gloYNG3bv1KlTBsMVEYmwlT/A2uVMmb18kXNum7DDybSCeY2Fc+5u4O4q5hkCDAHo0aOHmzx5ci5CExEpPOWbYOJDULoj7DbQD2NYScnsKj9bgMKsjTmfLbu9aUX1u9ASEZFU/fglPNIfXrkcvnjJj6tVArUKpoJ+2sIs2U0COprZTvgkdzKZ7d5KRERibdoA794Jb98CdRrCMUNgjxPDjioncpLGzexxfBcvu5rZPDM7M+gN/jx811GfA08GXVKJiEg2zHgd3rweOh0J506CPU8Cy2T/3vkrct2FmdlAYGCHDh3O/vrrr8MOR0QkXBvWwPyp0G5fcA7mToQ2yesCmtkU51yPHEaYE5G7QeucG+WcG1xamm6/pyIiEfPtu3D/vvDY8bBqsS/FVZLooixyyU5EpOitXQ6jL4FhR4DbBL9+AhpW9frJaCuYpgciIpKCdSt9aW75POh9Hhx0la+MUuSU7EREomDDWtiqHtTdGnqeDW37QKvIPXqrtsjdxgxeQzOkrKws7FBERLLPOZj+DNzZFeZ84Mfte4ESXZzIJTtVUBGRorH8e3jiFHj6DN8TSr3GYUeUt3QbU0SkEH00Al6+Ajatg0Ovh33+ACW6pCejLSMiUohWLYLtu8JRd0PzncOOJu8p2YmIFILyTTDhfmjSGnY/Gnqf62tbRrg/y0zSVhIRyXc/fAZDD4FXr4avXvXjIt5xc6ZFrmQX011Y2KGIiNTMxvXwzj9h3G2+8slxQ6HLcWFHVZAi97NAtTFFJDK+eQPG3gidfwXnToSuxxdNx82ZFrmSnYhIQVu/GuZPgZ32h136w1lvqM1cBkSuZCciUrBmjYf7+8BjJ2zuuFmJLiOU7EREwra2DEZdCI8O8MOnPln0HTdnmm5jioiEad0KuK8PrPgO+pwPfa+COg3CjipylOxERMKwYQ1sVR/qNoJe5/iOm3fsHnZUkaXbmCIiueQcTHs66Lh5gh/X53wluiyLXLLTWw9EJG+VzYfHT4ZnzoQmbaFek7AjKhqRS3ZqZycieenD/8K9+8CscXDYjXDmq7Btp7CjKhp6Zicikgurl8CO3WDgXdBsp7CjKTpKdiIi2bBpI0y4D5q23dxxc5/z1QNKSCJ3G1NEJHQLpsPQfvDan+Hr1/y4WiVKdCFSyU5EJFM2roPxt/t/9ZrA8f+GzseEHZWgZCcikjnfvAlv3wx7nAT9b4IGzcKOSAJKdiIiNbF+FcybDO0P9B03n/2m2szlocg9s1M7OxHJmZlj4b7eMOKkzR03K9HlpcglO7WzE5GsW7MMXjgPhh8NtWrDb55Rx815TrcxRUTSsW6Ffw3PigWw70XQ9wrfx6XkNSU7EZFUrF/t30ZQtxH0Pg/a9oaWvwg7KklR5G5jiohklHPw8f/gzi4w+30/rvcflegKjEp2IiLJLJsLoy+GGa9Bq57QQM/lCpWSnYhIIlOHwytXgiuH/jdDz7N9LyhSkJTsRCTSpsxeyoSZi+nVvjnd2zZN/YNrl0OrvWHgndC0Xdbik9xQshORnKl24qnB+k59eALrN5ZTp3YtHjurV/L1btoI79/jO27ufAz0+qPvvFn9WUaCkp2I5ERaiaeay49PpBNmLmb9xnLKHWzYWM6EmYsTr3PBNHjhXPj+Y+j2W5/saqn+XpQo2YlITqSceKohWSLt1b45dWrXYsPGcraqXYte7eMqmGxYC+NuhXfvhPpN4YRH/et4JHIil+zMbCAwsEOHDmGHIlI0Urk9WWXiqYFkibR726Y8dlav5LHNfAvG3wZ7ngKH3aCOmyPMnHNhx5AVPXr0cJMnTw47DJHIS+f2ZLae2VXEUJFIK71Fum4lzJ8M7fv6NnTffwwt98pYLIXOzKY453qEHUemRa5kJyK5lc7tyYrSVqZVWYKrMOMNGHURrF4EF033/Vkq0RUFJTsRqZFs3p5MR6WJdM1SGHM1fPQYNO8Iv3lWHTcXGSU7kSKRrVuIKZeqwrJuhX8Nz8qFsN8lcODlsFW9sKOSHFOyEykC2a72n63bkzWyfhXUaeg7bt73QmjbB3bYM+yoJCRqSCJSBBI9V4ss5+CjEXBHF5j9nh/X6w9KdEVOJTuRIpAvz9WybulsGH0RfPMmtO4FDbcJOyLJE0p2IkUg75+rZcLkf/tKKGZwxG3Q40z1giI/UbITKRJ5+Vwtkzaspmzb7rzQ6k903q4LzC2LdnKXtCjZiUhh2rQB3rsbmu4EXY5lyg4nc+pLO7F+5nJqvzsBnGNjuctKhRwpPCrjixShKbOXcu9bM5gye2mo66l2HN9/DA8dBG9cB9+OB2DCrKWs3+h+qoSzYZMrjgo5khKV7ESKTLabIaS6nmrFsWEtvH0TvHs3NGwBJ/0XdhsIbFkJp6SkFjjHpnIX7Qo5krLIJTt1BC3FpDoNxbP59oFU1lMR8/xla9KPY+ZYeOcO2Os3cNj1/k0FgfhKOBUx6JmdQASTnXNuFDCqR48eZ4cdi0g2VbeElqtmCPElrfnL1jDigzlcN/pT1m8sp3ZJLWrXsqpLX+tWwLzJsPNBsGt/+P24pG3m4ivhKMlJhcglO5FiUd0SWk2bIaRamqxYz7NT5/HU5Lk8MXEOtcwod/5Z2qZN5Zzcsw0tm9RPvqyvX4fRF7Fp1SL+3XM0v9h1Z7q3VeNwSZ+SnUiBqkkJrbrNENItTXZv25QJMxezsdwnOJyjVi3D8KW5Y7u1Svz51UtgzFXw8eOsKe3A79ZdycS3fqDO+B9Vs1KqRclOpECF0VC8OqXJ+KT8lwGdWbp6ffKYKzpuXr0IDvgTw9wxTHx9VtafMUq0KdmJFLBcNxSvTmky5aS8boXvtLluI9j/UmjbG7bvSs/ZS6kzdnb0uzqTrNKbykWKVHVf+ZPxVwU5Bx/+F169Bk4eAe32zf46JSm9qVxEIqMmbe0yWppc+i2MutA3KWi7LzTaPvvrlKKkZCdShGKfva3bUM6zU+flPplMGupLc1YCR/4Tuv+uWh03q9QnqVB3YSJFqFf75tQu8ae/A56aPDfrXYf9zMZ10HZfPjl6DPeuPJApc8vSXkRFCfX2V7/k1Icn5P47SMFQshMpQt3bNuX47q2wYHhTuct+/5GbNsDbt8L0Z/zwPucwZb8hnPjEnGonq0S1Q7PWH6cUNN3GFClSx3VrxbNT5+WmluP8qTDyfPhhun/PXJfjoFYtJsxaUqOuy+JrhzZtUCfz/XFKJCjZiRSpTLfTS/jsbMMaeOsf8P490HBbX9uy05E/faamXZfFf4eq2gHmql9QyT9KdiJFLFO1HJOWmGa+7d851+10OOTvUL/Jz9Zf04Qb/x0qS5656hdU8o/a2YkUmHysfXjvWzO4/dUvKXdQaqv5e481HHXc6X7igmmwfdecxVLV9snH7ZdP1M5OREKXr8+cKkpM+5ZP4fraQ9n28zWwegA0aJbTRAdVl1bVZq84qTamSAFJ9MwpH3RvUc67HZ9g6Fa30qRpC0oGjfaJTiRPqGQnUkDy8pnTuhVwf2+ar14CB15B/f0vhdp1wo5KZAtKdiIFJIw3HSQV23HzAf8HbfvAdp3DiyeD9FwvepTsRApM6M+cnIOpw+G1P8PJj/uOm3ueHV48GZavz0WlZiL3zM7MBprZkLKy9LseEpEqLJkJjw6EURfAdl2TdtxcyPL1uajUTOSSnXNulHNucGlpadihiETLxIfgvj7w3Ucw4E747ShovnPYUWVcxXPREiN/notKjek2poikpnwj7HQADLgDSncMO5qsyavnopIxalQuErIRH8zh5enfc3iXHThlnzZhh7PZxvXwzj+heQfoejyUl4OZ/xcCVRrJDTUqF5GMG/HBHK56bhoA479eBJAfCW/eFBh5Hiz8DHoO9smuGu+ayxRVGpGaitwzO5FC8vL07ysdzrn1q2HM1TC0H6xZBr/+Hxxxa7gxoUojUnNKdiIhOrzLDpUO59y34/0bCrr9Fs6dALv2DzeegCqNSE3pmZ1IyEJ/Zre2DOZOgo79/PAPn8F2u6e9mGx/Dz2zy42oPrNLOdmZ2aPAhc65ZcFwU+B259wZWYyv2pTsRFLw5csw+mKf8C7+tNr9WcY+ewT4xzFd8+PZo6QtqskunduYe1QkOgDn3FLgF5kPSUSybtUiePpMePxkqN8MBo1myo/GvW/NYMrspWkvLu+ePYrESac2Zi0zaxokOcysWZqfF5F8sG4F3Ncb1iyFvlfBfhczZf6qGtV2PLzLDj/VJq0YFskn6SSr24H3zewpwIDjgRuyEpWIZN7aMqhX6jtu7ns5tN0Xtt0NgGenfsm6DeU4Ntd2TCfZVdyyzMv2giKkkeycc8PNbDLwy2DUsc65z7ITlohkTHk5TB0Gr10LJz8GO+0Pe5/10+Qps5fy1OS5VDy9LympXm3HU/ZpoyQneavKZGdmjZ1zy4PblguAETHTmjnnlmQzQBGpgcXfwMgLYPY7vquv0lY/m2XCzMVsLPepzoDju7dSbUeJnFRKdiOAAcAUILbqpgXD7bMQl4jU1AdD/Gt4SurCUf+CX5yWsKuv+BfCHtft5wlRpNBVmeyccwPMzIADnXNzchCTiGSCK4edD4Yjb4fGySuMqONjKQbptLOb5pzrmuV4Mkbt7KTobFwH42+H5h1hjxP8S1YhtI6bpTCpnR1MNbO9sxaJiFTf3Enw4AHw9s0wP/iRF+IbCkTyTTpND/YBTjWz2cAqgmd2zrk9shKZiFRt/Sp48waYcB803hFOfRo6HhJqSOrWS/JROsnusKxFISLV8+07MOFe35Tg4L9CvcahhqNX8Ui+Suc25vXOudmx/4DrsxWYiCSxZhl8Ncb/vcth8McPfCWUkBMd6FU8kr/SKdl1jh0wsxKge2bDEZFkpsxeyg8Tn6bfzFuos3Hl5o6bt+0Udmg/iW/GoFfxSL5IpVH5lcBVQH0zW45/VgewHhiSxdhEClomn119/MVXLBxxPkfUmsDnri0cNZTdqvmGgmxSMwbJV6m0s7sRuNHMbnTOXZmDmEQKXkafXa1dTsdnDqWTreCWDScytHwAFyxvyW6ZDTljurdtqiQneSedZ3ZXm9lvzOzPAGbW2sx6ZikukYKWkWdXa4I3atVrzI89LuOY8pt5sPxXWO06uj0okqZ0kt29QG/glGB4ZTBOROJUPLsqMdiqdi2aNqiT+rviysth4kNwRxeYNR6Atoedx9/POo5LDt1VNRxFqiGtdnbOuW5m9iH4l7eaWZ0sxSVS0GKfXTVtUIfrRn+a2i3NRTNg5Pkw5z1ofxA0abPFMpXkRKonnZLdhqAGpgMws22A8qxEJRIB3ds25dyDOrB09frUbmlOeADu7wMLP4Wj74PTnoOmbXMbtEhEpVOyuxt4DtjWzG7Av7z1mqxEJRIhKVfHt1q+95Mjb4dG2+c2SJGIS7kjaAAz6wQcjG9+8IZz7vNsBZZg3bsBFwItgnXfX9n86ghacq2ypgYJp21YC+NugW06wR4n+o6b1ZelhCyqHUGnU7ID+AEYH3yuvpl1c85NrepDZvYI/p14C51zXWLG9wfuAkqAh51zNyVbRpBYzzGzWsBwoNJkJ5JLVTU1+NnztjkfwMjzYNFX0OuPPtkp0YlkTcrJzsz+DgwCvmHzS1wd8MsUPj4MuAefpCqWV4KvzXkIMA+YZGYj8YnvxrjPn+GcW2hmRwF/AP6TatwiuZCoqUHCyiTrVsIb18HEIVDaGn7zLHQ4OPcBZ4k6gZZ8lU7J7kRgZ+fc+nRX4pwbZ2bt4kb3BGY452YCmNkTwNFBI/YBSZYzEhhpZi/i36C+BTMbDAwGaNOmTfxkkaxJ+bnc7Pd8ous5GA7+C9TdOreBZpE6gZZ8lk6ymw40ARZmaN07AnNjhufhXyOUkJn1BY4F6gIvJZrHOTeEoAuzHj16pP4wUqSGKu0ma/USmDsRdu0PuxwK506EbXYJL9gsSbl0KxKCdJLdjcCHZjYdWFcx0jl3VMajSsA5NxYYm4t1iVTndlzCdnCfvQAvXgbrYzpujmCiA3UCLfktnWT3KHAzMI3MtK+bD7SOGW4VjBMJVUZux634AV66DD4fCdt3haOf9okuwtQJtOSzdJLdaufc3Rlc9ySgo5nthE9yJ7O5KzKR0NT4dtza5b5x+LoV/oWqfc6Hkq2yF3AeUS8vkq/SSXbjzexGYCRb3sZMpenB40BfoIWZzQP+6pwbambnAWPwNTAfcc59mk7wSdY1EBjYoUOHmi5KilRlt+Mqvb25ZinUb+pfonrwn6HtvtCiY46jF5FEUm5UbmZvJRjtnHOpND3IOTUql5pIlNSS3t4sL4dJD/kmBSePgPYHhhy9SPUVfaNy59xB2QxEJJ8kuh2X8PZmg4W+4+a5H0CHftBsp5AiFpHKpNWDipkdCXQG6lWMc85dl+mgRPJR/O3No9Y8Bw830/x3AAAR2klEQVTcAnUawjEPwh4nVdoLihpci4QnnR5UHgAaAAcBD+M7gp6YpbhE8k58bcPWC+ZCpyPh8Ftg620r/awaXIuEK51X/PRxzp0OLHXO/Q3/ItdoNhgSSWTDGrp/fRfnNpviE1XPs+GEYVUmOsjQm8tFpNrSSXZrg/9Xm1lLYAOwQ+ZDqhkzG2hmQ8rKysIORaJk9nvwwH7wzh2w4BM/LsWOm6fMXsp3y9ZQu5b99OZyNbgWya10ntmNMrMmwK3AVHwn0A9lJaoacM6NAkb16NHj7LBjkQhYtwJevxYmPezfGn7a87Bz6nW1Ym9f1i6pxUk9W3Nct1a6hSmSYyklu+C1Om8455YBz5jZaKCec07FJ4m22e/BpKH+NTy/vMZXRklD7O3LTZvK2bFJfSU6kRCklOycc+Vmdi/wi2B4HTENy0UiZfUSmPO+r3yyy2Fw3mRoUb1OCtRfpEh+SOc25htmdhzwrEvn9eYihcI5+PQ5eOn/YMMauHi678+ymokO1F+kSL5IJ9n9HrgE2GhmawHD96DSOCuRieTS8u99x81fjIYd9oKj781Yx83qL1IkfOn0oNIom4FkivrGlLStLfMdN29YDYdcB73OhZK0+lsQkTyXct+YAGbWFOjIlj2ojMtCXDWmvjGlSquXbC69TR3uO25uvnO4MYmELKp9Y6bczs7MzgLG4d9S8Lfg/2uzE5ZIFpVvgvfvgzu6wMyxfly305XoRCIsnUblFwJ7A7ODTqF/ASzLSlQi2bLwC3ikP4y5EtrtC811u1ukGKTzYGKtc26tmWFmdZ1zX5jZrlmLTCTT3rsH3vgb1Nkajn0Iup6Qci8oIlLY0kl284IeVJ4HXjOzpcDs7IQlkgW168JuA6H/zbD1NmFHIyI5lFYFlZ8+ZHYgUAq84pxbn/GoMkAVVIT1q2HsjbDt7rDXr307OpXkRCoV1QoqVZbszKwecA7QAZgGDHXOvZ3twKpLTQ8EgG/f8S9VXTIT+pzvxynRiRStVCqoPAr0wCe6w4HbsxpRDTnnRjnnBpeWloYdioRh7XIYfTEMOxJcOZw+Eg69PuyoRCRkqTyz29051xXAzIaiF7ZKPpszAaYMg97nwUFXQ50GKX9UbxIXia5Ukt2Gij+ccxtNt4Ik36xa7Dtu3m0A7HKo77g5zTZzepO4SLSlchtzTzNbHvxbAexR8beZLc92gCJJOQfTnoZ794ZnB/seUaBajcP1JnGRaKuyZOecK8lFICJpWf4dvHgpfPkStOwGR99To46b9SoekWhTb7dSeH7quHmtr3zS649Qq2a/yfQqHpFoU7KTwrFqMTRsDvVK4ZC/Q9s+Ge3PUq/iEYmudPrGFAlH+Sbf1dedXeCbN/24bqep42YRSVnkSnZqVB4xP3wGI8+D+VNgl8Nhm05hRyQiBShyJTs1Ko+Qd++CBw+ApbPh+Efg149D45ZhRyUiBShyJTuJkDoNofMx0P8m/6wuC9SQXKQ4KNlJ/li/Gt66AbbrDHudAj3OhL3Pytrq1JBcpHhE7jamFKhZ4+D+3vD+PfDjF35clnvrUUNykeKhkp2Ea20ZvPpnmPooNGsPg16EdvvlZNVqSC5SPJTsJFxzJ8KH/4E+F0DfK9PquLmm1JBcpHgo2UnurfzRd9y8+1HQ8RA4fyo02ymUUNSQXKQ4KNlJ7jgH056Cly+HTethp/2hftPQEp2IFA8lO8mNsnkw+hL4egy02huOuscnOhGRHFCyk+xbWwYP7Acb1/k2cz0H17jjZhGRdEQu2am7sDyyahE0bOE7bj7sH9Cmt25ZikgoItfOTt2F5YFNG31XX3fEdNy81ylKdCISmsiV7CRkC6b7jpu/+xA6DYBtdgs7IhERJTvJoHfugDev9xVPThgGu/8q672giIikQslOMqduY+hyPPS/ERo0CzsaEZGfRO6ZneTQupXw8hXw4X/9cI8z4NgHlehEJO8o2Un1fPOm77j5g/th8Td+nG5Zikie0m1MSc+apfDqNb4017wD/O5laNsn7KhERCqlZCfpmTcFPnoc9rsYDrwCtqoXdkQiIlVSspOqrVwIs9+Dzr+Cjv3ggg+haduwoxIRSZmSnSTnHHzyP3jlCti0Adof6JsVKNGJSIFRspPEls2F0RfBjNeh9T7quFlECpqSnfxcRcfNmzbA4bfA3mdDLVXcFZHCFblkp46ga2DlQth6W99xc/+bfC1L3bIUkQiI3M91dQRdDZs2wPh/wp1dYcYbftxev1aiE5HIiFzJTtL0/Sfwwrmw4BPY7SjYrkvYEYmIZJySXTEbfzu8eQM0aA4nDofdjw47IhGRrFCyK2b1m8IeJ8FhN6g/SxGJtMg9s5NKrFsJL/0Jpv7HD/c4A465X4lORCJPya5YzHgd7usNE4fA0m/DjkZEJKd0GzPqVi+BMVfDxyOgeUc44xVo0yvsqEREckrJLurmT4VpT8L+l8IBf1LHzSJSlJTsomjFAt9xc5djN3fc3KRN2FGJiIRGyS5KnIOPHoMxV0F5Oex8kK9xqUQnIkVOyS4qls6GURfCzLegTW846l/quFlEJKBkFwVry+DB/aF8ExxxG/Q4Ux03i4jEULIrZCsWQKPtfcfNR9zmS3RNWocdlYhI3tHP/0K0aQOMuxXu3GNzx817nKhEJyKShEp2hea7j+CF8+CHadD5GNi+a9gRiYjkPSW7QjLuVnjrRmjYAk56DHYbEHZEIiIFQcmukDTcBvY6BQ79u2paioikQc/s8tm6FfDiZTB1uB/uPgiOvkeJTkQkTZFLdmY20MyGlJWVhR1KzXz9GtzbCyY9DMvmhB2NiEhBi1yyc86Ncs4NLi0tDTuU6lm9BJ79PTx2PNTdGs58FX55TdhRiYgUND2zyzffTYXpz8CBl/vOm2vXDTsiEZGCp2SXD5Z/D7Pfha7HQ4d+cOFHUNoq7KhERCJDyS5MzsGH/4Ex14Arhw4H+8onSnQiIhmlZBeWJbN8x82z3oa2+8FRd6uWpYhIlijZhWFtGTx4oC/NDbgDug1Sx80iIlmkZJdLy7+Hxjv4jpuPvB3a9oHSHcOOSkQk8lScyIWN62HszXDXHjDjdT9ujxOU6EREckQlu2ybPwVeOB8Wfgqdj4Xt9ww7IhGRoqNkl01jb4a3b4Ktt4OTH4dOR4QdkYhIUVKyy6ZG20O30+GQ6/xzOhERCYWe2WXS2jIYdRFMGeaHu/8WBt6lRCciEjKV7DLlqzE+0a1c4G9biohI3lCyq6lVi+GVy2HaU7Dt7nDSf6FV97CjEhGRGEp2NfX9R/Dp89D3StjvEqhdJ+yIREQkjpJddSz/Dma/F3TcfDBc9Ak0bhl2VCIikoSSXTrKy2Hqo/DaX/xwh35Qv4kSnYhInlOyS9Xib3zHzd+Oh3b7Bx03Nwk7KhERSYGSXSrWlsGQgwAHA+/2befMwo5KRERSpGRXmbL5vv/KeqUw8A5o01u3LEVECpAalSeycR289Q+4a0/4+jU/rstxSnQiIgVKJbt48ybDC+fBj59D1xOhZbewIxIRkRpSsos19iZ4+2ZotAOc8iTscljYEYmISAYo2cVqvCN0HwT9/gb1GocdjYiIZEhxP7NbswxGXgCT/+2Hu50GA+5QohMRiZjiLdl98RK8eAms/MGX6EREJLKKL9mt/BFe/hN8+ixs2xlOHgE7qhKKiEiUFV+y+2EafDEaDroG9r1QHTeLiBSB4kh2ZfPg23dhz5Ng51/ChZ9A4x3CjkpERHKkYCqomFlDM5tsZgNS/lB5OUx6GO7tBS/9n6+QAkp0IiJFJuvJzsweMbOFZjY9bnx/M/vSzGaY2RUpLOpy4MmUV7xxHTw6AF681L9M9Zxx6rhZRKRI5eI25jDgHmB4xQgzKwHuBQ4B5gGTzGwkUALcGPf5M4A9gc+AeimvddGXsKAMjroHfvEbddwsIlLEzDmX/ZWYtQNGO+e6BMO9gWudc4cFw1cCOOfiE13F528AGgK7A2uAY5xz5QnmGwwMDga7ANPj5ylQpUBZBNZZ02VW5/PpfiaV+TMxTwtgURpx5SsdmzVbRjqfSXXequaravquzrlGKcZUOJxzWf8HtAOmxwwfDzwcM3wacE8KyxkEDEhxnZNz8d1ytP2GRGGdNV1mdT6f7mdSmT8T80Tl+NSxWbNlpPOZVOdN4dgrimMz/l9B1cZ0zg0LO4aQjIrIOmu6zOp8Pt3PpDJ/puaJAh2bNVtGOp9Jdd6q5iuWY3MLBXEbs5rrnOyc65Gp5Ylkko5PyVdRPTbDanowCehoZjuZWR3gZGBkhtcxJMPLE8kkHZ+SryJ5bGa9ZGdmjwN98Q/kfwD+6pwbamZHAHfia2A+4py7IauBiIhI0crJbUwREZEwFUwPKiIiItWlZCciIpFXtMmuWn1timSZme1mZg+Y2dNm9oew4xGJZWa/MrOHzOx/ZnZo2PGko+CSXWh9bYpUIRPHpnPuc+fcOcCJwL7ZjFeKS4aOz+edc2cD5wAnZTPeTCu4CipmdgCwEhge026vBPiKmL42gV9TeV+bzfF9bS5yzo3OTfQSZZk4Np1zC83sKOAPwH+ccyNyFb9EW6aOz+BztwOPOeem5ij8GiuoHlQAnHPjgkbqsXoCM5xzMwHM7Ang6KCR+s9uU5pZX2L62jSzl1yCvjZF0pGJYzNYzkhgpJm9CCjZSUZk6NppwE3Ay4WU6KAAk10SOwJzY4bnAfskm9k5dzWAmQ3Cl+yU6CRb0jo2gx9ixwJ1gZeyGplImscncD7QDyg1sw7OuQeyGVwmRSXZVUsR97Upeco5NxYYG3IYIgk55+4G7g47juoouAoqScwHWscMtwrGiYRNx6bks6I5PqOS7HLR16ZIdejYlHxWNMdnwSW7oK/N94FdzWyemZ3pnNsInAeMAT4HnnTOfRpmnFJ8dGxKPiv247Pgmh6IiIikq+BKdiIiIulSshMRkchTshMRkchTshMRkchTshMRkchTshMRkchTspPIMjMX9M5eMXyZmV0bYkhZZ2btzOyUSqa3NLOncxmTSD5QspMoWwcca2Ytwg4klplls0/adkDCZGdmtZ1z3znnjs/i+kXykpKdRNlGYAhwcfyEoAT0ppl9YmZvmFmbYPwwM7vbzN4zs5lmljAxBPM9ELzt/quKN94Hyx1vZlODf32C8X2D8SOBz4Jxz5vZFDP71MwGxyx7pZndGox/3cx6mtnYIJ6jgnlKgnkmBd/h98HHbwL2N7OPzOxiMxtkZiPN7E3gjSC+6THLuM3MpgfLOD8zm10kDznn9E//IvkP/6LKxsC3QClwGXBtMG0U8Nvg7zOA54O/hwFP4X8I7o5/11eiZQ8DXgnm64h/NUo9oAFQL5inIzA5+LsvsArYKWYZzYL/6wPTgebBsAMOD/5+DngV2Ar/0uGPgvGDgWuCv+sCk4GdgvWMjlnHoCC2inW1A6YHf/8BeBqoHRuP/ulfFP8V9St+JPqcc8vNbDhwAbAmZlJv/HvjAP4D3BIz7Xnn33H4mZltV8ninwzm+9rMZgKdgFnAPWa2F7AJ2CVm/onOuVkxwxeY2THB363xyXExsB6fSAGmAeuccxvMbBo+WQEcCuwRU/IsDT6/PkGcrznnliQY3w94wPn+EUkyj0gkKNlJMbgTmAr8O8X518X8bQBmdgNwJIBzbq9gWnzHsg5/y/QHfCmsFrA2ZvqqnxbqX9LaD+jtnFttZmPxJUOADc65imWXV8TjnCuPed5nwPnOuTGxAQTLjbcqwTiRoqJndhJ5QYnlSeDMmNHv4V9nAnAqML6KZVztnNsrJtEBnGBmtcxsZ6A98CW+hPV9UOI7DShJsshSYGmQ6DoBvdL8WmOAP5jZVgBmtouZNQRWAI1SXMZrwO8rEqiZNUszBpGCoWQnxeJ2ILZW5vnA78zsE3xSurAay5wDTAReBs5xzq0F7gN+a2Yf429rJitVvQLUNrPP8ZVKJqS57ofxFV2mBhVOHsTfqfkE2GRmH5vZzyrmJFjGHOCTIN6kTRZECp1e8SNSDWY2DF8RRG3WRAqASnYiIhJ5KtmJiEjkqWQnIiKRp2QnIiKRp2QnIiKRp2QnIiKRp2QnIiKRp2QnIiKR9/9Aqd+ejSNfeAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "loglog(npboot_std.to_vector(),pboot_std.to_vector(),'.')\n",
    "loglog(np.logspace(-4,-2,10),np.logspace(-4,-2,10),'--')\n",
    "xlabel('Non-parametric')\n",
    "ylabel('Parametric')\n",
    "xlim((1e-4,1e-2)); ylim((1e-4,1e-2))\n",
    "title('Scatter plot comparing param vs. non-param bootstrapping error bars.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}