full_reportD.tex

\documentclass{article}[11pt]
\usepackage{longtable}
\usepackage{graphicx}
\usepackage{fix-cm}
\usepackage[margin=1in,paperwidth=8.5in,paperheight=11in]{geometry}
\usepackage[section]{placeins}
\usepackage{flafter}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{etoolbox}
\usepackage{units}
\usepackage{multirow}
\usepackage{adjustbox}
\usepackage{hyperref}
\usepackage{pdfcomment}
\usepackage{color}
\usepackage{spverbatim}

\setcounter{topnumber}{3}
\setcounter{bottomnumber}{3}
\setcounter{totalnumber}{4}
\renewcommand{\topfraction}{0.9}
\renewcommand{\bottomfraction}{0.9}
\renewcommand{\textfraction}{0.1}
\renewcommand{\floatpagefraction}{0.7}

\newcommand{\rrangle}{\rangle\!\rangle} \newcommand{\llangle}{\langle\!\langle}
\newcommand{\ket}[1]{\ensuremath{\left|#1\right\rangle}}
\newcommand{\bra}[1]{\ensuremath{\left\langle#1\right|}}
\newcommand{\braket}[2]{\ensuremath{\left\langle#1|#2\right\rangle}}
\newcommand{\expec}[1]{\ensuremath{\left\langle#1\right\rangle}}
\newcommand{\ketbra}[2]{\ket{#1}\!\!\bra{#2}}
\newcommand{\braopket}[3]{\ensuremath{\bra{#1}#2\ket{#3}}}
\newcommand{\proj}[1]{\ketbra{#1}{#1}}
\newcommand{\sket}[1]{\ensuremath{\left|#1\right\rrangle}}
\newcommand{\sbra}[1]{\ensuremath{\left\llangle#1\right|}}
\newcommand{\sbraket}[2]{\ensuremath{\left\llangle#1|#2\right\rrangle}}
\newcommand{\sketbra}[2]{\sket{#1}\!\!\sbra{#2}}
\newcommand{\sbraopket}[3]{\ensuremath{\sbra{#1}#2\sket{#3}}}
\newcommand{\sproj}[1]{\sketbra{#1}{#1}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\def\Id{1\!\mathrm{l}}
\newcommand{\Tr}[0]{\mathrm{Tr}}
\providecommand{\e}[1]{\ensuremath{\times 10^{#1}}}

%Command used for python automatic substitution
\newcommand{\putfield}[2]{#2}

\newtoggle{confidences}
\newtoggle{LsAndGermsSet}
\newtoggle{debuggingaidsappendix}
\newtoggle{gaugeoptappendix}
\newtoggle{pixelplotsappendix}
\newtoggle{whackamoleappendix}
\toggletrue{confidences}
\toggletrue{LsAndGermsSet}
\toggletrue{debuggingaidsappendix}
\toggletrue{gaugeoptappendix}
\toggletrue{pixelplotsappendix}
\toggletrue{whackamoleappendix}


\hypersetup{
  pdfinfo={ Author={pyGSTi},
Title={GST report for $\mathcal{D}$},
Keywords={GST},
pyGSTi_Version={0.9.3},
opt_long_tables={False},
opt_table_class={pygstiTbl},
opt_template_path={None},
opt_latex_cmd={pdflatex},
L_germ_tuple_base_string_dict={Dict[[0, Gx]: {}, [0, Gy]: {}, [0, Gi]: {}, [0, GxGy]: {}, [0, GxGyGi]: {}, [0, GxGiGy]: {}, [0, GxGiGi]: {}, [0, GyGiGi]: {}, [0, GxGxGiGy]: {}, [0, GxGyGyGi]: {}, [0, GxGxGyGxGyGy]: {}, [1, Gx]: [Gx], [1, Gy]: [Gy], [1, Gi]: [Gi], [1, GxGy]: {}, [1, GxGyGi]: {}, [1, GxGiGy]: {}, [1, GxGiGi]: {}, [1, GyGiGi]: {}, [1, GxGxGiGy]: {}, [1, GxGyGyGi]: {}, [1, GxGxGyGxGyGy]: {}, [2, Gx]: [Gx]2, [2, Gy]: [Gy]2, [2, Gi]: [Gi]2, [2, GxGy]: [GxGy], [2, GxGyGi]: {}, [2, GxGiGy]: {}, [2, GxGiGi]: {}, [2, GyGiGi]: {}, [2, GxGxGiGy]: {}, [2, GxGyGyGi]: {}, [2, GxGxGyGxGyGy]: {}, [4, Gx]: [Gx]4, [4, Gy]: [Gy]4, [4, Gi]: [Gi]4, [4, GxGy]: [GxGy]2, [4, GxGyGi]: [GxGyGi], [4, GxGiGy]: [GxGiGy], [4, GxGiGi]: [GxGiGi], [4, GyGiGi]: [GyGiGi], [4, GxGxGiGy]: [GxGxGiGy], [4, GxGyGyGi]: [GxGyGyGi], [4, GxGxGyGxGyGy]: {}, [8, Gx]: [Gx]8, [8, Gy]: [Gy]8, [8, Gi]: [Gi]8, [8, GxGy]: [GxGy]4, [8, GxGyGi]: [GxGyGi]2, [8, GxGiGy]: [GxGiGy]2, [8, GxGiGi]: [GxGiGi]2, [8, GyGiGi]: [GyGiGi]2, [8, GxGxGiGy]: [GxGxGiGy]2, [8, GxGyGyGi]: [GxGyGyGi]2, [8, GxGxGyGxGyGy]: [GxGxGyGxGyGy]]},
cptpPenaltyFactor={0},
defaultBasename={None},
defaultDirectory={None},
distributeMethod={deriv},
fiducial_pairs={None},
gaugeOptParams={Dict[]},
hessianProjection={optimal gate CIs},
linlogPercentile={5},
max_length_list={[0, 1, 2, 4, 8]},
memLimit={None},
minProbClip={1e-06},
minProbClipForWeighting={0.0001},
objective={logl},
probClipInterval={[-1000000.0, 1000000.0]},
radius={0.0001},
weights={None}  }
}


\begin{document}

\title{GST report for $\mathcal{D}$}
\date{\vspace{-1cm}\today}
%\author{}

\begingroup
\let\center\flushleft
\let\endcenter\endflushleft
\maketitle
\endgroup

\section{Overview}
This report presents a gate-set tomography (GST) analysis of a dataset called ``$\mathcal{D}$''.  

GST characterizes logic operations on a quantum device (e.g., a qubit), by treating it as a black box.  This black box is equipped with a small set of ``buttons'' that apply quantum \emph{gates} to the quantum system inside.  One button initializes it, a second button triggers a 2-outcome measurement, and the remaining buttons perform transformations.  We avoid assumptions about the device's operation whenever possible.  Currently, we assume that:
\begin{itemize}
\item the quantum device is a qubit (has a Hilbert space of dimension 2),
\item each \emph{gate}, or logic operation, can be represented by a stationary Markov process (a.k.a. ``quantum channel'').
\end{itemize}
The core of GST is an algorithm that takes certain inputs, and produces certain outputs.  The \emph{input} to GST comprises (1) a list of data, and (2) ``target'' gate set describing the \emph{ideal} behavior of the device.  GST data comprises a list of experiments -- each described by the sequence of gates that was applied -- and, for each experiment, two integer \emph{counts} stating how often the ``plus'' and ``minus'' results were observed.  The target gates are used \emph{only} to (a) report how consistent the estimates are with the target, and (b) choose the best \emph{gauge} in which to report the results.  GST does not take them into account in its core analysis, and there is no possibility of circularity or other ``cheating''.

GST's primary output is an estimated \emph{gate set} that models or fits the device's observed behavior.  Gate sets are of the form $\{\rho_0,E_0,\{G_k\}\}$, where
\begin{itemize}
\item $\rho_0$ is an estimate of the density matrix in which the device gets initialized,
\item $\{E_0,\Id-E_0\}$ is an estimate of the POVM describing how it gets measured,
\item and each of the $G_k$ is an estimate of the superoperator (quantum process) describing the corresponding gate.
\end{itemize}
Unless something went wrong (usually it doesn't), the output of GST is the best possible fit to the data.  This should also mean that the output is a very accurate description of what happens when you trigger a gate on your device.  However, this happy conclusion relies on two assumptions:
\begin{enumerate}
\item The experiments were chosen wisely, so that the only gate sets consistent with their results are very close to the true behavior.  This is usually true.  The main failure mode occurs when you were not able to perform \emph{long} sequences (e.g., because your decoherence rate is very high), in which case accuracy may be limited. 
\item The operations you are performing really are stationary (time-independent), Markovian, and acting on a quantum system with the correct Hilbert space dimension.  These assumptions define the \emph{model} that GST fits to the data.  \textbf{They are usually not true!}  Quantum operations are usually at least a little bit non-Markovian.  In this report (Section \ref{secGoodness}) we provide extensive self-checks to identify and diagnose violations of the model.  If your system \emph{is} visibly non-Markovian, then (a) these checks will probably warn you of it, and (b) the other quantities reported here should be treated with caution -- using GST on non-Markovian gates violates the warranty!
\end{enumerate}

This document is organized into three main sections, which address three broad questions.
\begin{itemize}
\item Section \ref{secInput}:  What inputs did you give GST?
\item Section \ref{secOutput}:  What estimate did GST output, and what does it mean?
\item Section \ref{secGoodness}:  How reliable are the results? (How badly was the model violated?)
\end{itemize}
Section \ref{secInput} is primarily useful to verify that the inputs were correct.  Section \ref{secOutput} is the most important:  it presents the raw estimates derived by the GST algorithm, and also provides a variety of derived quantities that may be useful in interpreting what this estimate means.

Section \ref{secGoodness} is dedicated to summarizing how well the model imposed by GST was able to fit the data, relative to what is expected of a ``good'' model.  This is \emph{not} related to ``How close is the GST estimate to the target gates?'', which is addressed in Section \ref{secOutput}.  It is also not the same as ``How large are the error bars on the GST estimate?'', which is a good question that is addressed in section \ref{secOutput} when this report is generated with the confidence intervals option turned on.  Instead, Section \ref{secGoodness} is intended to tell you whether (a) you should take the GST estimate at face value, or (b) it should be treated skeptically because \emph{no} gate set was capable of fitting the data.

Finally, appendices may be present (depending on which options were chosen when this report was generated).  Appendices present more detailed debugging information, elaborating on the goodness-of-fit metrics presented in Section \ref{secGoodness}.

\section{Input Summary\label{secInput}}
The input for this GST analysis comprised: (1) a target gate set (see Tables \ref{targetSpamTable}-\ref{targetGatesTable}); and (2) a dataset called ``$\mathcal{D}$''.

\subsection{Target Gate set}

The target gate set describes the ideal initial state (density matrix), measurement (POVM effect), and gate operations (superoperators).  Typically, density matrices and POVM effects are represented as square $d\times d$ matrices on a Hilbert space $\mathcal{H}$.  In GST, it is often more convenient to represent them as $d^2$-element vectors in the Hilbert-Schmidt space $\mathcal{B}(\mathcal{H})$ of linear operators on $\mathcal{H}$.  Both representations are shown in Table \ref{targetSpamTable}.  Superoperators are sometimes represented in Choi or Kraus form, but for GST it is more convenient to represent them as square $d^2\times d^2$ matrices that multiply associatively and act on $\mathcal{B}(\mathcal{H})$.  These are shown in Table \ref{targetGatesTable}.

These Hilbert-Schmidt space representations require choosing a basis $\{M_i\}$ for $\mathcal{B}(\mathcal{H})$.  We use the \emph{Pauli basis}, comprising the four $2\times2$ Pauli matrices (including the identity $\Id$) for $d=2$.  In $d>2$, we use the analogous Gell-Mann matrices as a basis.   The choice of this basis is what is meant when state preparations and measurements are written as vectors and gate operations are written as matrices in the ``Pauli basis''.  Keep in mind that we want to use an orthonormal basis, so the basis matrices are normalized so that $\sbraket{M_i}{M_j} = \Tr M_i^\dagger M_j = \delta_{ij}$.  In $d=2$, this means that the basis matrices are $M_i = \frac{1}{\sqrt{2}}\sigma_i$.

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|c|}
\hline
Operator & Matrix & Hilbert-Schmidt vector (Pauli basis) \\ \hline
$\rho_{0}$ & $ \left(\!\!\begin{array}{cc}
1 & 0 \\ 
0 & 0
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0.707 \\ 
0 \\ 
0 \\ 
0.707
 \end{array} $
 \\ \hline
$E_{0}$ & $ \left(\!\!\begin{array}{cc}
0 & 0 \\ 
0 & 1
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0.707 \\ 
0 \\ 
0 \\ 
-0.707
 \end{array} $
 \\ \hline
\end{tabular}

\caption{\textbf{Target gate set: SPAM (state preparation and measurement) gates}.  These are the \emph{ideal} input state ($\rho_0$) and `plus' POVM effect $E_0$ for the device on which we report.  SPAM gates are given here both as $d\times d$ matrices, and in ``vectorized'' form as $d^2$-dimensional vectors in $\mathcal{B}(\mathcal{H})$.  See Table \ref{bestGatesetSpamTable} for GST estimates of the actual $\rho_0$ and $E_0$ implemented in this experiment.\label{targetSpamTable}}
\end{center}
\end{table}

The ideal state prepration and measurement (SPAM) operations for your particular case are given in Table \ref{targetSpamTable}.  The ideal \emph{logic gate} operations are given, as superoperators written in the Pauli basis, in Table \ref{targetGatesTable}.

In most cases, the ideal/target logic gates are reversible unitary rotations.  The corresponding superoperators are orthogonal rotations on $\mathcal{B}(\mathcal{H})$.  For your convenience, Table \ref{targetGatesTable} also lists (for each logic gate) an axis of rotation [as a vector in $\mathcal{B}(\mathcal{H})$] and an angle of rotation.  

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|c|c|}
\hline
Gate & Superoperator (Pauli basis) & Rotation axis & Angle \\ \hline
Gi & $ \left(\!\!\begin{array}{cccc}
1 & 0 & 0 & 0 \\ 
0 & 1 & 0 & 0 \\ 
0 & 0 & 1 & 0 \\ 
0 & 0 & 0 & 1
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0 \\ 
1 \\ 
0 \\ 
0
 \end{array} $
 & 0$\pi$ \\ \hline
Gx & $ \left(\!\!\begin{array}{cccc}
1 & 0 & 0 & 0 \\ 
0 & 1 & 0 & 0 \\ 
0 & 0 & 0 & -1 \\ 
0 & 0 & 1 & 0
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0 \\ 
1 \\ 
0 \\ 
0
 \end{array} $
 & 0.5$\pi$ \\ \hline
Gy & $ \left(\!\!\begin{array}{cccc}
1 & 0 & 0 & 0 \\ 
0 & 0 & 0 & 1 \\ 
0 & 0 & 1 & 0 \\ 
0 & -1 & 0 & 0
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0 \\ 
0 \\ 
1 \\ 
0
 \end{array} $
 & 0.5$\pi$ \\ \hline
\end{tabular}

\caption{\textbf{Target gate set: logic gates}.  These are the \emph{ideal} (generally unitary) logic gates.  Each has a name starting with ``G'', and is represented as a $d^2\times d^2$ \emph{superoperator} that acts by matrix multiplication on vectors in $\mathcal{B}(\mathcal{H})$.  For each gate, its axis of rotation (in $\mathcal{B}(\mathcal{H})$) and angle of rotation are also given.  See Table \ref{bestGatesetGatesTable} for GST estimates of the actual logic gates implemented in this experiment.\label{targetGatesTable}}
\end{center}
\end{table}

\subsection{GST Input Data}

The most important input to GST is a \emph{dataset} -- a list of experimental counts or frequencies, each associated with a \emph{gate sequences}.  Gate sequences are also referred to as ``gate strings''.  Each gate sequence defines an experiment, in which you (1) initialize the device, (2) apply the operations specified by the gate sequence, and (3) measure and record the result (``plus'' or ``minus'').

Typically, the gate sequences that appear in the dataset are generated by the following process:
\begin{enumerate}
\item A small set of short gate sequences called \emph{germs} are chosen,
\item A small set of short \emph{fiducial sequences} are chosen so that, when applied to $\rho_0$ or $E_0$, they generate an informationally complete set of states or effects.
\item Each germ is concatenated with itself to form \emph{base sequences} of length approximately $1,2,4,8,\ldots L_{max}$.
\item Each base sequence is sandwiched between every possible pair of fiducial sequences.
\end{enumerate}
The dataset comprises all sandwiched base sequences.  A few other short sequences (e.g., those corresponding to the empty base sequence) may also appear.

\iftoggle{LsAndGermsSet}{ The fiducial sequences and germs for \emph{this} dataset are given in Table \ref{fiducialAndGermListTables}. }{ Fiducial sequence and germ information was not given for this report, and may not be applicable.}  An overview of the information contained in the file you provided for dataset ``$\mathcal{D}$'' is given in Table \ref{datasetOverviewTable}.  

This table also contains one derived quantity, the spectrum of the largest \emph{Gram matrix} that GST could extract from the data.  This is included here rather than in the analysis because it is not useful for predictive purposes, and therefore is not part of the estimate.  It serves, instead, to tell you something about the quality of the data.  More precisely, it tells you about the dimension of the state space that is explored by the fiducial sequences.  This should be $d^2$-dimensional [because the fiducials are intended to explore all of $\mathcal{B}(\mathcal{H})$], and therefore the spectrum listed in Table \ref{datasetOverviewTable} should (ideally) have exactly $d^2$ elements that are large and nonzero.  In practice, you should see $d^2$ large elements, and a rapid drop in magnitude thereafter.  If fewer than $d^2$ elements are large, then the fiducials were poorly chosen and are not exploring the state space effectively.  If more than $d^2$ are large, then the system is experiencing strong non-Markovian effects (e.g., strong coupling to environmental degrees of freedom) or it has a larger Hilbert space dimension than expected.

\iftoggle{LsAndGermsSet}{

\begin{table}[h]
\begin{center}
\begin{minipage}[b]{0.40\linewidth}
\centering
\adjustbox{max width=\linewidth}{
\begin{tabular}[l]{|c|c|c|}
\hline
 & \multicolumn{2}{c|}{Fiducials} \\ \hline
\# & Prep. & Measure \\ \hline
1 & $$ & $$ \\ \hline
2 & $\mbox{Gx}$ & $\mbox{Gx}$ \\ \hline
3 & $\mbox{Gy}$ & $\mbox{Gy}$ \\ \hline
4 & $\mbox{Gx}\cdot\mbox{Gx}$ & $\mbox{Gx}\cdot\mbox{Gx}$ \\ \hline
5 & $\mbox{Gx}\cdot\mbox{Gx}\cdot\mbox{Gx}$ & $\mbox{Gx}\cdot\mbox{Gx}\cdot\mbox{Gx}$ \\ \hline
6 & $\mbox{Gy}\cdot\mbox{Gy}\cdot\mbox{Gy}$ & $\mbox{Gy}\cdot\mbox{Gy}\cdot\mbox{Gy}$ \\ \hline
\end{tabular}

}
%putfield{prepStrListTable}{List of state prep fiducials table will be placed here}
%putfield{effectStrListTable}{List of effect fiducials table will be placed here}
\end{minipage}
\begin{minipage}[b]{0.40\linewidth}
\adjustbox{max width=\linewidth}{
\begin{tabular}[l]{|c|c|}
\hline
\# & Germ \\ \hline
1 & $\mbox{Gx}$ \\ \hline
2 & $\mbox{Gy}$ \\ \hline
3 & $\mbox{Gi}$ \\ \hline
4 & $\mbox{Gx}\cdot\mbox{Gy}$ \\ \hline
5 & $\mbox{Gx}\cdot\mbox{Gy}\cdot\mbox{Gi}$ \\ \hline
6 & $\mbox{Gx}\cdot\mbox{Gi}\cdot\mbox{Gy}$ \\ \hline
7 & $\mbox{Gx}\cdot\mbox{Gi}\cdot\mbox{Gi}$ \\ \hline
8 & $\mbox{Gy}\cdot\mbox{Gi}\cdot\mbox{Gi}$ \\ \hline
9 & $\mbox{Gx}\cdot\mbox{Gx}\cdot\mbox{Gi}\cdot\mbox{Gy}$ \\ \hline
10 & $\mbox{Gx}\cdot\mbox{Gy}\cdot\mbox{Gy}\cdot\mbox{Gi}$ \\ \hline
11 & $\mbox{Gx}\cdot\mbox{Gx}\cdot\mbox{Gy}\cdot\mbox{Gx}\cdot\mbox{Gy}\cdot\mbox{Gy}$ \\ \hline
\end{tabular}

}
\end{minipage}
\caption{\textbf{Fiducial sequences and germs.}  See discussion in text.\label{fiducialAndGermListTables}}
\end{center}
\end{table}

}{}

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|}
\hline
Quantity & Value \\ \hline
Number of strings & 817 \\ \hline
Gate labels & Gx, Gy, Gi \\ \hline
SPAM labels & plus, minus \\ \hline
Counts per string & 1000 \\ \hline
\begin{tabular}{c}Gram singular values\\ (right column gives the values\\when using the target gate set)\end{tabular} & \small$ \begin{array}{cc}
0.003 & 0 \\ 
0.02 & 0 \\ 
0.663 & 1 \\ 
0.679 & 1 \\ 
0.747 & 1 \\ 
3.007 & 3
 \end{array} $
 \\ \hline
\end{tabular}

\caption{\textbf{General dataset properties}.  See discussion in text.\label{datasetOverviewTable}}
\end{center}
\end{table}

\section{Output from GST\label{secOutput}}

The primary output of GST is an estimated gate set.  This section presents the raw estimate, and then some useful derived quantities of the estimated gates, including comparisons to the target gates.  Some of these quanties are ``gauge-dependent'', meaning they will depend on unphysical gauge degrees of freedom that are a necessary byproduct of estimating an entire gate set at once (akin to a freedom of reference frame).  After finding a best-estimate based on the (physical) data, GST optimizes within the space of all (unphysical) gauge degrees of freedom.  Typically this is done to make the estimated gates and SPAM operations look as close to the ideal target gates as possible.  But tradeoffs can and almost always must be made regarding this optimization.  Table \ref{bestGatesetGaugeParamsTable} lists the relevant parameters used during the gauge optimization step which resulted in the estimates that follow.

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|}
\hline
\textbf{Quantity} & \textbf{Value} \\ \hline
\end{tabular}

\caption{\textbf{Gauge Optimization Details}.  A list of the parameters used when performing the gauge optimization that produced the final GST results found in subsequent tables and figures.\label{bestGatesetGaugeParamsTable}}
\end{center}
\end{table}


\subsection{Raw GST estimates}

Table \ref{bestGatesetSpamTable} reports the estimated SPAM operations, and Table \ref{bestGatesetGatesTable} reports the logic gate operations.  The estimated SPAM gates ($\rho_0$ and $E_0$) are vectors in $\mathcal{B}(\mathcal{H})$, and the estimated logic gates are superoperators represented as matrices acting on $\mathcal{B}(\mathcal{H})$, all in the Pauli basis.   By taking the dot product of state preparation and measurement vectors, estimated SPAM probabilites are computed in Table \ref{bestGatesetSpamParametersTable}.
\iftoggle{confidences}{
Tables \ref{bestGatesetSpamTable} and \ref{bestGatesetGatesTable} report 95\% confidence intervals for each of the gate matrix and SPAM vector elements.  A confidence region is obtained by approximating the log-likelihood (see below) as being quadratic about its minimum, and determining the ellipsoid where this approximation equals a value, $C$, defined below.  For a given parameter (e.g. gate or SPAM vector element) $x$, a confidence interval is obtained by projecting the ellipsiodal region onto that $x$'s axis.  This computes a 1-dimensional 95\% confidence interval for the profile log-liklihood for $x$, and for this reason the value of $C$ used above is chosen such that $\mathrm{CDF}[\chi^2_1](C) = 95\%$ (that is, at the value $C$ the cumulative density function of a $\chi^2_1$ distribution reaches 95\%).  If, instead, the interval corresponding to a projection of the 95\% multi-dimensional confidence region (defined by $C$ s.t.~$\mathrm{CDF}[\chi^2_n](C) = 95\%$, where $n=31$ is the number of non-gauge gate set parameters) is desired, then the all the interval widths reported here should be multiplied by 3.42.  The resulting confidence interval is always symmetric about the estimated value, and we report the half-width of the intervals in the tables.  In table \ref{bestGatesetSpamParametersTable} and those in the following section, we specify the 95\% confidence intervals of derived quantities in using \emph{value} $\pm$ \emph{half-width} notation.  The derived-quantity confidence intervals in section \ref{derivedQtySection} are computed by finding the minimum and maximum values of the linearization of the derived quantity (e.g.~fidelity).
}{}


\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|c|c|}
\hline
Operator & Matrix & Hilbert-Schmidt vector (Pauli basis) & 95\% C.I. $\nicefrac{1}{2}$-width \\ \hline
$\rho_{0}$ & $ \left(\!\!\begin{array}{cc}
0.951 & 0.001e^{-i0.16} \\ 
0.001e^{i0.16} & 0.049
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0.707 \\ 
0.002 \\ 
3\e{-4} \\ 
0.638
 \end{array} $
 & $ \begin{array}{c}
0 \\ 
0.006 \\ 
0.006 \\ 
0.004
 \end{array} $
 \\ \hline
$E_{0}$ & $ \left(\!\!\begin{array}{cc}
0.047 & 0.002e^{i1.62} \\ 
0.002e^{-i1.62} & 0.955
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0.708 \\ 
-1\e{-4} \\ 
-0.003 \\ 
-0.642
 \end{array} $
 & $ \begin{array}{c}
0.001 \\ 
0.005 \\ 
0.006 \\ 
0.005
 \end{array} $
 \\ \hline
\end{tabular}

\caption{\textbf{The GST estimate of the SPAM operations}.  Compare to Table \ref{targetSpamTable}.\label{bestGatesetSpamTable}}
\end{center}
\end{table}

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|c|}
\hline
 & $E_{0}$ & $E_C$ \\ \hline
$\rho_{0}$ & $ \begin{array}{c} 0.091 \\ \pm 0.006 \end{array} $ & $ \begin{array}{c} 0.909 \\ \pm 0.006 \end{array} $ \\ \hline
\end{tabular}

\caption{\textbf{GST estimate of SPAM probabilities}.  Computed by taking the dot products of vectors in Table \ref{bestGatesetSpamTable}.  The symbol $E_C$, when it appears, refers to the ``complement'' effect given by subtracting each of the other effects from the identity.\label{bestGatesetSpamParametersTable}}
\end{center}
\end{table}

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|c|}
\hline
Gate & Superoperator (Pauli basis) & 95\% C.I. $\nicefrac{1}{2}$-width \\ \hline
Gi & $ \left(\!\!\begin{array}{cccc}
1 & 0 & 0 & 0 \\ 
-0.001 & 0.945 & -0.006 & -0.009 \\ 
4\e{-4} & 0.001 & 0.947 & 0.004 \\ 
0.001 & 2\e{-5} & 0.001 & 0.949
 \end{array}\!\!\right) $
 & $ \left(\!\!\begin{array}{cccc}
0 & 0 & 0 & 0 \\ 
0.002 & 0.006 & 0.006 & 0.006 \\ 
0.002 & 0.006 & 0.006 & 0.006 \\ 
0.002 & 0.006 & 0.006 & 0.005
 \end{array}\!\!\right) $
 \\ \hline
Gx & $ \left(\!\!\begin{array}{cccc}
1 & 0 & 0 & 0 \\ 
3\e{-4} & 0.951 & -0.005 & -0.005 \\ 
2\e{-4} & 2\e{-4} & 9\e{-5} & -0.949 \\ 
2\e{-4} & 3\e{-4} & 0.949 & -0.002
 \end{array}\!\!\right) $
 & $ \left(\!\!\begin{array}{cccc}
0 & 0 & 0 & 0 \\ 
0.002 & 0.006 & 0.006 & 0.006 \\ 
0.001 & 0.005 & 0.005 & 0.003 \\ 
0.001 & 0.005 & 0.003 & 0.004
 \end{array}\!\!\right) $
 \\ \hline
Gy & $ \left(\!\!\begin{array}{cccc}
1 & 0 & 0 & 0 \\ 
2\e{-4} & 8\e{-5} & -0.004 & 0.949 \\ 
3\e{-4} & 0.001 & 0.951 & 0.001 \\ 
-2\e{-4} & -0.949 & -0.006 & -0.001
 \end{array}\!\!\right) $
 & $ \left(\!\!\begin{array}{cccc}
0 & 0 & 0 & 0 \\ 
0.001 & 0.006 & 0.006 & 0.003 \\ 
0.002 & 0.005 & 0.006 & 0.005 \\ 
0.001 & 0.003 & 0.005 & 0.004
 \end{array}\!\!\right) $
 \\ \hline
\end{tabular}

\caption{\textbf{The GST estimate of the logic gate operations}.  Compare to Table \ref{targetGatesTable}.\label{bestGatesetGatesTable}}
\end{center}
\end{table}

The estimated gates can be compared directly to the target gate set given in Section \ref{secInput}.  Ideally, they would match.  In practice, of course, they won't.  One of the best ways we have found to evaluate the significance of discrepancies is to compare \emph{derived} quantities -- i.e., certain properties calculated from the gate matrices and SPAM vectors.  Deriving quantities from these raw outputs occupies the remainder of this section.
\clearpage

\subsection{Derived quantities\label{derivedQtySection}}

Generally, the first thing that you want to know is ``How far from ideal are the gates?''.  To answer this, the report tabulates several well-known definitions of distance.  Table \ref{bestGatesetVsTargetTable} lists the discrepancy from each estimated gate to its corresponding target, as measured by:
\begin{enumerate}
\item \textbf{Process infidelity}.  Infidelity is simply $1-F$, where $F$ is the \emph{fidelity}.  The process fidelity between quantum processes $G_a$ and $G_b$ is given by $F = \Tr\left( \sqrt{ \sqrt{\chi_a} \chi_b \sqrt{\chi_a} } \right)^2$, where $\chi_a$ and $\chi_b$ are the Jamiolkowski states (normalized Choi process matrices) corresponding to gate matrices $G_a$ and $G_b$ respectively.  If the target is unitary (as is often the case), $F = \Tr\left( \chi_a \chi_b \right)$.  Process infidelity is roughly what is measured in randomized benchmarking protocols; it quantifies the \emph{incoherent} error rate if coherent errors (e.g. over-rotations) are not allowed to accumulate.
\item \textbf{Trace distance}.  This is the \emph{Jamiolkowski trace distance} between the Jamiolkowski states corresponding to the two processes:  $d_{tr} = \vert\chi_a - \chi_b\vert_1 = \Tr\left(\sqrt{(\chi_a-\chi_b)^2}\right)$.  This distance is useful primarily as a proxy for the \emph{diamond norm distance}, because $d_{tr} \leq d_{\diamond} \leq \mathrm{dim}(\mathcal{H}) d_{tr}$.
\item \textbf{Diamond Norm}.  The diamond norm between two quantum processes $G_a$ and $G_b$ is given by $\norm{G_a - G_b}_\Diamond = \sup_\rho \norm{(G_a \otimes I_k)(\rho) - (G_b \otimes I_k)(\rho)}_1$, where $I_k$ is the $k$-dimensional identity operation, $\norm{\cdot}_1$ denotes the trace norm, and the supremum is taken over all $k \ge 1$ and density matrices $\rho$ of dimension $nk$, with $n$ the dimension of $G_a$ and $G_b$.  The diamond norm is also called the \emph{completely bounded trace norm}, and plays the analogous role for quantum process distinguishability that the trace norm plays for density matrices.  Specifically, the optimal probability of distinguishing $G_a$ from $G_b$ after a \emph{single evaluation} is given by $\frac{1}{2} + \frac{1}{4}\norm{G_a - G_b}_\Diamond$.  The diamond norm distance is an upper bound on the rate of error under any possible circumstance (including coherent accumulation of errors) and is often used in proofs of fault tolerance.  For gates dominated by coherent/unitary error, it is common to see $d_{\diamond} \approx \sqrt{1-F}$.  For gates dominated by incoherent error, $d_{\diamond} \approx 1-F$.
\item \textbf{Frobenius-norm distance}.  The Frobenius norm distance between two gates $G_a$ and $G_b$ is simply $d_F = \sqrt{\Tr\left[\left(G_a-G_b\right)^2\right]}$.  It has no known \emph{operational} interpretation, but is very convenient as a rough measure of inaccuracy.  It is also equal to the sum of the RMS errors in the individual matrix elements of the gates.
\end{enumerate}

It's also useful to know \emph{how} the real gates (or, more precisely, GST's estimates of the real gates) differ from the targets.  There are several ways we could represent this, but the most useful involves an \emph{error generator}.  These are also given in Table \ref{bestGatesetVsTargetTable}.  The final column of the table lists, for each gate, a Lindbladian superoperator $\mathbb{L}$.  It is defined by the equation $\hat{G} = G_{\mathrm{target}}e^{\mathbb{L}}$, where $\hat{G}$ is the estimate and $G_{\mathrm{target}}$ is the ideal gate.  This Lindbladian would be zero if the gates were perfect, and its overall magnitude is approximately equal to the diamond distance (or Jamiolkowski trace distance) between the target gate and the estimate.

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|c|c|}
\hline
Gate & \begin{tabular}{c}Process\\Infidelity\end{tabular} & \begin{tabular}{c}$\nicefrac{1}{2}$ Trace\\Distance\end{tabular} & $\nicefrac{1}{2}$ $\Diamond$-Norm \\ \hline
Gi & $ \begin{array}{c} 0.04 \\ \pm 0.003 \end{array} $ & $ \begin{array}{c} 0.04 \\ \pm 0.003 \end{array} $ & $ \begin{array}{c} 0.04 \\ \pm 0.003 \end{array} $ \\ \hline
Gx & $ \begin{array}{c} 0.038 \\ \pm 0.002 \end{array} $ & $ \begin{array}{c} 0.038 \\ \pm 0.002 \end{array} $ & $ \begin{array}{c} 0.038 \\ \pm 0.002 \end{array} $ \\ \hline
Gy & $ \begin{array}{c} 0.038 \\ \pm 0.002 \end{array} $ & $ \begin{array}{c} 0.038 \\ \pm 0.002 \end{array} $ & $ \begin{array}{c} 0.038 \\ \pm 0.002 \end{array} $ \\ \hline
\end{tabular}

\vspace{2em}
\begin{tabular}[l]{|c|c|}
\hline
Gate & Error Generator \\ \hline
Gi & $ \left(\!\!\begin{array}{cccc}
0 & 0 & 0 & 0 \\ 
-0.001 & -0.057 & -0.007 & -0.009 \\ 
4\e{-4} & 0.001 & -0.054 & 0.004 \\ 
0.001 & 3\e{-5} & 0.001 & -0.053
 \end{array}\!\!\right) $
 \\ \hline
Gx & $ \left(\!\!\begin{array}{cccc}
0 & 0 & 0 & 0 \\ 
3\e{-4} & -0.051 & -0.005 & -0.006 \\ 
2\e{-4} & 3\e{-4} & -0.052 & -0.002 \\ 
-2\e{-4} & -3\e{-4} & -1\e{-4} & -0.052
 \end{array}\!\!\right) $
 \\ \hline
Gy & $ \left(\!\!\begin{array}{cccc}
0 & 0 & 0 & 0 \\ 
2\e{-4} & -0.052 & 0.007 & 0.001 \\ 
3\e{-4} & 0.001 & -0.05 & 0.001 \\ 
3\e{-4} & 9\e{-5} & -0.004 & -0.053
 \end{array}\!\!\right) $
 \\ \hline
\end{tabular}

\caption{\textbf{Comparison of GST estimated gates to target gates}.  This table presents, for each of the gates, three different measures of distance or discrepancy from the GST estimate to the ideal target operation.  See text for more detail.  The second table lists the ``Error Generator'' for each gate, which is the Lindbladian $\mathbb{L}$ that describes \emph{how} the gate is failing to match the target.  This error generator is defined by the equation $\hat{G} = G_{\mathrm{target}}e^{\mathbb{L}}$. \label{bestGatesetVsTargetTable}}
\end{center}
\end{table}

It's usually useful to understand \emph{how} gates fail.  The error generators in Table \ref{bestGatesetVsTargetTable} provide one view on this, but they are not necessarily intuitive.   For example, you might want to know whether your gate suffers depolarizing, dephasing, or over-rotation errors.  In Table \ref{bestGatesetDecompTable}, the estimated gates are decomposed into: (1) rotations (including angle and axis errors); (2) incoherent \emph{diagonal} decay rates (depolarizing or $T_1$ noise); and (3) incoherent \emph{off-diagonal} decay rates (dephasing or $T_2$ noise).  These analyses can be compared with a the similar decomposition of the target gates (cf. table \ref{targetGatesTable}).  Note that for some erroneous gates, this decomposition simply fails; if the numbers make no sense, this is probably the case.

\begin{table}[h]
\small
\begin{center}
\begin{tabular}[l]{|c|c|c|c|c|c|}
\hline
Gate & Eigenvalues & Fixed pt & Rotn. axis & Diag. decay & Off-diag. decay \\ \hline
Gi & $ \begin{array}{c}
0.949 \\ 
0.946e^{i0.00} \\ 
0.946e^{-i0.00} \\ 
1
 \end{array} $
 $\pm$ $ \begin{array}{c}
0.019 \\ 
0.013e^{i0.68} \\ 
0.013e^{i0.68} \\ 
0
 \end{array} $
 & $ \begin{array}{c}
0.999 \\ 
-0.023 \\ 
0.009 \\ 
0.021
 \end{array} $
 & $ \begin{array}{c}
0 \\ 
0.933 \\ 
-0.151 \\ 
-0.326
 \end{array} $
 & $ \begin{array}{c} 0.051 \\ \pm 0.019 \end{array} $ & $ \begin{array}{c} 0.054 \\ \pm 0.01 \end{array} $ \\ \hline
Gx & $ \begin{array}{c}
0.949e^{i1.57} \\ 
0.949e^{-i1.57} \\ 
0.951 \\ 
1
 \end{array} $
 $\pm$ $ \begin{array}{c}
0.005e^{i0.67} \\ 
0.005e^{i0.67} \\ 
0.006 \\ 
0
 \end{array} $
 & $ \begin{array}{c}
1 \\ 
0.007 \\ 
-2\e{-5} \\ 
2\e{-4}
 \end{array} $
 & $ \begin{array}{c}
0 \\ 
-1 \\ 
3\e{-6} \\ 
-3\e{-4}
 \end{array} $
 & $ \begin{array}{c} 0.049 \\ \pm 0.006 \end{array} $ & $ \begin{array}{c} 0.051 \\ \pm 0.003 \end{array} $ \\ \hline
Gy & $ \begin{array}{c}
0.949e^{i1.57} \\ 
0.949e^{-i1.57} \\ 
0.951 \\ 
1
 \end{array} $
 $\pm$ $ \begin{array}{c}
0.005e^{i0.73} \\ 
0.005e^{i0.73} \\ 
0.006 \\ 
0
 \end{array} $
 & $ \begin{array}{c}
1 \\ 
-2\e{-7} \\ 
0.006 \\ 
-2\e{-4}
 \end{array} $
 & $ \begin{array}{c}
0 \\ 
-0.005 \\ 
1 \\ 
-0.001
 \end{array} $
 & $ \begin{array}{c} 0.049 \\ \pm 0.006 \end{array} $ & $ \begin{array}{c} 0.051 \\ \pm 0.003 \end{array} $ \\ \hline
\end{tabular}

\vspace{2em}
\begin{tabular}[l]{|c|c|c|c|c|}
\hline
\multirow{2}{*}{Gate} & \multirow{2}{*}{Angle} & \multicolumn{3}{c|}{Angle between Rotation Axes} \\ \cline{3-5}
 & & Gi & Gx & Gy \\ \hline
Gi & $ \begin{array}{c}(0.001 \\ \pm 0.003)\pi \end{array} $ &  & $ \begin{array}{c}(0.883 \\ \pm 0.481)\pi \end{array} $ & $ \begin{array}{c}(0.55 \\ \pm 0.619)\pi \end{array} $ \\ \hline
Gx & $ \begin{array}{c}(0.5 \\ \pm 0.001)\pi \end{array} $ & $ \begin{array}{c}(0.883 \\ \pm 0.481)\pi \end{array} $ &  & $ \begin{array}{c}(0.498 \\ \pm 0.002)\pi \end{array} $ \\ \hline
Gy & $ \begin{array}{c}(0.5 \\ \pm 0.001)\pi \end{array} $ & $ \begin{array}{c}(0.55 \\ \pm 0.619)\pi \end{array} $ & $ \begin{array}{c}(0.498 \\ \pm 0.002)\pi \end{array} $ &  \\ \hline
\end{tabular}

\caption{\textbf{Eigen-decomposition of estimated gates}.  Each estimated gate is described in terms of: (1) the eigenvalues of the superoperator; (2) the gate's fixed point (as a vector in $\mathcal{B}(\mathcal{H})$, in the Pauli basis); (3)  the axis around which it rotates, as a vector in $\mathcal{B}(\mathcal{H})$; (4) the angle of the rotation that it applies; (5) the decay rate along the axis of rotation (``diagonal decay''); (6) the decay rate perpendicular to the axis of rotation (``off-diagonal decay''); and (7) the angle between each gate's rotation axis and the rotation axes of the other gates.  ``X'' indicates that the decomposition failed or couldn't be interpreted. \label{bestGatesetDecompTable}}
\end{center}
\end{table}

%It might be useful to know the closest \emph{unitary} operation to the estimated gate, and how close it is.  Usually, you were trying to implement a unitary.  If the closest unitary to $G$ was indeed $G_{\mathrm{target}}$, then all errors are incoherent; if not, you might be able to tweak the gate parameters to get closer relatively easily.  Also, implementing a particular unitary may be less important than just achieving \emph{some} set of mutually independent unitaries.  In these and other cases, the distance from an estimated gate to its closest unitary approximation is of interest.

%Table \ref{bestGatesetClosestUnitaryTable} lists, for each estimated gate, the properties of its closest unitary approximation.  The table defines the closest unitary, in terms of an axis and angle (in $\mathcal{B}(\mathcal{H})$) of rotation.  It also presents the process fidelity and Jamiolkowski trace distance between the estimated gate and its closest unitary approximation.  A sanity check is computed by comparing the fidelity of the obtained closest unitary with a theoretical upper bound (if a value greater than one appears in this column then the other values in that row may be inaccurate).  If these numbers are similar to those in Table \ref{bestGatesetVsTargetTable}, then the gates are as close to the targets as they are to \emph{any} unitary.

%\begin{table}[h]
%\begin{center}
%\XXXputfield{bestGatesetClosestUnitaryTable}{Best gate set overview table will be placed here}
%\caption{\XXXputfield{tt_bestGatesetClosestUnitaryTable}{}Information pertaining to the closest unitary gate to each of the estimated gates.\label{bestGatesetClosestUnitaryTable}}
%\end{center}
%\end{table}


Finally, Table \ref{bestGatesetChoiTable} presents each estimated gate's \emph{Choi matrix}, along with its spectrum.  The Choi matrix (sometimes ambiguously referred to as the ``process matrix'') is an alternative way to describe a process.  We usually prefer the ``superoperator representation'', which has the very useful property that the process matrix corresponding to applying $G_a$ and then $G_b$ is simply $G_bG_a$.  This is completely false for the Choi representation.  Nonetheless, the Choi representation is often useful, so we present it here -- but without a detailed discussion of its properties (see, e.g. the textbook by Nielsen and Chuang).

The Choi matrix $\chi(G)$ for a gate $G$ can be simply understood in either of two ways.  First, it is equivalent (up to choice of basis) to the \emph{Jamiolkowski state} defined by applying $G$ to one half of a maximally entangled bipartite state.  Second, it is the general (non-diagonal) form of the well-known Kraus representation, $G[\rho] = \sum_i{K_i\rho K_i^\dagger}$.  The Choi matrix behaves in many ways like a quantum state, and appears naturally in expressions for the process fidelity and Jamiolkowsi trace distance just as density matrices would enter these expressions when computing differences between states.  

Additionally, the condition of \emph{complete positivity} or CP (which all real quantum processes must satisfy) is simply the positivity of the Choi matrix.  Thus, negative eigenvalues in Table \ref{bestGatesetChoiTable} indicate that the estimate violates complete positivity.  If they are very small, they may simply indicate statistical fluctuations (unitary gates have $\chi$ matrices with zero eigenvalues, so any small fluctuation is likely to violate CP).  If they are large, they serve as a warning that (1) the model of CPTP maps is probably violated (usually because of non-Markovian behavior), and (2) this estimate may produce negative or greater-than-unity probabilities.  GST does \emph{not} generally impose complete positivity (although it is an option), precisely because violation of CP is a warning flag for non-Markovian behavior (which is very common in experimental qubits).

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|c|}
\hline
Gate & Choi matrix (Pauli basis) & Eigenvalues \\ \hline
Gi & $ \left(\!\!\begin{array}{cccc}
0.96 & 0.001e^{-i1.88} & 0.002e^{-i1.52} & 0.002e^{i1.43} \\ 
0.001e^{i1.88} & 0.012 & 0.001e^{-i2.94} & 0.002e^{i3.09} \\ 
0.002e^{i1.52} & 0.001e^{i2.94} & 0.013 & 0.001e^{i0.22} \\ 
0.002e^{-i1.43} & 0.002e^{-i3.09} & 0.001e^{-i0.22} & 0.014
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0.011 \\ 
0.013 \\ 
0.017 \\ 
0.96
 \end{array} $
 $\pm$ $ \begin{array}{c}
0.002 \\ 
0.002 \\ 
0.002 \\ 
0.003
 \end{array} $
 \\ \hline
Gx & $ \left(\!\!\begin{array}{cccc}
0.487 & 0.475e^{i1.57} & 0.001e^{-i1.54} & 0.001e^{i1.53} \\ 
0.475e^{-i1.57} & 0.488 & 0.001e^{-i3.10} & 0.001e^{i3.11} \\ 
0.001e^{i1.54} & 0.001e^{i3.10} & 0.013 & 9\e{-5}e^{-i1.91} \\ 
0.001e^{-i1.53} & 0.001e^{-i3.11} & 9\e{-5}e^{i1.91} & 0.012
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0.011 \\ 
0.013 \\ 
0.014 \\ 
0.962
 \end{array} $
 $\pm$ $ \begin{array}{c}
0.002 \\ 
0.002 \\ 
0.002 \\ 
0.002
 \end{array} $
 \\ \hline
Gy & $ \left(\!\!\begin{array}{cccc}
0.488 & 0.002e^{-i1.54} & 0.474e^{i1.57} & 0.001e^{i1.61} \\ 
0.002e^{i1.54} & 0.012 & 0.001e^{i3.08} & 9\e{-5}e^{i2.13} \\ 
0.474e^{-i1.57} & 0.001e^{-i3.08} & 0.488 & 0.001e^{-i3.09} \\ 
0.001e^{-i1.61} & 9\e{-5}e^{-i2.13} & 0.001e^{i3.09} & 0.012
 \end{array}\!\!\right) $
 & $ \begin{array}{c}
0.011 \\ 
0.012 \\ 
0.015 \\ 
0.962
 \end{array} $
 $\pm$ $ \begin{array}{c}
0.002 \\ 
0.002 \\ 
0.002 \\ 
0.002
 \end{array} $
 \\ \hline
\end{tabular}

\caption{\textbf{Choi matrix representation of the GST estimated gate set}.  This table lists Choi representations of the estimated gates, and their eigenvalues.  Unitary gates have a spectrum $(1,0,0\ldots)$, just like pure quantum states.  Negative eigenvalues are non-physical, and may represent either statistical fluctuations or violations of the CPTP model used by GST.\label{bestGatesetChoiTable}}
\end{center}
\end{table}



\section{Goodness-of-model Analysis\label{secGoodness}}

The previous section presented the estimated gate set, and compared it to the target gate set.  This section is concerned with a mostly orthogonal analysis which seeks to explain how much the estimated gate set can be trusted -- i.e., how well it fits the data.

To understand the goal of this section, consider the simple problem of fitting a line to a set of points.  For any set of points, there is \emph{always} a best-fit line -- but this doesn't mean that the best-fit line is a \emph{good} fit!  The data points may trace out a parabola, a square, or even something more complicated.  It is essential to understand not just what the best-fit line was (and perhaps how close it was to some desired line), but also \textbf{how well that linear model was able to fit all the data}.  Of course, we do not expect it to fit every data point perfectly.  The critical question is ``Did the linear model fit \emph{as well as we would expect it to} if the data really were generated by a linear process?''

In this analogy, GST's estimated gate set is like the best-fit line, and the target gate set like the desired line.  This section asks the question ``How well was GST able to fit all of the data -- and did it fit well enough to suggest that its model is valid?''. A central tool used to do this is the \emph{likelihood function}, which we denote $\mathcal{L}$, which formally is the probability of the observed data given a set of model parameters.  The basic idea is that we maximize the likelihood function to obtain the best set of model parameters (i.e.~gate set), and by looking at the value of this maximum we can determine the model's goodness-of-fit.  We will actually deal primarily with the logarithm of the likelihood function, $\log(\mathcal{L})$, which is simliarly maximized.

\subsection{Aggregated $\log(\mathcal{L})$}

The log-likelihood for an $n$-outcome system with predicted probabilities $p_i$ and observed frequencies $f_i$ ($i=1\ldots n$) is given by:
\begin{equation}
\log(\mathcal{L}) = \sum_i N f_i \log(p_i).
\end{equation}
where $N$ is the total number of counts. In \emph{this} analysis, $\log(\mathcal{L})$ is used to compare the set of probabilities predicted by a gate set ($p_s$) and the frequencies obtained from a dataset ($f_s$).  Each experiment (or gate sequence) $s$ is associated to two probabilities:  ``plus'' has probability $p_s$ and ``minus'' has probability $1-p_s$.  The $\log(\mathcal{L})$ contribution of a single gate string $s$ is
\begin{equation}
\log(\mathcal{L})_s = N f_s \log(p_s) + N (1-f_s) \log(1-p_s),\label{eqGateStringLogL}
\end{equation}
where $N$ is the number of times the experiment $s$ was performed, $p_s$ is the probability of a ``plus'' outcome as predicted by the gate set, and $f_s$ is the observed frequency of ``plus''.  The total log-likelihood for an entire dataset is just the sum 
\begin{equation}
\log(\mathcal{L}) = \sum_{s\in\mathcal{S}}{ \log(\mathcal{L})_s}.\label{eqDatasetLogL}
\end{equation}
A theoretical upper bound on the log-likelihood can be found by replacing $p_s$ with $f_s$ in Eq.~\ref{eqGateStringLogL} and evaluating Eq.~\ref{eqDatasetLogL}.  We will refer to this quantity as $\log(\mathcal{L})_{ub}$.

Statistical theory has quite a lot to say about the likelihood function (see any of the major textbooks).  Using some of these results, we can predict that if there are $N_p$ free parameters in the gate set that GST is fitting, and GST fits a dataset containing $N_s > N_p$ distinct experiments (gate sequences), then \emph{if the gate set model is correct}, then two times the difference between $\log(\mathcal{L})_{ub}$ and the maximum $\log(\mathcal{L})$ obtained is a random variable with a $\chi^2_{k}$ distribution, where 
$$k \equiv N_s - N_p.$$
Its expected value is $\expec{\chi^2}=k$, and its standard deviation is $\sqrt{2k}$.  Thus, if the fit is ``good'', then twice $\Delta\log(\mathcal{L}) \equiv \log(\mathcal{L})_{ub} - \max(\log(\mathcal{L}))$ should lie roughly within the interval $[k-\sqrt{2k},k+\sqrt{2k}]$.
Thus, by comparing the difference $2\Delta\log(\mathcal{L}) - k$ to $\sqrt{2k}$, one can determine how well the GST estimate was able to fit the data in dataset ``$\mathcal{D}$''.

\iftoggle{LsAndGermsSet}{
The MLEGST algorithm used to generate this estimate is iterative.  It starts by fitting only data from the shortest gate sequences (which are easy to fit \emph{and} insensitive to most non-Markovian noise), then successively adds longer and longer sequences (with base sequence length $L\leq 1,2,4,8,\ldots$) to the mix.  Since we get an estimate at each intermediate $L$, it is possible to quantify not just the goodness of the \emph{best} fit (presented in the previous section), but how the goodness-of-fit behaves as longer and longer sequences are added in.

This data is presented in Table \ref{progressTable}.  What you should be looking for here is whether -- at each value of $L$ -- the $2\Delta\log(\mathcal{L})$ quantity is roughly the same as $k$.  More precisely, is $|2\Delta\log(\mathcal{L})-k|$ less than or equal to $\sqrt{2k}$?  If not, then the model is not fitting as well as it should, which usually indicates non-Markovian noise (or, rarely, that the GST algorithm has simply failed to find a good fit even though one exists).  

As a rough rule of thumb, for GST experiments involving relatively long sequences (e.g. $L\geq100$):
%\begin{tabular}{rp{5.5in}}
%$\bigstar\bigstar\bigstar\bigstar\bigstar$ & ``Incredibly good'' experiments have $\chi^2 \approx k$, as predicted by theory (and seen in simulations). \\
%$\bigstar\bigstar\bigstar\bigstar$ & ``Great'' experiments have $\chi^2 \leq 2k$ or so. \\
%$\bigstar\bigstar\bigstar$ & ``Good'' experiments have $\chi^2 \leq 5k$ or so. \\
%$\bigstar\bigstar$ & ``Okay'' experiments have $\chi^2 \leq 10k$. \\
%$\bigstar$ & Experiments in which $\chi^2 > 10k$ have very significant non-Markovian noise, and the results in the previous section should be viewed very cautiously.
%\end{tabular}
\begin{itemize}
\item ``Incredibly good'' ($\bigstar\bigstar\bigstar\bigstar\bigstar$) experiments have $2\Delta\log(\mathcal{L}) \approx k$, as predicted by theory (and seen in simulations).
\item ``Great'' ($\bigstar\bigstar\bigstar\bigstar$) experiments have $2\Delta\log(\mathcal{L}) \leq 2k$ or so.
\item ``Good'' ($\bigstar\bigstar\bigstar$) experiments have $2\Delta\log(\mathcal{L}) \leq 5k$ or so.
\item ``Okay'' ($\bigstar\bigstar$) experiments have $2\Delta\log(\mathcal{L}) \leq 10k$.
\item Experiments in which $2\Delta\log(\mathcal{L}) > 10k$ ($\bigstar$) have very significant non-Markovian noise, and the results in the previous section should be viewed very cautiously.
\end{itemize}

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|c|c|c|c|c|c|c|c|}
\hline
L & $2\Delta\log(\mathcal{L})$ & $k$ & $2\Delta\log(\mathcal{L})-k$ & $\sqrt{2k}$ & $N_\sigma$ & $N_s$ & $N_p$ & Rating \\ \hline
0 & 53.75 & 61 & -7.251 & 11.05 & -0.66 & 92 & 31 & $\bigstar\bigstar\bigstar\bigstar\bigstar$ \\ \hline
1 & 53.75 & 61 & -7.251 & 11.05 & -0.66 & 92 & 31 & $\bigstar\bigstar\bigstar\bigstar\bigstar$ \\ \hline
2 & 106.8 & 137 & -30.25 & 16.55 & -1.83 & 168 & 31 & $\bigstar\bigstar\bigstar\bigstar\bigstar$ \\ \hline
4 & 439.5 & 410 & 29.48 & 28.64 & 1.03 & 441 & 31 & $\bigstar\bigstar\bigstar\bigstar$ \\ \hline
8 & 691.3 & 786 & -94.72 & 39.65 & -2.39 & 817 & 31 & $\bigstar\bigstar\bigstar\bigstar\bigstar$ \\ \hline
\end{tabular}

\caption{\textbf{Comparison between the computed and expected maximum $\log(\mathcal{L})$ for different values of $L$}.  $N_S$ and $N_p$ are the number of gate strings and parameters, respectively.  The quantity $2\Delta\log(\mathcal{L})$ measures the goodness of fit of the GST model (small is better) and is expected to lie within $[k-\sqrt{2k},k+\sqrt{2k}]$ where $k = N_s-N_p$. $N_\sigma = (2\Delta\log(\mathcal{L})-k)/\sqrt{2k}$ is the number of standard deviations from the mean (a $p$-value can be straightforwardly derived from $N_\sigma$).  The rating from 1 to 5 stars gives a very crude indication of goodness of fit as explained in the text.\label{progressTable}}
%$p$ is the p-value derived from a $\chi^2_k$ distribution.(For example, if $p=0.05$, then the probability of observing a $\chi^{2}$ value as large as, or larger than, the one indicated in the table is 5\%, assuming the GST model is valid.)
\end{center}
\end{table}

}{
\textbf{NOTE: Not enough information was given at the time of this report generation to know provide a goodness-of-model analysis}
}

\FloatBarrier

\iftoggle{LsAndGermsSet}{
\subsection{Detailed likelihood analysis}

The aggregated $2\Delta\log(\mathcal{L})$ numbers presented in Table \ref{progressTable} tell you how well the GST estimate fits the \emph{entire} dataset.  If they are in line with theory ($2\Delta\log(\mathcal{L}) \approx k$), then there is little more to be said.  But if the best fit to the data is not good, we can debug it by identifying \emph{which} experiments are inconsistent with the fit.

Figure \ref{bestEstimateColorBoxPlot} displays the $2\Delta\log(\mathcal{L})$ contribution from each individual gate sequence (Eq.~\ref{eqGateStringLogL}).  Each gate sequence corresponds to a single colored ``pixel'' in the plot.  Each block of pixels corresponds to a single base sequence (i.e., a germ power), and the individual pixels within a block correspond to the various fiducial sequence pairs between which that base sequence was sandwiched, as indicated in Figure \ref{colorBoxPlotKeyPlot}.  Base sequences are arranged in a grid; different rows correspond to different germs, while different columns correspond to different maximum lengths $L$.  Pixels are labeled with the $2\Delta\log(\mathcal{L})$ contribution for that sequence, and colored appropriately. 

Sequences whose observed frequencies are consistent with a Markovian gate set are shown in gray, with darker shades indicating greater inconsistency with the estimated gate set.  Data shown in red are \emph{not} consistent with a Markovian gate set.  It may appear contradictory to say that (a) gray is ``consistent" with Markovian, but (b) darker shades indicate ``greater inconsistency".  The resolution is that the $\chi^2$ values quantify inconsistency with the model, \emph{but} they themselves are also subject to random fluctuations.  Therefore, even if the data are perfectly consistent with the model, we expect to see (for example) a single $\chi^2_s \geq 10$ once per each 638 experiments.  Observing $\chi^2_s \geq 10$ for any given sequence does suggest that the data from
$s$ were relatively surprising, but we also expect to see one such fluctuation if there are more than about 600 experiments.  The gray/red threshold is chosen based on the total number of sequences so that \emph{if} the data are perfectly Markovian, then the probability of one or more experiments being colored red is only 5\%.

Identifying patterns and trends within such ``pixel plots'' can aid in identifying specific sources and types of non-Markovian noise which may be to blame if the GST algorithms are unable to produce a ``good'' estimate.  For example, it is often the case that all the short sequences [$L = O(1)$] can be fit reasonably well, but the right-hand side of Figure \ref{bestEstimateColorBoxPlot} becomes a sea of red.  This indicates that non-Markovian behavior (potentially due to slow drift of gate set parameters) is becoming more significant for longer experiments.  In other cases, a single row may be particularly bad, indicating that a particular gate or germ is especially problematic (e.g., was not stabilized using dynamical decoupling techniques).  Be cautious in debugging, however -- sometimes bad $\log(\mathcal{L})$ values for a particular gate or germ can result \emph{not} from faults in that operation, but because another operation failed so badly that it distorted the entire fit (e.g., in trying to fit catastrophically non-Markovian data at Point A, GST ended up failing to fit perfectly good data at Point B).

Similar pixel plots for the intermediate estimates whose total $2\Delta\log(\mathcal{L})$ is listed in Table \ref{progressTable} \iftoggle{pixelplotsappendix}{can be found in Appendix \ref{appendix_logL_pixelplots}.}{are included when the ``pixel plots'' appendix is enabled.}

\begin{figure}
\begin{center}
\includegraphics[width=1.80in,height=1.80in,keepaspectratio]{full_reportD_files/colorBoxPlotKey.png}
\caption{\textbf{Sub-block key for subsequent plots.} Shows how elements of the sub-blocks in Figure \ref{bestEstimateColorBoxPlot} correspond to preparation and measurement fiducial sequences.  Note that the column indicates the fiducial adjacent to state preparation, while the row indicates the fiducial adjacent to measurement.\label{colorBoxPlotKeyPlot}}
\end{center}
\end{figure}

\begin{figure}
\begin{center}
\includegraphics[width=6.50in,height=8.00in,keepaspectratio]{full_reportD_files/bestLogLBoxes.pdf}
\caption{\textbf{$2\Delta\log(\mathcal{L})$ contributions for every individual experiment in the dataset}.   Each pixel represents a single experiment (gate sequence), and its color indicates whether GST was able to fit the corresponding frequency well.  Shades of white/gray are typical. Red squares represent statistically significant evidence for model violation (non-Markovianity), and should appear with probability at most 5\% if the data really are Markovian. Square blocks of pixels correspond to base sequences (arranged vertically by germ and horizontally by length); each pixel within a block corresponds to a specific choice of pre- and post-fiducial sequences.  See text for further details.\label{bestEstimateColorBoxPlot}}
\end{center}
\end{figure}

Figure \ref{invertedBestEstimateColorBoxPlot} shows exactly the same $\log(\mathcal{L})$ analysis as Figure \ref{bestEstimateColorBoxPlot}, but arranged differently.  Here, blocks (not square) all correspond to a single fiducial pair (e.g., pre- and post-fiducial), and pixels within a block correspond to different base sequences.  This can be useful for diagnosing a single bad fiducial sequence.

\begin{figure}
\begin{center}
\includegraphics[width=6.50in,height=8.00in,keepaspectratio]{full_reportD_files/bestLogLBoxes_inverted.pdf}
\caption{\textbf{$2\Delta\log(\mathcal{L})$ contributions for each experiment, arranged differently.}  This figure shows the same data as Figure \ref{bestEstimateColorBoxPlot}, but arranged differently.  Each block now corresponds to a particular pair of fiducial sequences, while pixels within the block correspond to different base sequences sandwiched between those fiducials.\label{invertedBestEstimateColorBoxPlot}}
\end{center}
\end{figure}

}{}

\section{System and pyGSTi parameters\label{metadata}}
This section contains a raw dump of system information and various pyGSTi parameters.  It's purpose is to stamp this report with parameters indicating how exactly GST was run to create it, as well as to record the software environment in within which the report creation was run.  Note that if the core GST computation was done on a machine different from the one that created this report, the software information contained here will be of less value.

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|p{3in}|}
\hline
\textbf{Quantity} & \textbf{Value} \\ \hline
Long tables (option) & \spverb!False! \\ \hline
Table class (option) & \spverb!pygstiTbl! \\ \hline
Template path (option) & \spverb!None! \\ \hline
Latex command (option) & \spverb!pdflatex! \\ \hline
cptpPenaltyFactor & \spverb!0! \\ \hline
defaultBasename & \spverb!None! \\ \hline
defaultDirectory & \spverb!None! \\ \hline
distributeMethod & \spverb!deriv! \\ \hline
fiducial pairs & \spverb!None! \\ \hline
gaugeOptParams & \spverb![{}]! \\ \hline
hessianProjection & \spverb!optimal gate CIs! \\ \hline
linlogPercentile & \spverb!5! \\ \hline
max length list & \spverb![0, 1, 2, 4, 8]! \\ \hline
memLimit & \spverb!None! \\ \hline
minProbClip & \spverb!1e-06! \\ \hline
minProbClipForWeighting & \spverb!0.0001! \\ \hline
objective & \spverb!logl! \\ \hline
probClipInterval & \spverb!(-1000000.0, 1000000.0)! \\ \hline
radius & \spverb!0.0001! \\ \hline
weights & \spverb!None! \\ \hline
rho0 parameterization & \spverb!TP! \\ \hline
E0 parameterization & \spverb!full! \\ \hline
Gi parameterization & \spverb!TP! \\ \hline
Gx parameterization & \spverb!TP! \\ \hline
Gy parameterization & \spverb!TP! \\ \hline
\end{tabular}

\caption{\textbf{Listing of GST parameters and meta-data.}  These parameters and related metadata describe how the GST computation was performed which led to this report. \label{metadataTable}}
\end{center}
\end{table}

\begin{table}[h]
\begin{center}
\begin{tabular}[l]{|c|p{3in}|}
\hline
\textbf{Quantity} & \textbf{Value} \\ \hline
pyGSTi version & \spverb!0.9.3! \\ \hline
numpy & \spverb!1.11.2! \\ \hline
scipy & \spverb!0.18.1! \\ \hline
matplotlib & \spverb!1.5.3! \\ \hline
pyparsing & \spverb!2.0.1! \\ \hline
cvxopt & \spverb!1.1.8! \\ \hline
cvxpy & \spverb!0.4.5! \\ \hline
pptx & \spverb!0.6.0! \\ \hline
nose & \spverb!1.3.7! \\ \hline
PIL & \spverb!3.4.1! \\ \hline
psutil & \spverb!4.3.1! \\ \hline
Python version & \spverb!2.7.10! \\ \hline
Python type & \spverb!CPython! \\ \hline
Python compiler & \spverb!GCC 4.2.1 Compatible Apple LLVM 7.0.0 (clang-700.0.59.5)! \\ \hline
Python build & \spverb!('default', 'Oct 23 2015 19:19:21')! \\ \hline
Python branch & \spverb!! \\ \hline
Python revision & \spverb!! \\ \hline
Platform summary & \spverb!Darwin-15.6.0-x86_64-i386-64bit! \\ \hline
System & \spverb!Darwin! \\ \hline
Sys Release & \spverb!15.6.0! \\ \hline
Sys Version & \spverb!Darwin Kernel Version 15.6.0: Mon Jan  9 23:07:29 PST 2017; root:xnu-3248.60.11.2.1~1/RELEASE_X86_64! \\ \hline
Machine & \spverb!x86_64! \\ \hline
Processor & \spverb!i386! \\ \hline
\end{tabular}

\caption{\textbf{Listing of the software environment.}  Note that this describes the software environment of the machine used to generate this report, and not necessarily the machine used to perform the core GST gate set estimatino.\label{metadataTable}}
\end{center}
\end{table}


\input{full_reportD_appendices.tex}

\end{document}