MXB201 Exam Notes.tex

%!TEX TS-program = xelatex
%!TEX options = -aux-directory=Debug -shell-escape -file-line-error -interaction=nonstopmode -halt-on-error -synctex=1 "%DOC%"
\documentclass{article}
\input{LaTeX-Submodule/template.tex}

% Additional packages & macros
\usepackage{mathdots}
\setitemize{leftmargin=*,topsep=1ex,partopsep=0ex,itemsep=-1ex,partopsep=0ex,parsep=1ex}

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\nullity}{nullity}

\usepackage{changepage} % Modify page width
\usepackage{multicol} % Use multiple columns
\usepackage[explicit]{titlesec} % Modify section heading styles

\titleformat{\section}{\raggedright\normalfont\bfseries}{}{0em}{#1}
\titleformat{\subsection}{\raggedright\normalfont\small\bfseries}{}{0em}{#1}

%% A4 page
\geometry{
a4paper,
margin = 10mm
}

%% Hide horizontal rule 
\renewcommand{\headrulewidth}{0pt}
\fancyhead{}

%% Hide page numbers
\pagenumbering{gobble}

%% Multi-columns setup
\setlength\columnsep{4pt}

%% Paragraph setup
\setlength\parindent{0pt}
\setlength\parskip{0pt}

%% Customise section heading styles
% \titleformat*\section{\raggedright\bfseries}

\begin{document}
% Modify spacing
\titlespacing*\section{0pt}{0.5ex}{1ex}
\titlespacing*\subsection{0pt}{0.5ex}{1ex}
%
\setlength\abovecaptionskip{8pt}
\setlength\belowcaptionskip{-15pt}
\setlength\textfloatsep{0pt}
%
\setlength\abovedisplayskip{1pt}
\setlength\belowdisplayskip{1pt}

\begin{multicols*}{3}
    \subsection{General Solution to a Linear System}
    If \(\symbfit{b} \in \columnspace{A}\): \(\symbfit{x}_g = \symbfit{x}_p + \symbfit{x}_n\)
    where \(\symbfit{x}_p \in \R^n\) and \(\symbfit{x}_n \in \nullspace{A}\).
    \subsection{Minimum Norm Solution}
    \(\symbfit{x}_r \in \rowspace{A}\) where \(\symbfit{x}_r = \proj_{\rowspace{A}}{\left( \symbfit{x}_g \right)}\).
    \section{Least Squares (LS)}
    If \(\symbfit{b} \not\in \columnspace{A}\): \(\symbfit{x} = \argmin_{\symbfit{x}^\ast \in \R^n} \norm{\symbfit{b} - \symbf{A}\symbfit{x}^\ast}\).
    \(\symbfit{b} - \symbf{A} \symbfit{x} \in \nullspace{A} \implies \symbf{A}^\top \left( \symbfit{b} - \symbf{A}\symbfit{x} \right) = \symbfup{0}\).
    \begin{align*}
        \symbf{A}^\top \symbf{A}\symbfit{x} & = \symbf{A}^\top \symbfit{b} &  & \text{(Normal Equations)}
    \end{align*}
    \subsection{Orthogonal Projection}
    \begin{align*}
        \symbf{P}             & = \symbf{A} \left( \symbf{A}^\top \symbf{A} \right)^{-1} \symbf{A}^\top     \\
        \symbf{P} \symbfit{b} & = \proj_{\columnspace{A}} \left( \symbfit{b} \right) = \symbf{A}\symbfit{x}
    \end{align*}
    \(\symbf{P}\) is idempotent (\(\symbf{P}^2 = \symbf{P}\)) and \(\symbf{P}^\top = \symbf{P}\).
    \subsection{Dependent Columns}
    If \(\nullity{\left( \symbf{A} \right)} > 0\), NE yields infinitely many solutions
    as \(\nullspace{A} = \mathcal{N}\left(\symbf{A}^\top \symbf{A}\right)\).
    \subsection{Orthogonal Complement Projections}
    Given \(\symbf{P} = \proj_V\): \(\symbf{Q} = \proj_{V^\perp} = \symbf{I} - \symbf{P}\)
    \begin{equation*}
        \symbfit{b} = \proj_V(\symbfit{b}) + \proj_{V^\perp}(\symbfit{b}) = \symbf{P}\symbfit{b} + \symbf{Q}\symbfit{b}
    \end{equation*}
    \begin{align*}
        \left( \symbf{P}\symbfit{b} \right)^\top \symbf{Q}\symbfit{b} & = 0                                   \\
        \symbf{P} \symbf{Q}                                           & = \symbf{0} &  & \text{(zero matrix)}
    \end{align*}
    \subsection{Change of Basis}
    Given the basis \(W = \left\{ \symbfit{w}_1,\: \dots,\: \symbfit{w}_n \right\}\)
    \begin{align*}
        \symbfit{b} & = c_1 \symbfit{w}_1 + \cdots + c_n \symbfit{w}_n                         \\
        \symbfit{b} & = \symbf{W} \symbfit{c} \iff \left( \symbfit{b} \right)_W = \symbfit{c}.
    \end{align*}
    \subsection{Orthonormal Basis}
    Normalised and orthogonal basis vectors.
    For \(Q = \left\{ \symbfit{q}_1,\: \dots,\: \symbfit{q}_n \right\}\),
    \(\symbfit{q}_i^\top \symbfit{q}_j = \delta_{ij}\), where
    \begin{equation*}
        \delta_{ij} = \begin{cases}
            1, & i = j   \\
            0, & i \ne j
        \end{cases}
    \end{equation*}
    \begin{equation*}
        \symbf{Q} \symbfit{c} = \symbfit{b}
        \iff
        \symbf{Q}^\top \symbfit{b} = \symbfit{c} = \left( \symbfit{b} \right)_Q
    \end{equation*}
    \subsection{Orthogonal Matrices}
    \begin{equation*}
        \symbf{Q}^\top = \symbf{Q}^{-1}
        \iff
        \symbf{Q}^\top \symbf{Q} = \symbf{Q}\symbf{Q}^\top = \symbf{I}.
    \end{equation*}
    \subsection{Projection onto a Vector}
    \begin{align*}
        \proj_{\symbfit{a}} \left( \symbfit{b} \right) & = \symbfit{a} \left( \symbfit{a}^\top \symbfit{a} \right)^{-1} \symbfit{a}^\top \symbfit{b} \\
                                                       & = \frac{\symbfit{a}}{\norm{\symbfit{a}}^2} \symbfit{a} \cdot \symbfit{b}
    \end{align*}
    Using a unit vector \(\symbfit{q}\):
    \begin{equation*}
        \proj_{\symbfit{q}} \left( \symbfit{b} \right) = \symbfit{q} \left( \symbfit{q} \cdot \symbfit{b} \right)
    \end{equation*}
    \subsection{Gram-Schmidt Process}
    Converts the basis \(W\) that spans \(\columnspace{A}\) to an orthonormal basis \(Q\).
    \begin{align*}
        \symbfit{v}_i & = \symbfit{w}_i - \sum_{j = 1}^{i - 1} \symbfit{q}_j \abracket*{\symbfit{q}_j,\: \symbfit{w}_i} & \symbfit{q}_i & = {\scriptstyle \symbfit{v}_i / \norm*{\symbfit{v}_i}}
    \end{align*}
    \(V\) and \(Q\) span \(W\), and \(V\) is orthogonal.
    \subsection{QR Decomposition}
    \begin{align*}
        \symbf{A} & = \symbf{Q} \symbf{R}                                                                                                                                                                  \\
        \symbf{R} & = \begin{bmatrix}
                          \norm{\symbfit{v}_1} & \abracket*{\symbfit{q}_1,\: \symbfit{w}_2} & \cdots & \abracket*{\symbfit{q}_1,\: \symbfit{w}_n}       \\
                          0                    & \norm{\symbfit{v}_2}                       & \ddots & \vdots                                           \\
                          \vdots               & \ddots                                     & \ddots & \abracket*{\symbfit{q}_{n - 1},\: \symbfit{w}_n} \\
                          0                    & \cdots                                     & 0      & \norm{\symbfit{v}_n}
                      \end{bmatrix}
    \end{align*}
    where \(\symbf{Q}\) is found by applying the Gram-Schmidt process and \(\symbf{R}\) is upper triangular. \(\symbf{R} \symbfit{x} = \symbf{Q}^\top \symbfit{b}\) solves LS\@.
    \section{Eigenvalues and Eigenvectors}
    \begin{equation*}
        \symbf{A} \symbfit{v} = \lambda \symbfit{v} \iff \left( \lambda \symbf{I} - \symbf{A} \right) \symbfit{v} = \symbfup{0} : \symbfit{v} \neq \symbfup{0}
    \end{equation*}
    \subsection{Characteristic Polynomial}
    \begin{equation*}
        P\left( \lambda \right) = \det{\left( \lambda \symbf{I} - \symbf{A} \right)} = 0.
    \end{equation*}
    \subsection{Eigen Decomposition}
    \begin{align*}
        \symbf{A} \symbf{V} = \symbf{V} \symbf{D}
                  & \iff
        \symbf{A} = \symbf{V} \symbf{D} \symbf{V}^{-1}                       \\
        \symbf{V} & = \begin{bmatrix}
                          \symbfit{v}_1 & \cdots & \symbfit{v}_n
                      \end{bmatrix}                 \\
        \symbf{D} & = \diag{\left( \lambda_1,\: \dots,\: \lambda_n \right)}.
    \end{align*}
    \subsection{Eigenspace} 
    The eigenspace associated with \(\lambda_i\) is the span of eigenvectors: \(\mathcal{N}{\left( \lambda_i \symbf{I} - \symbf{A} \right)}\). 
    \subsection{Algebraic Multiplicity \texorpdfstring{\(\mu\left( \lambda_i \right)\)}{mu(lambda i)}}
    Multiplicity of \(\lambda_i\) in \(P(\lambda)\),
    for \(d \leq n\) distinct eigenvalues,
    \begin{equation*}
        P\left( \lambda \right) = \left( \lambda - \lambda_1 \right)^{\mu\left( \lambda_1 \right)} \cdots \left( \lambda - \lambda_d \right)^{\mu\left( \lambda_d \right)}.
    \end{equation*}
    In general
    \begin{gather*}
        1 \leq \mu\left( \lambda_i \right) \leq n         \\
        \sum_{i = 1}^d \mu \left( \lambda_i \right) = n
    \end{gather*}
    If \(\nullity{\left( \symbf{A} \right)} > 0\)
    \begin{equation*}
        \exists k : \lambda_k = 0 : \mu\left( \lambda_k \right) = \nullity{\left( \symbf{A} \right)}
    \end{equation*}
    \subsection{Geometric Multiplicity \texorpdfstring{\(\gamma\left( \lambda_i \right)\)}{gamma(lambda i)}}
    The dimension of each eigenspace \(\lambda_i\)
    \begin{equation*}
        \gamma \left( \lambda_i \right) = \nullity{\left( \lambda_i \symbf{I} - \symbf{A} \right)}.
    \end{equation*}
    Given \(d \leq n\) distinct eigenvalues,
    \begin{gather*}
        1 \leq \gamma\left( \lambda_i \right) \leq \mu\left( \lambda_i \right) \leq n \\
        d \leq \sum_{i = 1}^d \gamma \left( \lambda_i \right) \leq n.
    \end{gather*}
    Eigenvectors corresponding to distinct eigenvalues are linearly dependent.
    \subsection{Defective Matrix}
    \(\symbf{A}\) lacks a complete eigenbasis:
    \begin{equation*}
        \exists k : \gamma\left( \lambda_k \right) < \mu\left( \lambda_k \right)
    \end{equation*}
    \subsection{Matrix Similarity}
    \(\symbf{A}\) and \(\symbf{B}\) are similar if \(\symbf{B} = \symbf{P}^{-1} \symbf{A} \symbf{P}\).

    They share \(P(\lambda)\), ranks, determinants, traces, and eigenvalues (also \(\mu\) and \(\gamma\)).
    \subsection{Symmetric Matrices \texorpdfstring{\(\symbf{S}^\top = \symbf{S}\)}{S' = S}}
    \(\symbf{S}\) is always diagonalisable and has
    real eigenvalues with real orthogonal eigenspaces: \(\symbf{S} = \symbf{Q} \symbf{D} \symbf{Q}^\top\), where \(\symbf{Q}\) is found through QR\@: \(\symbf{V} = \symbf{Q} \symbf{R}\).
    \subsection{Skew-Symmetric Matrices \texorpdfstring{\(\symbf{K}^\top = -\symbf{K}\)}{K' = -K}}
    Eigenvalues are always purely imaginary.
    \subsection{Positive-Definite Matrices}
    \(\symbf{S}\) is (symmetric) positive definite (SPD) if all its eigenvalues are positive, likewise
    \begin{equation*}
        \symbfit{x}^\top \symbf{S} \symbfit{x} > 0 : \forall \symbfit{x} \in \R^n \backslash \left\{ \symbfup{0} \right\}
    \end{equation*}
    \subsection{Matrix Functions}
    Given a nondefective matrix:
    \begin{align*}
        f\left( \symbf{A} \right) & = \symbf{V} f\left( \symbf{D} \right) \symbf{V}^{-1}                                                               \\
                                  & = \symbf{V} \diag{\left( f\left( \lambda_1 \right),\: \ldots,\: f\left( \lambda_n \right) \right)} \symbf{V}^{-1}.
    \end{align*}
    for an analytic function \(f\).
    \subsection{Cayley-Hamilton Theorem}
    \begin{align*}
        \forall \symbf{A} : P\left( \symbf{A} \right) = \symbfup{0} &  & \text{(zero matrix)}
    \end{align*}
    \section{Singular Value Decomposition}
    \begin{gather*}
        \symbf{A} \symbf{V} = \symbf{U} \symbf{\Sigma}
        \iff
        \symbf{A} = \symbf{U} \symbf{\Sigma} \symbf{V}^\top                                        \\
        \symbf{V}^\top = \symbf{V}^{-1}, \quad \symbf{U}^\top = \symbf{U}^{-1}                                                          \\
        \symbf{\Sigma} = \diag{\left( \sigma_1,\: \dots,\: \sigma_r,\: 0,\: \dots,\: 0 \right)}.
    \end{gather*}
    Left singular vectors \(\symbfit{u}\): \(\symbf{U} \in \R^{m \times m}\)
    \begin{align*}
        \columnspace{A}   & = \vspan{\left( \left\{ \symbfit{u}_{i \leq r} \right\} \right)}     \\
        \leftnullspace{A} & = \vspan{\left( \left\{ \symbfit{u}_{r < i \leq m} \right\} \right)}
    \end{align*}
    Right singular vectors \(\symbfit{v}\): \(\symbf{V} \in  \R^{n \times n}\)
    \begin{align*}
        \rowspace{A}  & = \vspan{\left( \left\{ \symbfit{v}_{i \leq r} \right\} \right)}     \\
        \nullspace{A} & = \vspan{\left( \left\{ \symbfit{v}_{r < i \leq n} \right\} \right)}
    \end{align*}
    Singular values \(\sigma_i\): \(\symbf{\Sigma} \in \R^{m \times n}\)

    The eigenvalues of \(\symbf{A}^\top\symbf{A}\) and \(\symbf{A}\symbf{A}^\top\)
    are equal, \(\symbf{\Sigma}^\top \symbf{\Sigma}\) and \(\symbf{\Sigma} \symbf{\Sigma}^\top\) have the same diagonal entries, and
    when \(m = n\), \(\symbf{\Sigma}^\top\symbf{\Sigma} = \symbf{\Sigma} \symbf{\Sigma}^\top = \symbf{\Sigma}^2\).
    To find \(\sigma_i\) compute:
    \begin{align*}
        \symbf{A}^\top \symbf{A} & = \symbf{V} \symbf{\Sigma}^\top \symbf{\Sigma} \symbf{V}^\top \\
        \symbf{A} \symbf{A}^\top & = \symbf{U} \symbf{\Sigma} \symbf{\Sigma}^\top \symbf{U}^\top
    \end{align*}
    so that \(\sigma_i = \sqrt{\lambda}_i\) where \(\sigma_1 \geq \cdots \geq \sigma_r > 0\).
    \subsection{Reduced SVD}
    Ignores \(m - n\) ``0'' rows in \(\symbf{\Sigma}\) so that \(\symbf{U} \in \R^{m \times n}\), \(\symbf{\Sigma} \in \R^{n \times n}\), and \(\symbf{V} \in \R^{n \times n}\).
    \subsection{Pseudoinverse}
    Consider the inverse mapping \(\symbfit{u}_i \mapsto \frac{1}{\sigma_i} \symbfit{v}_i\)
    \begin{gather*}
        \symbf{A}^\dagger \symbfit{u}_i = \frac{1}{\sigma_i} \symbfit{v}_i
        \iff
        \symbf{A}^\dagger \symbfit{u}_i = \frac{1}{\sigma_i} \symbfit{v}_i \symbfit{u}^\top_i \symbfit{u}_i \\
        \symbf{A}^\dagger = \sum_{i = 1}^r \frac{1}{\sigma_i} \symbfit{v}_i \symbfit{u}^\top_i
        \iff
        \symbf{A}^\dagger = \symbf{V} \symbf{\Sigma}^\dagger \symbf{U}^\top
    \end{gather*}
    where \(\symbf{\Sigma}^\dagger = \diag{\left( \frac{1}{\sigma_1},\: \dots,\: \frac{1}{\sigma_r},\: 0,\: \dots,\: 0 \right)}\).
    \(\symbfit{x} = \symbf{A}^\dagger \symbfit{b}\) solves LS\@.
    \subsection{Truncated SVD}
    Express \(\symbf{A}\) as the sum of rank-1 matrices:
    \begin{equation*}
        \symbf{A} = \sum_{i = 1}^n \sigma_i \symbfit{u}_i \symbfit{v}^\top_i \approx \tilde{\symbf{A}} = \sum_{i = 1}^\nu \sigma_i \symbfit{u}_i \symbfit{v}^\top_i.
    \end{equation*}
    for the rank-\(\nu\) approximation of \(\symbf{A}\).

    Using the SVD\@:
    \begin{equation*}
        \tilde{\symbf{A}} = \symbf{U} \symbf{\Sigma} \symbf{V}^\top
    \end{equation*}
    \(\symbf{U} \in \R^{m \times \nu}\), \(\symbf{\Sigma} \in \R^{\nu \times \nu}\), and \(\symbf{V} \in \R^{n \times \nu}\).

    When \(\nu \geq r\), \(\tilde{\symbf{A}} = \symbf{A}\) as \(\sigma_{i > r} = 0\).
    \section{General Vector Spaces}
    \(V\) is a vector space with vectors \(\symbfit{v} \in V\) if the following 10 axioms are satisfied
    for \(\forall \symbfit{u}, \symbfit{v}, \symbfit{w} \in V\) and \(\forall k, m \in \mathbb{F}\),
    given an addition and scalar multiplication operation.

    \underline{For the addition operation:}
    \begin{itemize}
        \item Closure: \(\symbfit{u} + \symbfit{v} \in V\)
        \item Commutativity: \(\symbfit{u} + \symbfit{v} = \symbfit{v} + \symbfit{u} \in V\)
        \item Associativity: \begin{equation*}\symbfit{u} + \left( \symbfit{v} + \symbfit{w} \right) = \left( \symbfit{u} + \symbfit{v} \right) + \symbfit{w}\end{equation*}
        \item Identity: \(\exists \symbfup{0} \in V : \symbfit{u} + \symbfup{0} = \symbfup{0} + \symbfit{u} = \symbfit{u}\)
        \item Inverse: \(\exists \left( -\symbfit{u} \right) \in V : \symbfit{u} + \left( -\symbfit{u} \right) = \symbfup{0}\)
    \end{itemize}
    \underline{For the scalar multiplication operation:}
    \begin{itemize}
        \item Closure: \(k \symbfit{u} \in V\)
        \item Distributivity: \(k \left( \symbfit{u} + \symbfit{v} \right) = k\symbfit{u} + k\symbfit{v}\)
        \item Distributivity: \(\left( k + m \right) \symbfit{u} = k\symbfit{u} + m\symbfit{u}\)
        \item Associativity: \(k \left( m\symbfit{u} \right) = \left( k m \right) \symbfit{u}\)
        \item Identity: \(\exists 1 \in \mathbb{F} : 1 \symbfit{u} = \symbfit{u}\)
    \end{itemize}

    \subsection{Examples of Vector Spaces}
    The set of all \(m \times n\) matrices \(\mathscr{M}_{mn}\) with matrix addition and scalar matrix multiplication.

    The set of all functions \(\mathscr{F}\left( \Omega \right) : \Omega \to \R\) with addition and scalar multiplication defined pointwise.
    \subsection{Subspaces}
    The subset \(W \subset V\) is itself a vector space if it is closed under addition and scalar multiplication.
    \subsection{Examples of Subspaces}
    \underline{Subspaces of \(\R^n\):}
    \begin{itemize}
        \item Lines, planes and higher-dimensional analogues in \(\R^n\) \emph{passing through the origin}.
    \end{itemize}
    \underline{Subspaces of \(\mathscr{M}_{nn}\):}
    \begin{itemize}
        \item The set of all \emph{symmetric} \(n \times n\) matrices, denoted \(\mathscr{S}_n \subset \mathscr{M}_{nn}\).
        \item The set of all \emph{skew symmetric} \(n \times n\) matrices, denoted \(\mathscr{K}_n \subset \mathscr{M}_{nn}\).
    \end{itemize}
    \underline{Subspaces of \(\mathscr{F}\):}
    \begin{itemize}
        \item The set of all \emph{polynomials} of degree \(n\) or less, denoted \(\mathscr{P}_n\left( \Omega \right) \subset \mathscr{F}\left( \Omega \right)\).
        \item The set of all \emph{continuous functions}, denoted \(C\left( \Omega \right) \subset \mathscr{F}\left( \Omega \right)\).
        \item The set of all continuous functions with \emph{continuous \(n\)th derivatives}, denoted \(C^n\left( \Omega \right) \subset C\left( \Omega \right)\).
        \item The set of all functions \(f\) defined on \(\interval{0}{1}\) satisfying \(f\left( 0 \right) = f\left( 1 \right)\).
    \end{itemize}
    \subsection{General Vector Space Terminology}
    Let \(S = \left\{ \symbfit{v}_1,\: \dots,\: \symbfit{v}_k \right\}\) and \(c_1,\: \dots,\: c_k \in \mathbb{F}\):
    \begin{itemize}
        \item The linear combination of \(S\)
              is a vector of the form \(\symbfit{v} = c_1 \symbfit{v}_1 + \cdots + c_k \symbfit{v}_k\).
        \item \(S\) is linearly independent iff
              \(c_1 \symbfit{v}_1 + \cdots + c_k \symbfit{v}_k = \symbfup{0}\) has the trivial solution.
        \item \(\vspan{\left( S \right)}\) is the set of all linear combinations of \(S\).
    \end{itemize}
    \(S\) is a \textit{basis} for a vector space \(V\) if
    \begin{itemize}
        \item \(S\) is linearly independent.
        \item \(\vspan{\left( S \right)} = V\).
    \end{itemize}
    The number of basis vectors denotes the dimension of \(V\).

    \(C\) is infinite dimensional.
    \subsection{Examples of Standard Bases}
    \begin{itemize}
        \item \(\mathscr{M}_{22}\):
              \begin{equation*}
                  \left\{ \begin{bmatrix*}
                      1 & 0 \\
                      0 & 0
                  \end{bmatrix*},\: \begin{bmatrix*}
                      0 & 0 \\
                      1 & 0
                  \end{bmatrix*},\: \begin{bmatrix*}
                      0 & 1 \\
                      0 & 0
                  \end{bmatrix*},\: \begin{bmatrix*}
                      0 & 0 \\
                      0 & 1
                  \end{bmatrix*} \right\}
              \end{equation*}
        \item \(\mathscr{S}_{22}\): \(\left\{ \begin{bmatrix*}
                  1 & 0 \\
                  0 & 0
              \end{bmatrix*},\: \begin{bmatrix*}
                  0 & 1 \\
                  1 & 0
              \end{bmatrix*},\: \begin{bmatrix*}
                  0 & 0 \\
                  0 & 1
              \end{bmatrix*} \right\}\)
        \item \(\mathscr{K}_{22}\): \(\left\{ \begin{bmatrix*}
                  0 & 1 \\
                  -1 & 0
              \end{bmatrix*}\right\}\)
        \item \(\mathscr{P}_3\): \(\left\{ 1,\: x,\: x^2,\: x^3 \right\}\)
    \end{itemize}
    \subsection{Linear Transformations}
    \(T:V \to W\) satisfying
    \begin{align*}
        T\left( k\symbfit{u} \right)              & = k T\left( \symbfit{u} \right)                             \\
        T\left( \symbfit{u} + \symbfit{v} \right) & = T\left( \symbfit{u} \right) + T\left( \symbfit{v} \right)
    \end{align*}
    \underline{Constructing \(\symbf{A} = \left( T \right)_{B',\: B}\):}

    Consider the map of \(\left( \symbfit{v} \right)_B = \symbfit{x}\) of \(\symbfit{v} \in V\) to
    \(\left( \symbfit{w} \right)_{B'} = \symbfit{b}\) of \(\symbfit{w} \in W\),
    where \(B = \left\{ \symbfit{v}_1,\: \dots,\: \symbfit{v}_n \right\}\) and \(B' = \left\{ \symbfit{w}_1,\: \dots,\: \symbfit{w}_m \right\}\).
    \begin{align*}
        T\left( \symbfit{v} \right)                                                                           & = \symbfit{w} \\
        \begin{bmatrix}
            T\left( \symbfit{v}_1 \right) & \cdots & T\left( \symbfit{v}_n \right)
        \end{bmatrix} \symbfit{x}                               & = \symbf{W} \symbfit{b}                                     \\
        \begin{bmatrix}
            \left( T\left( \symbfit{v}_1 \right) \right)_{B'} & \cdots & \left( T\left( \symbfit{v}_n \right) \right)_{B'}
        \end{bmatrix} \symbfit{x} & = \symbfit{b}                  \\
        \symbf{A} \symbfit{x}                                                                                 & = \symbfit{b}
    \end{align*}
    \subsection{Isomorphism (\texorpdfstring{\(\cong\)}{≅})}
    \(T : V \to W\) is an isomorphism between \(V\) and \(W\) if there exists a bijection between the two vector spaces.

    \(\forall V : \dim{\left( V \right)} = n : V \cong \R^n\), \(\mathscr{M}_{mn} \cong \R^{mn}\)
    and \(\mathscr{P}_n \cong \R^{n + 1}\).
    \subsection{Fundamental Subspaces of \texorpdfstring{\(T\)}{T}}
    \begin{itemize}
        \item The set of all vectors in \(V\) that map to \(W\) is the \textbf{image} of \(T\), denoted \(\vim{\left( T \right)}\).
        \item The set of all vectors in \(W\) that is mapped to by a vector in \(V\) is the \textbf{range} of \(T\), denoted \(\vrange{\left( T \right)}\).
        \item The set of all vectors in \(V\) that \(T\) maps to \(\symbfup{0}_W\) is the \textbf{kernel} of \(T\), denoted \(\vker{\left( T \right)}\).
    \end{itemize}
    If finite, \(\dim{\left( \vrange{\left( T \right)} \right)} = \vrank{\left( T \right)}\)
    and \(\dim{\left( \vker{\left( T \right)} \right)} = \nullity{\left( T \right)}\).
    \begin{equation*}
        \vrank{\left( T \right)} + \nullity{\left( T \right)} = \dim{\left( V \right)}.
    \end{equation*}
    \subsection{Inner Product Spaces}
    \begin{equation*}
        \abracket*{\cdot,\: \cdot} : V \times V \to \R.
    \end{equation*}
    For \(\symbfit{u},\: \symbfit{v},\: \symbfit{w} \in V\)
    and \(k \in \R\):
    \begin{itemize}
        \item Symmetry: \(\abracket*{\symbfit{u},\: \symbfit{v}} = \abracket*{\symbfit{v},\: \symbfit{u}}\)
        \item Linearity:
              \begin{equation*}
                  \abracket*{\symbfit{u} + \symbfit{v},\: \symbfit{w}} = \abracket*{\symbfit{u},\: \symbfit{w}} + \abracket*{\symbfit{v},\: \symbfit{w}}
              \end{equation*}
        \item Linearity: \(\abracket*{k \symbfit{u},\: \symbfit{v}} = k\abracket*{\symbfit{u},\: \symbfit{v}}\)
        \item Positive semi-definitiveness:
              \begin{equation*}
                  \abracket*{\symbfit{u},\: \symbfit{u}} \geq 0,\: \abracket*{\symbfit{u},\: \symbfit{u}} = 0 \iff \symbfit{u} = \symbfup{0}
              \end{equation*}
    \end{itemize}
    \underline{For \(\symbfit{u},\: \symbfit{v} \in \R^n\):}
    \begin{itemize}
        \item \(\abracket*{\symbfit{u},\: \symbfit{v}} = \symbfit{u} \cdot \symbfit{v} = \symbfit{u}^\top \symbfit{v}\).
        \item \(\abracket*{\symbfit{u},\: \symbfit{v}} = \symbfit{u}^\top \symbf{A} \symbfit{v}\) where \(\symbf{A}\) is SPD\@.
    \end{itemize}
    \underline{For \(\symbf{A},\: \symbf{B} \in \mathscr{M}_{mn}\):}
    \begin{itemize}
        \item \(\abracket*{\symbf{A},\: \symbf{B}} = \Tr{\left( \symbf{A}^\top \symbf{B} \right)}\).
    \end{itemize}
    \underline{For \(f,\: g \in C\left( \interval{a}{b} \right)\):}
    \begin{itemize}
        \item \(\abracket*{f,\: g} = \int_a^b f\left( x \right) g\left( x \right) \odif{x}\).
        \item \(\abracket*{f,\: g} = \int_a^b f\left( x \right) g\left( x \right) w\left( x \right) \odif{x}\).
    \end{itemize}
    where \(w\left( x \right) > 0 : \forall x \in \interval{a}{b}\).
    \subsection{Norms}
    \begin{itemize}
        \item \(\norm*{\symbfit{v}} = \sqrt{\abracket*{\symbfit{v},\: \symbfit{v}}}\).
        \item \(\norm*{\symbfit{v}} \geq 0\), and \(\norm*{\symbfit{v}} = 0 \iff \symbfit{v} = \symbfup{0}\).
        \item \(\norm*{k \symbfit{v}} = \abs*{k} \norm*{\symbfit{v}} : \forall k \in \R\).
        \item \(\norm*{\symbfit{u} + \symbfit{v}} \leq \norm*{\symbfit{u}} + \norm*{\symbfit{v}}\).
    \end{itemize}
    \underline{Examples:}
    \begin{itemize}
        \item \(\forall \symbf{A} \in \mathscr{M}_{mn} : \norm*{\symbf{A}} = \sqrt{\sum_{i = 1}^m \sum_{j = 1}^n a_{ij}^2}\).
        \item \(\forall f \in C\left( \interval{a}{b} \right) : \norm*{f} = \sqrt{\int_a^b f\left( x \right)^2 \odif{x}}\).
    \end{itemize}
    \subsection{Orthogonality}
    \begin{equation*}
        \abracket*{\symbfit{v},\: \symbfit{v}} = 0.
    \end{equation*}
    \subsection{Orthogonal Complements of \texorpdfstring{\(\mathscr{M}_{n}\)}{Mn}}
    Given \(\symbf{P}_{\mathscr{S}_n} = \proj_{\mathscr{S}_n}\) and
    \(\symbf{P}_{\mathscr{K}_n} = \proj_{\mathscr{K}_n}\)
    \begin{equation*}
        \symbf{P}_{\mathscr{S}_n} = \symbf{I} - \symbf{P}_{\mathscr{K}_n}                                                                                                    \\
    \end{equation*}
    \begin{align*}
        \symbf{S} & = \symbf{P}_{\mathscr{S}_n} \symbf{M} = \proj_{\symbf{P}_{\mathscr{S}_n}}{\left( \symbf{M} \right)} = \frac{\symbf{M} + \symbf{M}^\top}{2} \\
        \symbf{K} & = \symbf{P}_{\mathscr{K}_n} \symbf{M} = \proj_{\symbf{P}_{\mathscr{K}_n}}{\left( \symbf{M} \right)} = \frac{\symbf{M} - \symbf{M}^\top}{2}
    \end{align*}
    \(\symbf{S} \in \mathscr{S}_n\), \(\symbf{K} \in \mathscr{K}_n\), and \(\symbf{S} + \symbf{K} = \symbf{M} \in \mathscr{M}_n\).
    \section{Theorems}
    \begin{itemize}
        \item \(\symbf{A}^\top \symbf{A}\) is always positive semi-definite,
              and \(\mathcal{N}\left( \symbf{A}^\top\symbf{A} \right) = \nullspace{A}\) so that
              \(\vrank{\left( \symbf{A}^\top \symbf{A} \right) = \vrank{\left( \symbf{A} \right)}}\).
              \(\symbf{A}^\top \symbf{A}\) is positive definite and
              \(\symbf{A}^\top \symbf{A}\) is invertible when \(\nullity{\left( \symbf{A} \right)} = 0\).
        \item When \(\symbf{A}\) is square and invertible, \(\left( \symbf{A}^\top \symbf{A} \right)^{-1} = \symbf{A}^{-1} \symbf{A}^{-\top}\) and \(\symbf{P} = \symbf{I}\)
              otherwise \(\symbf{P} = \symbf{Q} \symbf{Q}^\top\) using QR\@.
        \item \(\symbf{P}^2 = \symbf{P} \land \symbf{P}^\top = \symbf{P} \iff \symbf{P} = \proj_{\columnspace{P}}\).
              \(\symbf{P} \symbfit{v} = \symbf{P}^2 \symbfit{v} \iff \lambda \symbfit{v} = \lambda^2 \symbfit{v}\) implies \(\lambda = 0,\: 1\).
        \item \(\symbf{A}^\top \symbf{A}\) and \(\symbf{A} \symbf{A}^\top\) share eigenvalues,
              \begin{align*}
                  \symbf{A}^\top \symbf{A} \symbfit{v}                                         & = \lambda \symbfit{v}                           \\
                  \left( \symbf{A} \symbf{A}^\top \right) \left( \symbf{A} \symbfit{v} \right) & = \lambda \left( \symbf{A} \symbfit{v} \right).
              \end{align*}
              \(\symbf{A} \symbfit{v} = \symbfup{0} \implies \lambda = 0\), else \(\symbfit{w} = \symbf{A} \symbfit{v}\) is an eigenvector of \(\symbf{A} \symbf{A}^\top\).
        \item For symmetric \(\symbf{S} \in \R^{n \times n}\):
              \begin{equation*}
                  \symbf{S} = \sum_{i = 1}^n \lambda_i \symbfit{q}_i \symbfit{q}_i^\top = \sum_{i = 1}^n \lambda_i \proj_{\symbfit{q}_i}
              \end{equation*}
        \item For \(\symbf{W} = \symbfit{w} \in \R^{n \times 1}\):
              \begin{align*}
                  \symbf{W}         & = \begin{bmatrix*}
                                            \hat{\symbfit{w}}
                                        \end{bmatrix*} \begin{bmatrix*}
                                                           \norm*{\symbfit{w}}
                                                       \end{bmatrix*} \begin{bmatrix*}
                                                                          1
                                                                      \end{bmatrix*} \\
                  \symbf{W}^\dagger & = \hat{\symbfit{w}}^\top / \norm*{\symbfit{w}}
              \end{align*}
    \end{itemize}
    \section{Identities}
    \begin{itemize}
        \item \(\left( \symbf{A} \symbf{B} \right)^\top = \symbf{B}^\top \symbf{A}^\top\).
        \item \(\left( \symbf{A} \symbf{B} \right)^{-1} = \symbf{B}^{-1} \symbf{A}^{-1}\) if \(\symbf{A}\), \(\symbf{B}\) invertible.
        \item \(\left( \symbf{A}^\top \right)^{-1} = \left( \symbf{A}^{-1} \right)^\top\) if \(\symbf{A}\) invertible:
              \begin{align*}
                  \symbf{A}^\top \left( \symbf{A}^{-1} \right)^\top & = \left( \symbf{A}^{-1} \symbf{A} \right)^\top = \symbf{I} \\
                  \left( \symbf{A}^{-1} \right)^\top \symbf{A}^\top & = \left( \symbf{A} \symbf{A}^{-1} \right)^\top = \symbf{I}
              \end{align*}
        \item \(\abracket*{\symbf{A}\symbfit{x},\: \symbfit{y}} = \abracket*{\symbfit{x},\: \symbf{A}^\top \symbfit{y}}\):
              \begin{equation*}
                  \left( \symbf{A} \symbfit{x} \right)^\top \symbfit{y} = \symbfit{x}^\top \left( \symbf{A}^\top \symbfit{y} \right)
              \end{equation*}
        \item \(\det{\left( \symbf{A} \symbf{B} \right)} = \det{\left( \symbf{A} \right)} \det{\left( \symbf{B} \right)}\).
        \item If \(\symbf{A}\) is triangular, \(\det{\left( \symbf{A} \right)} = \prod_{i = 1}^n a_{ii}\).
        \item For \(\symbf{A} \in \R^{n \times n}\):
              \begin{align*}
                  \Tr{\left( \symbf{A} \right)}                 & = \sum_{i = 1}^n a_{ii} = \sum_{i = 1}^n \lambda_i              \\
                  \det{\left( \symbf{A} \right)}                & = \prod_{i = 1}^n \lambda_i                                     \\
                  \det{\left( \symbf{A}^\top \symbf{A} \right)} & = \det{\left( \symbf{A} \right)}^2 = \prod_{i = 1}^n \sigma_i^2
              \end{align*}
        \item For \(\symbf{A} \in \R^{m \times n}\):
              \begin{align*}
                  \Tr{\left( \symbf{A}^\top \symbf{A} \right)}  & = \sum_{j = 1}^m \sum_{i = 1}^n a_{ij}^2 \\
                                                                & = \sum_{i = 1}^n \sigma_i^2              \\
                  \det{\left( \symbf{A}^\top \symbf{A} \right)} & = \prod_{i = 1}^n \sigma_i^2
              \end{align*}
    \end{itemize}
\end{multicols*}
\end{document}