Merge pull request #34 from tfjgeorge/jaco_revamp

Jaco revamp

docs/derivations/grads.lyx

@@ -0,0 +1,139 @@
+#LyX 2.3 created this file. For more info see http://www.lyx.org/
+\lyxformat 544
+\begin_document
+\begin_header
+\save_transient_properties true
+\origin unavailable
+\textclass article
+\use_default_options true
+\maintain_unincluded_children false
+\language english
+\language_package default
+\inputencoding auto
+\fontencoding global
+\font_roman "default" "default"
+\font_sans "default" "default"
+\font_typewriter "default" "default"
+\font_math "auto" "auto"
+\font_default_family default
+\use_non_tex_fonts false
+\font_sc false
+\font_osf false
+\font_sf_scale 100 100
+\font_tt_scale 100 100
+\use_microtype false
+\use_dash_ligatures true
+\graphics default
+\default_output_format default
+\output_sync 0
+\bibtex_command default
+\index_command default
+\paperfontsize default
+\use_hyperref false
+\papersize default
+\use_geometry false
+\use_package amsmath 1
+\use_package amssymb 1
+\use_package cancel 1
+\use_package esint 1
+\use_package mathdots 1
+\use_package mathtools 1
+\use_package mhchem 1
+\use_package stackrel 1
+\use_package stmaryrd 1
+\use_package undertilde 1
+\cite_engine basic
+\cite_engine_type default
+\use_bibtopic false
+\use_indices false
+\paperorientation portrait
+\suppress_date false
+\justification true
+\use_refstyle 1
+\use_minted 0
+\index Index
+\shortcut idx
+\color #008000
+\end_index
+\secnumdepth 3
+\tocdepth 3
+\paragraph_separation indent
+\paragraph_indentation default
+\is_math_indent 0
+\math_numbering_side default
+\quotes_style english
+\dynamic_quotes 0
+\papercolumns 1
+\papersides 1
+\paperpagestyle default
+\tracking_changes false
+\output_changes false
+\html_math_output 0
+\html_css_as_file 0
+\html_be_strict false
+\end_header
+
+\begin_body
+
+\begin_layout Standard
+WeightNorm1d
+\end_layout
+
+\begin_layout Standard
+\begin_inset Formula 
+\begin{align*}
+y_{i} & ^{\left(t\right)}=\left\langle \frac{w_{i}}{\left\Vert w_{i}\right\Vert _{2}},x{}^{\left(t\right)}\right\rangle \\
+ & =\left\langle \left\langle w_{i},w_{i}\right\rangle ^{-\frac{1}{2}}w_{i},x{}^{\left(t\right)}\right\rangle 
+\end{align*}
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+\begin_inset Formula 
+\begin{align*}
+\frac{\partial L}{\partial w_{i}} & =\sum_{t}\frac{\partial\ell^{\left(t\right)}}{\partial w_{i}}\\
+ & =\sum_{t}\frac{\partial\ell^{\left(t\right)}}{\partial y_{i}^{\left(t\right)}}\frac{\partial y_{i}^{\left(t\right)}}{\partial w_{i}}\\
+ & =\sum_{t}\frac{\partial\ell^{\left(t\right)}}{\partial y_{i}^{\left(t\right)}}\left(\frac{1}{\left\Vert w_{i}\right\Vert _{2}}x^{\left(t\right)}-w_{i}\left\langle w_{i},w_{i}\right\rangle ^{-\frac{3}{2}}\left\langle w_{i},x^{\left(t\right)}\right\rangle \right)\\
+ & =\frac{1}{\left\Vert w_{i}\right\Vert _{2}}\sum_{t}\frac{\partial\ell^{\left(t\right)}}{\partial y_{i}^{\left(t\right)}}\left(x^{\left(t\right)}-\frac{w_{i}}{\left\langle w_{i},w_{i}\right\rangle }\left\langle w_{i},x^{\left(t\right)}\right\rangle \right)
+\end{align*}
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+WeightNorm2d
+\end_layout
+
+\begin_layout Standard
+\begin_inset Formula 
+\begin{align*}
+y_{i} & ^{\left(s,t\right)}=\left\langle \frac{w_{i}}{\left\Vert w_{i}\right\Vert _{2}},x{}^{\left(s,t\right)}\right\rangle \\
+ & =\left\langle \left\langle w_{i},w_{i}\right\rangle ^{-\frac{1}{2}}w_{i},x{}^{\left(t\right)}\right\rangle 
+\end{align*}
+
+\end_inset
+
+
+\end_layout
+
+\begin_layout Standard
+\begin_inset Formula 
+\begin{align*}
+\frac{\partial L}{\partial w_{i}} & =\sum_{t}\frac{\partial\ell^{\left(t\right)}}{\partial w_{i}}\\
+ & =\sum_{t}\sum_{s}\frac{\partial\ell^{\left(t\right)}}{\partial y_{i}^{\left(s,t\right)}}\frac{\partial y_{i}^{\left(s,t\right)}}{\partial w_{i}}\\
+ & =\sum_{t}\sum_{s}\frac{\partial\ell^{\left(t\right)}}{\partial y_{i}^{\left(s,t\right)}}\left(\frac{1}{\left\Vert w_{i}\right\Vert _{2}}x^{\left(s,t\right)}-w_{i}\left\langle w_{i},w_{i}\right\rangle ^{-\frac{3}{2}}\left\langle w_{i},x^{\left(s,t\right)}\right\rangle \right)\\
+ & =\frac{1}{\left\Vert w_{i}\right\Vert _{2}}\sum_{t}\sum_{s}\frac{\partial\ell^{\left(t\right)}}{\partial y_{i}^{\left(s,t\right)}}\left(x^{\left(s,t\right)}-\frac{w_{i}}{\left\langle w_{i},w_{i}\right\rangle }\left\langle w_{i},x^{\left(s,t\right)}\right\rangle \right)
+\end{align*}
+
+\end_inset
+
+
+\end_layout
+
+\end_body
+\end_document