$$ \newcommand{sca}[1]{\langle #1 \rangle} \newcommand{\scalong}[1]{(#1_1,\dots,#1_k)} \newcommand{\red}[1]{\textcolor{OrangeRed}{#1}} \newcommand{\blue}[1]{\textcolor{blue}{#1}} \newcommand{\green}[1]{\textcolor{OliveGreen}{#1}} \newcommand{\orange}[1]{\textcolor{orange}{#1}} \newcommand{\purple}[1]{\textcolor{purple}{#1}} \newcommand{\gray}[1]{\textcolor{gray}{#1}} \newcommand{\teal}[1]{\textcolor{teal}{#1}} \newcommand{\gold}[1]{\textcolor{gold}{#1}} \newcommand{\bluea}[1]{\textcolor{RoyalBlue}{#1}} \newcommand{\reda}[1]{\textcolor{Red}{#1}} \newcommand{\redb}[1]{\textcolor{RubineRed}{#1}} \newcommand{\greena}[1]{\textcolor{LimeGreen}{#1}} \newcommand{\golden}[1]{\textcolor{GoldenRod}{#1}} \newcommand{\filter}[1]{\green{#1}} \newcommand{\param}[1]{\purple{#1}} \newcommand{\state}[1]{\blue{#1}} \newcommand{\statex}[1]{\bluea{#1}} \newcommand{\stateu}[1]{\greena{#1}} \newcommand{\statez}[1]{\golden{#1}} \newcommand{\input}[1]{\gray{#1}} \newcommand{\gain}[1]{\red{#1}} \newcommand{\gainx}[1]{\reda{#1}} \newcommand{\trust}[1]{\teal{#1}} \newcommand{\schedule}[1]{\gold{#1}} $$ $$ \newcommand{\trobjsca}{\mathcal{D}[t,i]} \newcommand{\trobjmat}{\mathcal{D}[t]} \newcommand{\Tr}{\mathrm{Tr}} \newcommand{\step}{\Delta[t+1, i] = -\alpha[t,i]\,\mathbf{g}[t,i]} \newcommand{\stepv}{\Delta[t+1, i] = -\alpha[t,i]\,\mathbf{v}[t,i]} \newcommand{\matstep}{\Delta[t+1] = -\alpha[t]\,\mathbf{g}[t]} \newcommand{\matstepv}{\Delta[t+1] = -\alpha[t]\,\mathbf{v}[t]} \newcommand{\ngrad}{\bar{\mathbf{g}}[t,i]} \newcommand{\ngradv}{\bar{\mathbf{v}}[t,i]} \newcommand{\ngradsq}{\bar{\mathbf{g}}^2[t,i]} \newcommand{\ngradvsq}{\bar{\mathbf{v}}^2[t,i]} \newcommand{\nmatgrad}{\bar{\mathbf{g}}[t]} \newcommand{\nmatgradv}{\bar{\mathbf{v}}[t]} \newcommand{\fof}{\mathbb{H}_{\beta,\,\gamma}} \newcommand{\expg}{\mathbb{E}\big[\mathbf{g}[t,i]\big]} \newcommand{\expv}{\mathbb{E}\big[\mathbf{v}[t,i]\big]} $$ $$ \newcommand{\stepmom}{\mathbb{E}\big[{\Delta}^2[t+1, i]\big]} \newcommand{\matstepmom}{\mathbb{E}\big[\Tr\big(\Delta^\intercal[t+1]\,\Delta[t+1]\big]} \newcommand{\gmatstepmom}{\mathbb{E}\big[\Tr\big(\left<{\Delta[t+1],\Delta[t+1]}\right>\big)\big]} $$ $$ \newcommand{\stepcorrng}{\mathbb{E}\big[\Delta[t+1, i]\,\ngrad\big]} \newcommand{\stepcorrngv}{\mathbb{E}\big[\Delta[t+1, i]\,\ngradv\big]} \newcommand{\matstepcorrng}{\mathbb{E}\big[\Tr\big(\Delta^\intercal[t+1]\,\nmatgrad \big) \big]} \newcommand{\matstepcorrngv}{\mathbb{E}\big[\Tr\big(\Delta^\intercal[t+1]\,\nmatgradv \big) \big]} \newcommand{\gmatstepcorrng}{\mathbb{E}\big[\Tr\big(\left\langle{\Delta[t+1],\nmatgrad}\right\rangle \big)\big]} \newcommand{\gmatstepcorrngv}{\mathbb{E}\big[\Tr\big(\left\langle{\Delta[t+1],\nmatgradv}\right\rangle \big)\big]} $$ $$ \newcommand{\stepcorru}{\mathbb{E}\big[\Delta[t+1, i]\,\mathbf{u}[t,i]\big]} \newcommand{\matstepcorru}{\mathbb{E}\big[\Delta^\intercal[t+1]\,\mathbf{u}[t]\big]} $$ $$ \newcommand{\numngradcorr}{\mathbb{E}\big[\ngradsq\big]} \newcommand{\numwgcorr}{\mathbb{E}\big[\mathbf{w}[t,i]\,\ngrad\big]} \newcommand{\numngradvcorr}{\mathbb{E}\big[\ngradvsq\big]} \newcommand{\numwvcorr}{\mathbb{E}\big[\mathbf{w}[t,i]\,\ngradv\big]} \newcommand{\dengmom}{\mathbb{E}\big[\mathbf{g}^2[t,i]\big]} \newcommand{\dengmomv}{\mathbb{E}\big[\mathbf{v}^2[t,i]\big]} \newcommand{\matdengmom}{\mathbb{E}\big[\mathbf{g}[t]\mathbf{g}^\intercal[t]\big]} \newcommand{\matdengmomv}{\mathbb{E}\big[\mathbf{v}[t]\mathbf{v}^\intercal[t]\big]} \newcommand{\dengmomsqrt}{\sqrt{\mathbb{E}\big[\mathbf{g}^2[t,i]\big]}} \newcommand{\matdengmomsqrt}{\mathbb{E}\big[\mathbf{g}[t]\mathbf{g}^\intercal[t]\big]^{\text{-}\frac{1}{2}}} \newcommand{\matdenvmomsqrt}{\mathbb{E}\big[\mathbf{v}[t]\mathbf{v}^\intercal[t]\big]^{\text{-}\frac{1}{2}}} \newcommand{\matngradcorr}{\mathbb{E}\big[\nmatgrad\nmatgrad^{\intercal}\big]} \newcommand{\matngradvcorr}{\mathbb{E}\big[\nmatgradv\nmatgradv^{\intercal}\big]} \newcommand{\matwngradcorr}{\mathbb{E}\big[\mathbf{w}[t]\nmatgrad^{\intercal}\big]} \newcommand{\matwngradvcorr}{\mathbb{E}\big[\mathbf{w}[t]\nmatgradv^{\intercal}\big]} $$
Skip to main content Signal processing meets deep learning optimization
A notebook for my current research. From control theory and signal processing foundations to modern optimization methods for deep learning.
About Me
AutoSGM View it on GitHub
PID View it on GitHub