\classheader{2012-12-04}

Today, we'll prove the Central Limit Theorem.

Let $g$ be a Schwartz function, and let $\widehat{g}$ be its Fourier transform, i.e.
\[\widehat{g}(y)=\int_{-\infty}^\infty e^{-ixy}\,g(x)\,dx.\]
Recall that the inverse Fourier transform can be obtained as
\[g(y)=\frac{1}{2\pi}\int_{-\infty}^\infty e^{ixy}\,\widehat{g}(x)\,dx.\]
The function $e^{-x^2/2}$ acts (almost) as an identity:
\[\widehat{e^{-x^2/2}}=\sqrt{2\pi}\,e^{-y^2/2}.\]
We will also need the fact that
\[\int f\widehat{g}=\int f(x)\int e^{-ixy}\,g(y)\,dy\,dx=\int g(y)\underbrace{\int e^{-ixy} f(x)\,dx}_{\widehat{f}}\,dy=\int g\widehat{f}.\]
\begin{definition}
The characteristic function $\varphi$ of $X$ is defined by $\varphi(t)= E(e^{iXt})$.
In general, this is a complex-valued function.
\end{definition}
Observe that $\varphi(0)=1$, and that for any $t$, we have $|\varphi(t)|\leq 1$. Also, $\varphi$ is continuous because the dominated convergence theorem implies that
\[\lim_{s\to t}\varphi(s)=\lim_{s\to t}E(e^{iXs})=\varphi(t).\]
If $X$ has density $f$, then
\[\varphi(t)=\int e^{ixt}\,f(x)\,dx=\widehat{f}(-t).\]
It turns out that if $X$ and $Y$ have the same characteristic function, then they have the same distribution. To prove this, we need some sort of ``inversion'' (when $X$ and $Y$ have densities, we could apply actual Fourier inversion, but we need something that works in general).
If $X_1,\ldots,X_n$ are independent random variables, then
\[\varphi_{X_1+\cdots+X_n}(t)=E(e^{i(X_1+\cdots+X_n)t})=E(e^{iX_1t}\cdots e^{iX_nt})\underset{\mathclap{\substack{\uparrow\\\text{independence}}}}{=}\varphi_{X_1}(t)\cdots\varphi_{X_n}(t).\]
If $X$ has a normal distribution with mean 0 and variance 1, then
\[\varphi(t)=\int e^{ixt}\frac{1}{\sqrt{2\pi}}\,e^{-x^2/2}\,dx=e^{-t^2/2}\]
because $e^{ixt}e^{-x^2/2}=e^{-(x-t)^2/2}e^{-t^2/2}$.

For a random variable $X$ and any $a,b$,
\[\varphi_{aX+b}=e^{ibt}\varphi_X(at),\]
so the characteristic function of a normal distribution with mean $\mu$ and variance $\sigma^2$ is $e^{i\mu t}e^{-\sigma^2t^2/2}$.

If $E(|X|)<\infty$, then $\varphi'(0)=iE(X)$. More generally, if the higher moments of $X$ are finite, i.e. if we have $E(|X|^k)<\infty$ for some $k$, then $\varphi^{(j)}(0)=i^jE(X^j)$ for all $1\leq j\leq k$.

\begin{lemma}
Let $X_1,\ldots,X_n$ be i.i.d. random variables, with mean $\mu$ and variance $\sigma^2$. Let
\[Y_n=\frac{X_1+\cdots+X_n-n\mu}{\sqrt{\sigma^2 n}}.\]
Then $\varphi_{Y_n}(t)\to e^{-t^2/2}$ (the normal distribution) as $n\to\infty$.
\end{lemma}
\begin{proof}
We can write
\[\varphi(t)=1+\varphi'(0)t+\frac{1}{2}\varphi''(0)t^2+\epsilon_tt^2,\]
where $\epsilon_t\to 0$ as $t\to 0$. We can assume WLOG that $\mu=0$ and $\sigma=1$, so we get
\[\varphi(t)=1-\frac{t^2}{2}+\epsilon_tt^2.\]
We have that
\[\varphi_{Y_n}(t)=\left[\varphi\left(\frac{t}{\sqrt{n}}\right)\right]^n,\]
so
\[\lim_{n\to\infty}\varphi_{Y_n}(t)=\lim_{n\to\infty}\left(1-\frac{t^2}{2n}+\epsilon_{t/\sqrt{n}}\frac{t^2}{n}\right)^n=\left(1-\frac{t^2}{2n}+\frac{\delta_n}{n}\right)^n,\]
where $\delta_n\to 0$ as $n\to\infty$ (because $\epsilon_{t/\sqrt{n}}\to 0$ as $n\to\infty$).
Taking logarithms,
\[\lim_{n\to\infty}\log(\varphi_{Y_n}(t))=\lim_{n\to\infty}n\underbrace{\log\left(1-\frac{t^2}{2n}+\frac{\delta_n}{n}\right)}_{-\frac{t^2}{2n}+\frac{\rho_n}{n}}\]
where $\rho_n\to 0$ as $n\to\infty$, so 
\[\lim_{n\to\infty}\log(\varphi_{Y_n}(t))=-\frac{t^2}{2},\]
and hence $\varphi_{Y_n}(t)=e^{-t^2/2}$.
\end{proof}
\begin{theorem}[Central limit theorem]
Let $X_1,\ldots,X_n$ be i.i.d. random variables with mean $\mu$ and variance $\sigma^2$. Then
\[\lim_{n\to\infty}P\left(a\leq\frac{X_1+\cdots+X_n-\mu n}{\sigma\sqrt{n}}\leq b\right)=\frac{1}{\sqrt{2\pi}}\int_a^b e^{-x^2/2}\,dx.\]
\end{theorem}
\begin{proof}
We need to prove that if $\mu_n$ is a sequence of distributions with $\varphi_n(t)\to e^{-t^2/2}$, then $\lim\limits_{n\to\infty}\mu_n([a,b])=\frac{1}{\sqrt{2\pi}}\int_a^be^{-x^2/2}\,dx$.

Our approach will be to approximate $\chi_{[a,b]}$ with Schwartz functions. We can find a Schwartz function $g=g_\epsilon$ with $0\leq g\leq 1$, $g(x)=1$ on $[a,b]$, and $g(x)=0$ on $(-\infty,a-\epsilon)\cup (b+\epsilon,\infty)$. We claim that
\[\lim_{n\to\infty}\int g(x)\,d\mu_n(x)=\frac{1}{\sqrt{2\pi}}\int g(x)e^{-x^2/2}.\]
Note that, if this is true, then as $\epsilon\to 0$, we get
\[\mu_n([a,b])=\frac{1}{\sqrt{2\pi}}\int_a^b e^{-x^2/2}\,dx\]
as desired. Now, we need to prove this claim.

Observe that
\[\int_{-\infty}^\infty g(x)\,d\mu_n(x)=\int\frac{1}{2\pi}\int e^{-xy}\,\widehat{g}(y)\,dy\,d\mu_n(x)\]
where $\widehat{g}(y)=\int e^{-ixy}\,g(x)\,dx$, so
\[\int_{-\infty}^\infty g(x)\,d\mu_n(x)=\frac{1}{2\pi}\int\left(\int e^{ixy}\,d\mu_n(x)\right)\widehat{g}(y)\,dy=\frac{1}{2\pi}\int \varphi_n(y)\widehat{g}(y)\,dy.\]
Taking the limit as $n\to\infty$,
\[\lim_{n\to\infty}\int_{-\infty}^\infty g(x)\,d\mu_n(x)=\frac{1}{2\pi}\int e^{-y^2/2}\,\widehat{g}(y)\,dy.\]
Now use that $\int f\widehat{g}=\int \widehat{f}g$ and $\widehat{e^{-y^2/2}}=\sqrt{2\pi} e^{-y^2/2}$.
\end{proof}

Let $X$ be a random variable with characteristic function $\varphi$ and distribution $\mu$. We want to express $\varphi$ in terms of $\mu$ somehow. We claim that
\[\mu([a,b])=\lim_{T\to\infty}\frac{1}{2\pi}\int_{-T}^T\frac{e^{-iya}-e^{-iyb}}{iy}\,\varphi(y)\,dy,\]
as long as $a,b$ are points where $F$ (the distribution function) is continuous, or equivalently, as long as $a,b$ are not atoms for the measure $\mu$. The integrand in the above expression can tend to both $-\infty$ and $\infty$, so the integral over all of $\R$ may not exist, but this limit will exist because there will be cancellations. The denominator of $iy$ doesn't lead to infinite values, because
\[\left|\varphi(y)\cdot\frac{e^{-iya}-e^{-iyb}}{iy}\right|\leq\left|\frac{e^{-iya}-e^{-iyb}}{iy}\right|\leq b-a.\]
\begin{proof}
Note that
\begin{align*}
\frac{1}{2\pi}\int_{-T}^T\frac{e^{-iya}-e^{-iyb}}{iy}\,\varphi(y)\,dy&=\frac{1}{2\pi}\int_{-T}^T\frac{e^{-iya}-e^{-iyb}}{iy}\int_{-\infty}^\infty e^{ixy}\,d\mu(x)\,dy\\
&=\frac{1}{2\pi}\int_{-\infty}^\infty\int_{-T}^T\frac{e^{iy(x-a)}-e^{iy(x-b)}}{iy}\,dy\,d\mu(x)
\end{align*}
For any $c>0$,
\[\int_{-T}^T\frac{e^{icx}}{ix}\,dx=2\int_0^{cT}\frac{\sin(x)}{x}\,dx\]
(we can see this by splitting the exponential into sine and cosine). Also, recall that
\[\lim_{T\to\infty}\int_0^T\frac{\sin(x)}{x}=\frac{\pi}{2}.\]
Thus, if $x-a$ and $x-b$ have the same sign, then \[\lim_{T\to\infty}\int_{-T}^T\frac{e^{iy(x-a)}-e^{iy(x-b)}}{iy}\,dy=0,\] and if $x-a>0$ and $x-b<0$, we get that it equals $4\cdot \frac{\pi}{2}=2\pi$. 
\end{proof}