\documentclass[11pt]{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{epsfig}
\usepackage{amsmath}
\usepackage{url}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf MATH888: High-dimensional probability and statistics } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{example}[theorem]{Example}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex
%\renewcommand{\baselinestretch}{1.25}

\begin{document}

\lecture{5 --- September 17, 2021}{Fall 2021}{Sebastien Roch, UW-Madison}{Zhuoyan Xu}

\section{Overview}

In the previous lecture we introduced some basic notations in point estimation and the definition of unbiasedness, mean squared error and Cram\'{e}r-Rao Lower Bound. We introduced the main theorem of CR lower bound and talked about the intuition behind it.

In this lecture we mainly prove the Cram\'{e}r-Rao Lower Bound and illustrate the property of it.


\section{ Cram\'{e}r-Rao Lower Bound main theory}
In this section we consider the special discrete case of random variables. For general results, see \emph{Theorem 6.6} in Lehmann and Casella \cite{lehmcass}.
We redefine the following setting:

\begin{enumerate}
    \item The \emph{sample space} $\mathcal{X}$ is finite, i.e. $|\mathcal{X}| < \infty$. 
    \item The \emph{parameter space} $\Theta \subseteq \mathbb{R}$ is an open set.
    \item The \emph{family of distribution}
    $$
        \mathcal{P} = \{ \ p( \ \cdot \ ; \theta) \ : \ \theta \in \Theta \ \}
    $$
    satisfies $p(x, \theta) > 0$ and $\frac{\partial}{\partial \theta}p(x, \theta)$ exists for $\forall x \in \mathcal{X}$ and $\forall \theta \in \Theta$.
\end{enumerate}

\begin{theorem}
Let $\hat{\vartheta}^{(n)}, \ n \in \mathbb{N}$ be an unbiased estimators of $g(\theta)$ i.e. $\mathbb{E}[\hat{\vartheta}^{(n)}] = g(\theta)$ for all $n \in \mathbb{N}$, where $g : \mathbb{R} \rightarrow \mathbb{R}$ is a continuously differentiable function, then: 
$$
   \text{Var}\left[{\hat{\vartheta}^{(n)}} \right]
        \geq 
    \frac{[g'(\theta)]^2}{n \mathbb{E}\left[ \frac{\partial}{\partial \theta} \log p(X_1 ; \theta) \right]^2}
$$
where $\mathbb{E}\left[ \frac{\partial}{\partial \theta} \log p(X_1 ; \theta) \right]^2 =: I(\theta)$ is the Fisher Information for $\theta$. 
\end{theorem}

\begin{remark}
The Fisher information 
$$
    I(\theta) := \mathbb{E} \left[ \left(\frac{\partial}{\partial \theta} \log p(x ; \theta) \right)^2 \right] = \mathbb{E} \left[ \left( \frac{\frac{\partial }{\partial \theta}p(x ; \theta)}{p(x ; \theta)} \right)^2 \right]
$$

quantifies the expected relative rate of change of the likelihood with respect to a small perturbation in $\theta$. It can be roughly seen as the relative ``derivative'' of pdf with respect to $\theta$. A larger fisher information indicates the steep change in log-likelihood function in changing of parameter $\theta$. It makes it easier to distinguish two likelihood function with different values of $\theta$. In this sense, $I(\theta)$ captures information about the parameter $\theta$. A larger Fisher information value also leads to a lower Cram\'{e}r-Rao bound.
\end{remark}


To illustrate the theorem of Cram\'{e}r-Rao bound, we consider following example.

\begin{example}
Suppose we have true parameter $\theta^*$ for some distribution $p$, define $\hat{\vartheta}^{(n)} = \theta^*$. Then we have 
$$
\mathbb{E}[\hat{\vartheta}^{(n)}] = \theta^*  \text{and} ~ \text{Var}[\hat{\vartheta}^{(n)}] = 0
$$
\end{example}

The estimator in this example has lower variance than CR lower bound, but it's not an unbiased estimator indeed. Recall the definition of unbiasedness: An estimator $\hat{\theta}$ is said to be unbiased if $\mathrm{bias}(\hat{\theta}, \theta) = 0$ for all $P \in \mathcal{P}$. An unbiased estimator must have zero bias for \emph{all} possible distributions. In this example, the constant estimator $\hat{\vartheta}^{(n)}$ has zero bias for any $P$ with $\theta(P) = \theta^*$, but this is not an unbiased estimator for any $P$ with a different value of $\theta$.

Now we begin our proof of the main theorem.
\begin{proof}
Without abuse of notation, we define $\boldsymbol{X} = (X_1,\ldots, X_n)$ as random vector where $X_1,\ldots, X_n \overset{\text{iid}}{\sim} p(X,\theta)$, $\boldsymbol{x} = (x_1,\ldots,x_n)$ as the realization of $\boldsymbol{X}$.

We have $p^{(n)}(\boldsymbol{X},\theta) = \prod_{i = 1}^n p(X_i,\theta)$. Recall 
$$\text{Cov}(X,Y) = \mathbb{E}[(X-\mathbb{E}X)(Y-\mathbb{E}Y)] = \mathbb{E}XY - \mathbb{E}X\mathbb{E}Y)$$

for any function $\psi(\boldsymbol{X},\theta)$, by Cauchy-Schwarz inequality, we have
\begin{align*} 
    |\text{Cov}(\hat{\vartheta},\psi(\boldsymbol{X},\theta))|^2 &\le \text{Var}(\hat{\vartheta})\text{Var} (\psi(\boldsymbol{X},\theta)) \\
    \text{Var}(\hat{\vartheta}) &\ge \frac{|\text{Cov}(\hat{\vartheta},\psi(\boldsymbol{X},\theta))|^2}{\text{Var} (\psi(\boldsymbol{X},\theta)))}
\end{align*}

We choose $\psi(\boldsymbol{X},\theta) = \frac{\partial}{\partial \theta} \log p^{(n)}(\boldsymbol{X},\theta)$, then 
\begin{align*}
    \mathbb{E}(\psi) &= \sum_{x \in \mathcal{X}}  p^{(n)}(\boldsymbol{X},\theta) \frac{\partial}{\partial \theta} \log p^{(n)}(\boldsymbol{X},\theta) \\
    &= \sum_{x \in \mathcal{X}}  p^{(n)}(\boldsymbol{X},\theta) \frac{\frac{\partial}{\partial \theta} p^{(n)}(\boldsymbol{X},\theta)}{p^{(n)}(\boldsymbol{X},\theta)} \\
    &= \frac{\partial}{\partial \theta} \left( \sum_{x \in \mathcal{X}}  p^{(n)}(\boldsymbol{X},\theta) \right) \\
    &= 0
\end{align*}
Since $X_1,\ldots, X_n \overset{\text{iid}}{\sim} p(X_1,\theta)$, we have:
\begin{align*}
    \text{Var} (\psi) &= \text{Var}\left(\frac{\partial}{\partial \theta} \log p^{(n)}(\boldsymbol{X},\theta)  \right)\\
    &= \text{Var}\left(\sum_{i=1}^n \frac{\partial}{\partial \theta} \log p(X,\theta)  \right) \\
    &= n  \text{Var}\left(\frac{\partial}{\partial \theta} \log p(X,\theta)  \right) \\
    &= n  \mathbb{E}\left[\left(\frac{\partial}{\partial \theta} \log p(X,\theta)  \right)^2 \right]
\end{align*}

Consider covariance, we have:
\begin{align*}
    \text{Cov} (\hat{\vartheta},\psi) &= \text{Cov} \left[\hat{\vartheta}, \frac{\partial}{\partial \theta} \log p^{(n)}(\boldsymbol{X},\theta)\right] \\
    &= \mathbb{E} \left[\hat{\vartheta} \cdot \frac{\partial}{\partial \theta} \log p^{(n)}(\boldsymbol{X},\theta)\right] \\
    &= \sum_{x \in \mathcal{X}} \left[ p^{(n)}(\boldsymbol{X},\theta) \cdot \hat{\vartheta} \cdot \frac{\frac{\partial}{\partial \theta} p^{(n)}(\boldsymbol{X},\theta)}{p^{(n)}(\boldsymbol{X},\theta)} \right] \\
    &= \frac{\partial}{\partial \theta} \left[ \sum_{x \in \mathcal{X}}  p^{(n)}(\boldsymbol{X},\theta) \cdot \hat{\vartheta} \right] \\
    &= \frac{\partial}{\partial \theta} \left[ g(\theta) \right] \\
    &= g'(\theta)
\end{align*}

plug in previous equation, we have desired result:

$$
   \text{Var}\left[{\hat{\vartheta}^{(n)}} \right]
        \geq 
    \frac{[g'(\theta)]^2}{n \mathbb{E}\left[ \frac{\partial}{\partial \theta} \log p(X_1 , \theta) \right]^2}
$$
\end{proof}

\begin{example}
Consider Bernoulli distribution:
\begin{itemize}
     \item $\mathcal{X}$ is  = \{0,1\}  $\Theta$ = (0,1)
     \item $p(X,\theta) = \theta^X(1-\theta)^{1-X}  > 0$ for $\forall x \in \mathcal{X}$ and $\forall \theta \in \Theta$.
     \item $\frac{\partial}{\partial \theta}p(x, \theta) = \frac{X}{\theta} - \frac{1-X}{1-\theta}$  exists for $\forall x \in \mathcal{X}$ and $\forall \theta \in \Theta$.
     \item $g(\theta) = \theta$.
\end{itemize}
We have Fisher information 
\begin{align*}
    I(\theta) &= \mathbb{E} \left[ \left(\frac{\partial}{\partial \theta} \log p(x ; \theta) \right)^2 \right]  \\
    &= \mathbb{E} \left[ \left(\frac{\partial}{\partial \theta} \left[ X\log(\theta) + (1-X) \log(1-\theta) \right] \right)^2 \right] \\
    &= \mathbb{E} \left[ \left(\frac{X}{\theta} - \frac{1-X}{1-\theta}\right)^2 \right]  \\
    &= \mathbb{E} \left[ \frac{(X-\theta)^2}{(\theta(1-\theta))^2} \right] \\
    &= \frac{1}{\theta(1-\theta)}
\end{align*}
We have $$
\text{Var}\left[{\hat{\vartheta}^{(n)}} \right]
        \geq \frac{\theta(1-\theta)}{n}
$$
Consider empirical mean estimator $\hat{\theta}^{(n)} = \frac{1}{n}\sum_{i=1}^n X_i$ , we have 
\begin{align*}
    \mathbb{E}\hat{\theta}^{(n)} &= \theta \\
    \text{Var} (\hat{\theta}^{(n)}) &= \frac{\theta(1-\theta)}{n}
\end{align*}
achieves the CR lower bound.
\end{example}

\begin{remark}
We may not always find such unbiased estimator, consider $g(\theta) = 1/\theta$ in previous example, the unbiased estimator does not exist, see details on \cite{link}.
\end{remark}





%\bibliography{mybib}
\bibliographystyle{alpha}

\begin{thebibliography}{77}
\bibitem{lehmcass}
Lehmann, E. L. and Casella, G. (1998). \emph{Theory of point estimation.} Springer.

\bibitem{link}
Uon-existence of unbiased estimator: \url{https://math.stackexchange.com/questions/681638/for-the-binomial-distribution-why-does-no-unbiased-estimator-exist-for-1-p}



\end{thebibliography}

\end{document}