%&pdflatex
\documentclass[a4paper,11pt]{article}
\usepackage{amsmath,amsfonts}

\usepackage[margin=2.5cm]{geometry}




\begin{document}

%+Title
\title{Statistical Inference and Learning- Ex-4}
\author{Yiqing Wang and Boaz Nadler}
\date{due by Sun. Jan. 10, 2016}
\maketitle
%-Title

\begin{description}
\item{Q1.} 
Let $X$ be a Gaussian distributed random variable with mean $\mu$ and variance $\sigma^2$. We would like to estimate its mean $\mu$ from a single observed realisation. We are interested in a \textit{linear} estimator of the form $aX+b$ and define its risk function under the square loss as
\begin{align*}
R(aX+b, \mu) = \mathbb{E}[(aX+b-\mu)^2].
\end{align*}
Calculate the risk function under the conditions below and show that they are inadmissible.
 
1. when $a > 1$.

2. when $a = 1, b \neq 0$.

3. when $a < 0$.

\end{description}

\begin{description}
\item{Q2.}
Consider the same setup as in Question 1 of Exercise 2: let $(x_i)_{1\leq i\leq n}$ be \(n\) i.i.d. samples from a Gaussian distribution $\mathcal{N}(\mu, \sigma^2)$ with known variance $\sigma^2$ but unknown population mean $\mu \in \mathbb{R}$. We want to estimate \(\mu\) under square error loss. 

Recall that the MLE for \(\mu\) is the sample average, $\hat\mu_{\mbox{\tiny ML}}= \frac{1}{n}\sum_i X_i$. The goal of this exercise is to show that in one dimension ($X$ is a real-valued random variable), the MLE is an admissible estimator.  

1. Assume a Gaussian prior $\mathcal{N}(0, \Delta^2)$ on $\mu$. Derive the associated Bayesian estimator $\hat{\mu}_{\Delta}$ for $\mu$. Show that as $\Delta^2$ tends to infinity, $\hat{\mu}_{\Delta}$ converges to the MLE. Can you explain this  phenomenon ?

2. Calculate the risk functions $R(\hat{\mu}_{\mbox{\tiny ML}}, \mu)$ and $R(\hat{\mu}_{\Delta}, \mu)$.

3. Let $\hat\mu$ be another estimator of $\mu$. Denote
\begin{align*}
\mathbb{E}_{\Delta} R(\hat\mu, \mu) :=
\frac{1}{\sqrt{2\pi\Delta^2}} 
\int_{\mathbb{R}} R(\hat\mu, \mu) e^{-\frac{\mu^2}{2\Delta^2}} d\mu.
\end{align*}
Show that $\mathbb{E}_{\Delta} R(\hat\mu, \mu) \geq \mathbb{E}_{\Delta} R(\hat{\mu}_{\Delta}, \mu)$.

4. Assume that the risk function $R(\hat\mu, \mu)$ is continuous in $\mu$. Show that if the estimator $\hat\mu$ dominates the MLE, that is, 
$R(\hat{\mu}_{\mbox{\tiny ML}}, \mu)\geq R(\hat{\mu},\mu)$ for all $\mu\in\mathbb{R}$, and 
\begin{align*}
\sup_{\mu\in \mathbb{R}}  \left[R(\hat{\mu}_{\mbox{\tiny ML}}, \mu) - R(\hat\mu, \mu)\right] > 0,
\end{align*}
we can find a strictly positive constant $\epsilon$ and a finite (but non-empty) interval $I$ such that
\begin{align*}
\mathbb{E}_{\Delta} R(\hat{\mu}_{\mbox{\tiny ML}}, \mu) - \mathbb{E}_{\Delta} R(\hat{\mu}_{\Delta}, \mu) \geq \frac{\epsilon}{\sqrt{2\pi\Delta^2}} 
\int_{I} e^{-\frac{\mu^2}{2\Delta^2}} d\mu.
\end{align*}

5. Show that such a strictly positive $\epsilon$ cannot exist (Hint: what happens to the quantities on both sides of the inequality as $\Delta$ tends to infinity).
Conclude that $\hat\mu_{\mbox{\tiny ML}}$ is admissible. 
\end{description}


\begin{description}
\item{Q3.} Consider a random vector $(X_1, \cdots, X_d)\in\mathbb{R}^d$ where $d \geq 3$. Assume that its entries satisfy 
\begin{align*}
\forall i \in \{1, \cdots, d\}, \quad X_i = \mu_i  + N_i
\end{align*}
with unknown but fixed $(\mu_1, \cdots, \mu_d) \in \mathbb{R}^d$ and Gaussian distributed i.i.d. $(N_i)_{1\leq i\leq d}$ of zero mean and known variance $\sigma^2$. We want to show that, rather surprisingly and in contrast to the one dimensional case, the MLE here is \textit{inadmissible} under the square error loss.  

1. Suppose that only one observation $\boldsymbol{x} = (x_1, \cdots, x_d)$ is available. Derive the MLE and its risk function under the square error loss.

2. Calculate the risk function for the following estimator (known as the  James-Stein estimator),  
\begin{align*}
\hat{\mu}_{\mbox{\tiny JS}} = \left(1 - \frac{(d-2)\sigma^2}{\|\boldsymbol{x}\|^2} \right) \boldsymbol{x}
\end{align*}
where $\|\boldsymbol{x}\|^2=\sum_{i=1}^d x_i^2$. Conclude that this estimator dominates the MLE. 
\end{description}


\begin{description}
\item{Q4.} Let $(x_i)_{1\leq i\leq n}$ be $n$ i.i.d. real-valued samples from a Gaussian distribution with known variance $\sigma^2$ and unknown expectation $\mu$. We want to test $H_0: \mu = 0$ against $H_1: \mu > 0$.

1. Construct a size $\alpha$ test (type I\ error rate $\alpha$). Is your test optimal? 

2. Describe how, under the alternative hypothesis $H_1$, your test's power varies with $n$ and $\mu$.
\end{description}


\end{document}


