
%% bare_jrnl.tex
%% V1.4
%% 2012/12/27
%% by Michael Shell
%% see http://www.michaelshell.org/
%% for current contact information.
%%
%% This is a skeleton file demonstrating the use of IEEEtran.cls
%% (requires IEEEtran.cls version 1.8 or later) with an IEEE journal paper.
%%
%% Support sites:
%% http://www.michaelshell.org/tex/ieeetran/
%% http://www.ctan.org/tex-archive/macros/latex/contrib/IEEEtran/
%% and
%% http://www.ieee.org/



% *** Authors should verify (and, if needed, correct) their LaTeX system  ***
% *** with the testflow diagnostic prior to trusting their LaTeX platform ***
% *** with production work. IEEE's font choices can trigger bugs that do  ***
% *** not appear when using other class files.                            ***
% The testflow support page is at:
% http://www.michaelshell.org/tex/testflow/


%%*************************************************************************
%% Legal Notice:
%% This code is offered as-is without any warranty either expressed or
%% implied; without even the implied warranty of MERCHANTABILITY or
%% FITNESS FOR A PARTICULAR PURPOSE! 
%% User assumes all risk.
%% In no event shall IEEE or any contributor to this code be liable for
%% any damages or losses, including, but not limited to, incidental,
%% consequential, or any other damages, resulting from the use or misuse
%% of any information contained here.
%%
%% All comments are the opinions of their respective authors and are not
%% necessarily endorsed by the IEEE.
%%
%% This work is distributed under the LaTeX Project Public License (LPPL)
%% ( http://www.latex-project.org/ ) version 1.3, and may be freely used,
%% distributed and modified. A copy of the LPPL, version 1.3, is included
%% in the base LaTeX documentation of all distributions of LaTeX released
%% 2003/12/01 or later.
%% Retain all contribution notices and credits.
%% ** Modified files should be clearly indicated as such, including  **
%% ** renaming them and changing author support contact information. **
%%
%% File list of work: IEEEtran.cls, IEEEtran_HOWTO.pdf, bare_adv.tex,
%%                    bare_conf.tex, bare_jrnl.tex, bare_jrnl_compsoc.tex,
%%                    bare_jrnl_transmag.tex
%%*************************************************************************

% Note that the a4paper option is mainly intended so that authors in
% countries using A4 can easily print to A4 and see how their papers will
% look in print - the typesetting of the document will not typically be
% affected with changes in paper size (but the bottom and side margins will).
% Use the testflow package mentioned above to verify correct handling of
% both paper sizes by the user's LaTeX system.
%
% Also note that the "draftcls" or "draftclsnofoot", not "draft", option
% should be used if it is desired that the figures are to be displayed in
% draft mode.
%
\documentclass[journal]{IEEEtran}
%
% If IEEEtran.cls has not been installed into the LaTeX system files,
% manually specify the path to it like:
% \documentclass[journal]{../sty/IEEEtran}





% Some very useful LaTeX packages include:
% (uncomment the ones you want to load)


% *** MISC UTILITY PACKAGES ***
%
%\usepackage{ifpdf}
% Heiko Oberdiek's ifpdf.sty is very useful if you need conditional
% compilation based on whether the output is pdf or dvi.
% usage:
% \ifpdf
%   % pdf code
% \else
%   % dvi code
% \fi
% The latest version of ifpdf.sty can be obtained from:
% http://www.ctan.org/tex-archive/macros/latex/contrib/oberdiek/
% Also, note that IEEEtran.cls V1.7 and later provides a builtin
% \ifCLASSINFOpdf conditional that works the same way.
% When switching from latex to pdflatex and vice-versa, the compiler may
% have to be run twice to clear warning/error messages.

\usepackage[utf8]{inputenc}





% *** CITATION PACKAGES ***
%
% \usepackage{cite}
% cite.sty was written by Donald Arseneau
% V1.6 and later of IEEEtran pre-defines the format of the cite.sty package
% \cite{} output to follow that of IEEE. Loading the cite package will
% result in citation numbers being automatically sorted and properly
% "compressed/ranged". e.g., [1], [9], [2], [7], [5], [6] without using
% cite.sty will become [1], [2], [5]--[7], [9] using cite.sty. cite.sty's
% \cite will automatically add leading space, if needed. Use cite.sty's
% noadjust option (cite.sty V3.8 and later) if you want to turn this off
% such as if a citation ever needs to be enclosed in parenthesis.
% cite.sty is already installed on most LaTeX systems. Be sure and use
% version 4.0 (2003-05-27) and later if using hyperref.sty. cite.sty does
% not currently provide for hyperlinked citations.
% The latest version can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/cite/
% The documentation is contained in the cite.sty file itself.






% *** GRAPHICS RELATED PACKAGES ***
%
\ifCLASSINFOpdf
  \usepackage[pdftex]{graphicx}
  % declare the path(s) where your graphic files are
  % \graphicspath{{../pdf/}{../jpeg/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
  \DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\else
  % or other class option (dvipsone, dvipdf, if not using dvips). graphicx
  % will default to the driver specified in the system graphics.cfg if no
  % driver is specified.
  % \usepackage[dvips]{graphicx}
  % declare the path(s) where your graphic files are
  % \graphicspath{{../eps/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
  % \DeclareGraphicsExtensions{.eps}
\fi
% graphicx was written by David Carlisle and Sebastian Rahtz. It is
% required if you want graphics, photos, etc. graphicx.sty is already
% installed on most LaTeX systems. The latest version and documentation
% can be obtained at: 
% http://www.ctan.org/tex-archive/macros/latex/required/graphics/
% Another good source of documentation is "Using Imported Graphics in
% LaTeX2e" by Keith Reckdahl which can be found at:
% http://www.ctan.org/tex-archive/info/epslatex/
%
% latex, and pdflatex in dvi mode, support graphics in encapsulated
% postscript (.eps) format. pdflatex in pdf mode supports graphics
% in .pdf, .jpeg, .png and .mps (metapost) formats. Users should ensure
% that all non-photo figures use a vector format (.eps, .pdf, .mps) and
% not a bitmapped formats (.jpeg, .png). IEEE frowns on bitmapped formats
% which can result in "jaggedy"/blurry rendering of lines and letters as
% well as large increases in file sizes.
%
% You can find documentation about the pdfTeX application at:
% http://www.tug.org/applications/pdftex





% *** MATH PACKAGES ***
%
\usepackage[cmex10]{amsmath}
\usepackage{amssymb, amsthm}
\newcommand{\argmax}{\operatornamewithlimits{argmax}}
\newcommand{\argmin}{\operatornamewithlimits{argmin}}
\newcommand{\var}{\operatornamewithlimits{var}}
% A popular package from the American Mathematical Society that provides
% many useful and powerful commands for dealing with mathematics. If using
% it, be sure to load this package with the cmex10 option to ensure that
% only type 1 fonts will utilized at all point sizes. Without this option,
% it is possible that some math symbols, particularly those within
% footnotes, will be rendered in bitmap form which will result in a
% document that can not be IEEE Xplore compliant!
%
% Also, note that the amsmath package sets \interdisplaylinepenalty to 10000
% thus preventing page breaks from occurring within multiline equations. Use:
%\interdisplaylinepenalty=2500
% after loading amsmath to restore such page breaks as IEEEtran.cls normally
% does. amsmath.sty is already installed on most LaTeX systems. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/amslatex/math/





% *** SPECIALIZED LIST PACKAGES ***
%
\usepackage{algorithmic, algorithm}
% algorithmic.sty was written by Peter Williams and Rogerio Brito.
% This package provides an algorithmic environment fo describing algorithms.
% You can use the algorithmic environment in-text or within a figure
% environment to provide for a floating algorithm. Do NOT use the algorithm
% floating environment provided by algorithm.sty (by the same authors) or
% algorithm2e.sty (by Christophe Fiorio) as IEEE does not use dedicated
% algorithm float types and packages that provide these will not provide
% correct IEEE style captions. The latest version and documentation of
% algorithmic.sty can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/algorithms/
% There is also a support site at:
% http://algorithms.berlios.de/index.html
% Also of interest may be the (relatively newer and more customizable)
% algorithmicx.sty package by Szasz Janos:
% http://www.ctan.org/tex-archive/macros/latex/contrib/algorithmicx/




% *** ALIGNMENT PACKAGES ***
%
%\usepackage{array}
% Frank Mittelbach's and David Carlisle's array.sty patches and improves
% the standard LaTeX2e array and tabular environments to provide better
% appearance and additional user controls. As the default LaTeX2e table
% generation code is lacking to the point of almost being broken with
% respect to the quality of the end results, all users are strongly
% advised to use an enhanced (at the very least that provided by array.sty)
% set of table tools. array.sty is already installed on most systems. The
% latest version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/tools/


% IEEEtran contains the IEEEeqnarray family of commands that can be used to
% generate multiline equations as well as matrices, tables, etc., of high
% quality.




% *** SUBFIGURE PACKAGES ***
%\ifCLASSOPTIONcompsoc
%  \usepackage[caption=false,font=normalsize,labelfont=sf,textfont=sf]{subfig}
%\else
  \usepackage[caption=false,font=footnotesize]{subfig}
%\fi
% subfig.sty, written by Steven Douglas Cochran, is the modern replacement
% for subfigure.sty, the latter of which is no longer maintained and is
% incompatible with some LaTeX packages including fixltx2e. However,
% subfig.sty requires and automatically loads Axel Sommerfeldt's caption.sty
% which will override IEEEtran.cls' handling of captions and this will result
% in non-IEEE style figure/table captions. To prevent this problem, be sure
% and invoke subfig.sty's "caption=false" package option (available since
% subfig.sty version 1.3, 2005/06/28) as this is will preserve IEEEtran.cls
% handling of captions.
% Note that the Computer Society format requires a larger sans serif font
% than the serif footnote size font used in traditional IEEE formatting
% and thus the need to invoke different subfig.sty package options depending
% on whether compsoc mode has been enabled.
%
% The latest version and documentation of subfig.sty can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/subfig/




% *** FLOAT PACKAGES ***
%
% \usepackage{fixltx2e}
% fixltx2e, the successor to the earlier fix2col.sty, was written by
% Frank Mittelbach and David Carlisle. This package corrects a few problems
% in the LaTeX2e kernel, the most notable of which is that in current
% LaTeX2e releases, the ordering of single and double column floats is not
% guaranteed to be preserved. Thus, an unpatched LaTeX2e can allow a
% single column figure to be placed prior to an earlier double column
% figure. The latest version and documentation can be found at:
% http://www.ctan.org/tex-archive/macros/latex/base/


\usepackage{stfloats}
% stfloats.sty was written by Sigitas Tolusis. This package gives LaTeX2e
% the ability to do double column floats at the bottom of the page as well
% as the top. (e.g., "\begin{figure*}[!b]" is not normally possible in
% LaTeX2e). It also provides a command:
%\fnbelowfloat
% to enable the placement of footnotes below bottom floats (the standard
% LaTeX2e kernel puts them above bottom floats). This is an invasive package
% which rewrites many portions of the LaTeX2e float routines. It may not work
% with other packages that modify the LaTeX2e float routines. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/sttools/
% Do not use the stfloats baselinefloat ability as IEEE does not allow
% \baselineskip to stretch. Authors submitting work to the IEEE should note
% that IEEE rarely uses double column equations and that authors should try
% to avoid such use. Do not be tempted to use the cuted.sty or midfloat.sty
% packages (also by Sigitas Tolusis) as IEEE does not format its papers in
% such ways.
% Do not attempt to use stfloats with fixltx2e as they are incompatible.
% Instead, use Morten Hogholm'a dblfloatfix which combines the features
% of both fixltx2e and stfloats:
%
% \usepackage{dblfloatfix}
% The latest version can be found at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/dblfloatfix/




%\ifCLASSOPTIONcaptionsoff
%  \usepackage[nomarkers]{endfloat}
% \let\MYoriglatexcaption\caption
% \renewcommand{\caption}[2][\relax]{\MYoriglatexcaption[#2]{#2}}
%\fi
% endfloat.sty was written by James Darrell McCauley, Jeff Goldberg and 
% Axel Sommerfeldt. This package may be useful when used in conjunction with 
% IEEEtran.cls'  captionsoff option. Some IEEE journals/societies require that
% submissions have lists of figures/tables at the end of the paper and that
% figures/tables without any captions are placed on a page by themselves at
% the end of the document. If needed, the draftcls IEEEtran class option or
% \CLASSINPUTbaselinestretch interface can be used to increase the line
% spacing as well. Be sure and use the nomarkers option of endfloat to
% prevent endfloat from "marking" where the figures would have been placed
% in the text. The two hack lines of code above are a slight modification of
% that suggested by in the endfloat docs (section 8.4.1) to ensure that
% the full captions always appear in the list of figures/tables - even if
% the user used the short optional argument of \caption[]{}.
% IEEE papers do not typically make use of \caption[]'s optional argument,
% so this should not be an issue. A similar trick can be used to disable
% captions of packages such as subfig.sty that lack options to turn off
% the subcaptions:
% For subfig.sty:
% \let\MYorigsubfloat\subfloat
% \renewcommand{\subfloat}[2][\relax]{\MYorigsubfloat[]{#2}}
% However, the above trick will not work if both optional arguments of
% the \subfloat command are used. Furthermore, there needs to be a
% description of each subfigure *somewhere* and endfloat does not add
% subfigure captions to its list of figures. Thus, the best approach is to
% avoid the use of subfigure captions (many IEEE journals avoid them anyway)
% and instead reference/explain all the subfigures within the main caption.
% The latest version of endfloat.sty and its documentation can obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/endfloat/
%
% The IEEEtran \ifCLASSOPTIONcaptionsoff conditional can also be used
% later in the document, say, to conditionally put the References on a 
% page by themselves.




% *** PDF, URL AND HYPERLINK PACKAGES ***
%
\usepackage{url}
% url.sty was written by Donald Arseneau. It provides better support for
% handling and breaking URLs. url.sty is already installed on most LaTeX
% systems. The latest version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/url/
% Basically, \url{my_url_here}.




% *** Do not adjust lengths that control margins, column widths, etc. ***
% *** Do not use packages that alter fonts (such as pslatex).         ***
% There should be no need to do such things with IEEEtran.cls V1.6 and later.
% (Unless specifically asked to do so by the journal or conference you plan
% to submit to, of course. )


% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}


\begin{document}
%
% paper title
% can use linebreaks \\ within to get better formatting as desired
% Do not put math or special symbols in the title.
\title{Exploiting the Redundancy of Deep Neural Networks for Image Denoising}
%
%
% author names and IEEE memberships
% note positions of commas and nonbreaking spaces ( ~ ) LaTeX will not break
% a structure at a ~ so this keeps an author's name from being broken across
% two lines.
% use \thanks{} to gain access to the first footnote area
% a separate \thanks must be used for each paragraph as LaTeX2e's \thanks
% was not built to handle multiple paragraphs
%

\author{Yi-Qing~Wang,
        and~Jean-Michel~Morel% <-this % stops a space
\thanks{The authors are with CMLA, Ecole Normale Supérieure de Cachan,
Cachan, 94230, France (e-mail: yqwang9@gmail.com; morel@cmla.ens-cachan.fr).}% <-this % stops a space
\thanks{}}

% note the % following the last \IEEEmembership and also \thanks - 
% these prevent an unwanted space from occurring between the last author name
% and the end of the author line. i.e., if you had this:
% 
% \author{....lastname \thanks{...} \thanks{...} }
%                     ^------------^------------^----Do not want these spaces!
%
% a space would be appended to the last name and could cause every name on that
% line to be shifted left slightly. This is one of those "LaTeX things". For
% instance, "\textbf{A} \textbf{B}" will typeset as "A B" not "AB". To get
% "AB" then you have to do: "\textbf{A}\textbf{B}"
% \thanks is no different in this regard, so shield the last } of each \thanks
% that ends a line with a % and do not let a space in before the next \thanks.
% Spaces after \IEEEmembership other than the last one are OK (and needed) as
% you are supposed to have spaces between the names. For what it is worth,
% this is a minor point as most people would not even notice if the said evil
% space somehow managed to creep in.



% The paper headers
\markboth{IEEE Signal Processing Letters}%
{Wang \MakeLowercase{\textit{et al.}}: On Deep Neural Network's Redundancy}
% The only time the second header will appear is for the odd numbered pages
% after the title page when using the twoside option.
% 
% *** Note that you probably will NOT want to include the author's ***
% *** name in the headers of peer review papers.                   ***
% You can use \ifCLASSOPTIONpeerreview for conditional compilation here if
% you desire.




% If you want to put a publisher's ID mark on the page you can do it like
% this:
%\IEEEpubid{0000--0000/00\$00.00~\copyright~2012 IEEE}
% Remember, if you use this you must call \IEEEpubidadjcol in the second
% column for its text to clear the IEEEpubid mark.



% use for special paper notices
%\IEEEspecialpapernotice{(Invited Paper)}




% make the title area
\maketitle

% As a general rule, do not put math, special symbols or citations
% in the abstract or keywords.
\begin{abstract}
Deep neural networks have been shown to hold great promise for various computer vision and image processing tasks. However, they are usually computationally expensive to train and deploy because of their enormous sizes. In this note, using certain invariances of the natural patch distribution, we show that state-of-the-art image denoising networks are redundant in both their architectures and their domain of definition, which suggests that a much smaller network trained on a much smaller dataset might do the task just as well. Two crucial numerical experiments lend full support to our invariance arguments. 
\end{abstract}

% Note that keywords are not normally used for peerreview papers.
\begin{IEEEkeywords}
deep neural network, natural patch space, distribution invariance, redundancy analysis 
\end{IEEEkeywords}






% For peer review papers, you can put extra information on the cover
% page as needed:
% \ifCLASSOPTIONpeerreview
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
% \fi
%
% For peerreview papers, this IEEEtran command inserts a page break and
% creates the second title. It will be ignored for other modes.
\IEEEpeerreviewmaketitle



\section{Introduction}
% The very first letter is a 2 line initial drop letter followed
% by the rest of the first word in caps.
% 
% form to use if the first word consists of a single letter:
% \IEEEPARstart{A}{demo} file is ....
% 
% form to use if you need the single drop letter followed by
% normal text (unknown if ever used by IEEE):
% \IEEEPARstart{A}{}demo file is ....
% 
% Some journals put the first two words in caps:
% \IEEEPARstart{T}{his demo} file is ....
% 
% Here we have the typical use of a "T" for an initial drop letter
% and "HIS" in caps to complete the first word.
\IEEEPARstart{R}{ecently}, a deep neural network based approach \cite{burger2012image} to image denoising has been shown to outperform BM3D \cite{dabov2008image}, widely accepted as the state-of-the-art until now. This dramatic turn of event clearly demonstrates the supremacy of a pure learning strategy over careful handcrafting, a prevailing practice in image processing. A philosophical difference sets these two patch-based methods apart: BM3D, a major spin-off of the original non-local means \cite{buades2005review}, seeks information exclusively inside the noisy image while the neural network derives all its power by looking at noisy and clean patch pairs gathered from other images. Other algorithms exist \cite{zoran2011learning, wang2013sure, burger2013learning} which fall between these two ends of the spectrum. 

It is well known \cite{barron1994approximation} that neural networks form a class of universal approximators. Recent studies \cite{bengio2009learning} further suggest that multi-layer, or deep, neural networks tend to be more efficient in signal representation than their traditional one-hidden-layer counterpart. Whatever their architectures, neural networks in a regression framework seek to approximate the conditional expectation under some input distribution. In our setting, let $x$, $\tilde{x}$, and $y$ denote the clean, noisy and denoised patch. Note that $y$ does not necessarily have the same dimension as $\tilde{x}$. All the neural networks \cite{burger2012image} under this study, for instance, produce a $y \in \mathbb{R}^{17\times 17}$ based on a noisy observation $\tilde{x} \in \mathbb{R}^{39\times 39}$ which includes not only $y$'s corresponding noisy pixels but also those surrounding ones. Also note that with the \textit{true} patch distribution beyond reach, a huge number of patches are drawn from a large natural image dataset to provide the conditional expectation's underlying distribution
\begin{align}
\theta^* &=\argmin_{\theta}\mathbb{E}\|f(\tilde{x}, \theta) - x\|_2^2 \label{ann} \\
	&= \argmin_{\theta}\mathbb{E}\|f(\tilde{x}, \theta) - \mathbb{E}[x | \tilde{x}]\|_2^2 \notag 
\end{align}
where $f(\cdot, \theta): \tilde{x} \mapsto y$ is a neural network and $\theta$ its connection weights. Due to this problem's non-convex nature in general, instead of the potentially intractable $\theta^*$, a good $\theta$, judged on the basis of the resulting network's generalization error, is usually accepted as a solution. 

Despite their impressive performance, the proposed deep networks \cite{burger2012image} are impractical for the majority of industrial applications because they impose a daunting computational cost, besides being inflexible in response to different noise levels. In this work, by analyzing the interplay between the neural networks and the underlying patch distribution they seek to learn, we construct a generic neural network able to handle all levels of Gaussian noise and suggest two ways to scale down these networks without compromising their power.

\section{Towards one generic network}

% needed in second column of first page if using \IEEEpubid
%\IEEEpubidadjcol

% An example of a floating figure using the graphicx package.
% Note that \label must occur AFTER (or within) \caption.
% For figures, \caption should occur after the \includegraphics.
% Note that IEEEtran v1.7 and later has special internal code that
% is designed to preserve the operation of \label within \caption
% even when the captionsoff option is in effect. However, because
% of issues like this, it may be the safest practice to put all your
% \label just after \caption rather than within \caption{}.
%
% Reminder: the "draftcls" or "draftclsnofoot", not "draft", class
% option should be used if it is desired that the figures are to be
% displayed while in draft mode.
%
%\begin{figure}[!t]
%\centering
%\includegraphics[width=2.5in]{myfigure}
% where an .eps filename suffix will be assumed under latex, 
% and a .pdf suffix will be assumed for pdflatex; or what has been declared
% via \DeclareGraphicsExtensions.
%\caption{Simulation Results.}
%\label{fig_sim}
%\end{figure}

% Note that IEEE typically puts floats only at the top, even when this
% results in a large percentage of a column being occupied by floats.


% An example of a double column floating figure using two subfigures.
% (The subfig.sty package must be loaded for this to work.)
% The subfigure \label commands are set within each subfloat command,
% and the \label for the overall figure must come after \caption.
% \hfil is used as a separator to get equal spacing.
% Watch out that the combined width of all the subfigures on a 
% line do not exceed the text width or a line break will occur.
%
%\begin{figure*}[!t]
%\centering
%\subfloat[Case I]{\includegraphics[width=2.5in]{box}%
%\label{fig_first_case}}
%\hfil
%\subfloat[Case II]{\includegraphics[width=2.5in]{box}%
%\label{fig_second_case}}
%\caption{Simulation results.}
%\label{fig_sim}
%\end{figure*}
%
% Note that often IEEE papers with subfigures do not employ subfigure
% captions (using the optional argument to \subfloat[]), but instead will
% reference/describe all of them (a), (b), etc., within the main caption.


% An example of a floating table. Note that, for IEEE style tables, the 
% \caption command should come BEFORE the table. Table text will default to
% \footnotesize as IEEE normally uses this smaller font for tables.
% The \label must come after \caption as always.
%
%\begin{table}[!t]
%% increase table row spacing, adjust to taste
%\renewcommand{\arraystretch}{1.3}
% if using array.sty, it might be a good idea to tweak the value of
% \extrarowheight as needed to properly center the text within the cells
%\caption{An Example of a Table}
%\label{table_example}
%\centering
%% Some packages, such as MDW tools, offer better commands for making tables
%% than the plain LaTeX2e tabular which is used here.
%\begin{tabular}{|c||c|}
%\hline
%One & Two\\
%\hline
%Three & Four\\
%\hline
%\end{tabular}
%\end{table}


% Note that IEEE does not put floats in the very first column - or typically
% anywhere on the first page for that matter. Also, in-text middle ("here")
% positioning is not used. Most IEEE journals use top floats exclusively.
% Note that, LaTeX2e, unlike IEEE journals, places footnotes above bottom
% floats. This can be corrected via the \fnbelowfloat command of the
% stfloats package.

A standard algorithm for Gaussian noise removal needs to know the noise's variance, which seems to require, however unrealistically, a series of networks, one for each noise level. Fortunately, this is not the case. To show it, we started by investigating the statistical regularity of the \textit{natural patch space}: we drew $10^6$ random patches of dimension $39\times 39$ from the \textit{Berkeley Segmentation Dataset} (BSD500) rendered to grayscale with \textit{Matlab}'s \texttt{rgb2gray} function. These patches were then normalized using the formula
\begin{align*}
\bar{p} = (p / 255 - 0.5) \cdot 5
\end{align*} 
provided in \cite{burger2012image}. For each normalized patch, we computed the mean and standard deviation of its $1521$ pixels and then plotted their population distribution along these two dimensions. For comparison, we did the same with the \textit{Kodak} image dataset (\url{http://r0k.us/graphics/kodak}). 
\begin{figure*}[htb]
\centering
\subfloat[]{\label{bsd}\includegraphics[width = 3in, height = 2in]{Figures/heatmap}}
\hspace{1cm}
\subfloat[]{\label{kodak}\includegraphics[width = 3in, height = 2in]{Figures/heatmapKodak}}
\caption{2D histogram of random patches from (\ref{bsd}) the grayscale BSD500 and (\ref{kodak}) the grayscale \textit{Kodak}. The horizontal (resp. vertical) axis represents the normalized patch's mean (resp. standard variation). In both cases, patches concentrate around $(-0.5, 0.5)$. Also note that at the two ends of the horizontal axis, there are two important flat patch clusters. } \label{heatmap}
\vspace{-3mm} 
\end{figure*}
The results (Fig.\ref{heatmap}) show that most normalized natural patches are distributed around $(-0.5, 0.5)$. Even without access to the supervised pairs used in \cite{burger2012image}, we can therefore expect their neural networks trained according to the criterion (\ref{ann}) to do well with patches from this neighborhood because of the sheer number of examples available. Moreover, the patch-wide variance peaking at the patch-wide means around $-0.5$ implies a higher tolerance for linear transforms there. That is, it is more probable for a natural patch $p$ transformed by 
\begin{align*}
q = a \cdot ( p - s ) \;\; \text{with $s$ some constant patch and $a \geq 0$} 
\end{align*}
to remain \textit{natural} if $q$'s mean is close to $-0.5$.

To verify this hypothesis, we shifted the mean of individual patches before denoising them. Running Algorithm \ref{setMean} on $10^5$ $39\times 39$ random patches drawn from the grayscale BSD500, we computed the resulting root mean square error (RMSE). Fig.\ref{meanShift} shows that regardless of the applied noise strength $\sigma$, the best empirical performance is attained with the patch mean shift set to $-0.5$, thereby confirming our conjecture. In addition, it is important to note that thanks to a high ratio between the patch size and $\sigma$, estimating the clean patch mean from its noisy version (Line 7 in Algorithm \ref{setMean}) is fairly reliable. The fact that the patch mean shift yields a slightly worse RMSE may be attributed to our indiscriminate treatment of all patches. 

\begin{algorithm}
\caption{Patch Mean Normalization Test}
\fontsize{3.0mm}{4.0mm}\selectfont
\begin{algorithmic}[1]
\STATE \textbf{Input:} $n$ clean patches $p_j$ of dimension $39 \times 39$ 
\STATE \textbf{Output:} $n$ denoised (resp. clean) $17\times 17$ patches $\widehat{\mathfrak{p}}_j$ (resp. $\mathfrak{p}_j$)
\STATE \textbf{Parameter:} noise variance $\sigma^2$ and desired patch mean value $\mathfrak{m}$ 
\FOR{$j=1$ to $n$}
\STATE retrieve the $17\times 17$ clean patch $\mathfrak{p}_j$ in the center of $p_j$
\STATE generate the normalized noisy patch  
\begin{align*}
\tilde{x}_j \gets ((p_j + n_j)/255 - 0.5)\cdot 5
\end{align*} 
with $n_j$ having $1521$ i.i.d. Gaussian random variables $\mathcal{N}(0, \sigma^2)$ and $n_j$ independent of $n_{j'}$ for $j \neq j'$
\STATE compute the patch mean shift $s_j = \frac{1}{1521} \sum_{k=1}^{1521}\tilde{x}_{jk} - \mathfrak{m}$ where $\tilde{x}_{jk}$ is the $k$-th pixel in $\tilde{x}_j$
\STATE shift the network's input $\forall k,\; \tilde{x}_{jk} \gets \tilde{x}_{jk} - s_j$ 
\STATE denoise $y_j = f_\sigma(\tilde{x}_{j}, \theta)$ where $f_\sigma(\cdot, \theta)$ is the deep neural network \cite{burger2012image} trained with Gaussian noise standard deviation $\sigma$
\STATE shift the network's output by the same amount $\forall k,\; y_{jk} \gets y_{jk}+ s_j$ in the opposite direction
\STATE reverse the normalization $\widehat{\mathfrak{p}}_{j} \gets (y_j /5 + 0.5)\cdot 255$ 
\ENDFOR
\end{algorithmic}\label{setMean}
\end{algorithm}

\begin{figure}[htb]
\centering
\subfloat[]{\label{s25}\includegraphics[width = 3in, height = 1.8in]{Figures/rmses25}}\\
%\hspace{1cm}
\subfloat[]{\label{s75}\includegraphics[width = 3in, height = 1.8in]{Figures/rmses75}}
\caption{(\ref{s25}) (resp. (\ref{s75})) plots the RMSEs resulting from Algorithm \ref{setMean} with $\sigma = 25$ (resp. $75$). The blue curve records the dedicated network's performance with the patch mean values ranging from $-1.5$ to $0.9$. And the red line is the RMSE achieved on the same test data without patch mean shift. Other available neural networks have also been tested with very similar results. } \label{meanShift}
\vspace{-3mm} 
\end{figure}

The previous analysis paves the way for a generic network able to handle all levels of noise. Let us use the neural network trained with the noise level $\sigma^*$ for instance. To restore a noisy patch $\tilde{p}$ with noise standard deviation at $\sigma$, one multiplies $\tilde{p}$ by $\sigma^*\sigma^{-1}$, apply the patch mean shift and let the neural network operate on the normalized patch (Algorithm \ref{lt}). Henceforth $\sigma^*$ itself becomes a parameter for further optimization, too. 

\begin{algorithm}
\caption{Generic Neural Network Denoising}
\fontsize{3.0mm}{4.0mm}\selectfont
\begin{algorithmic}[1]
\STATE \textbf{Input:} noisy patch $\tilde{p}$ of dimension $39\times 39$ and its noise level $\sigma$ 
\STATE \textbf{Output:} denoised $17\times 17$ patch $\widehat{\mathfrak{p}}$ 
\STATE \textbf{Parameter:} noise level $\sigma^*$ of the trained neural network $f_{\sigma^*}(\cdot, \theta)$ and optimal shift $\mathfrak{m} = -0.5$ 
\STATE Scale the noise $\tilde{p} \gets \sigma^* \sigma^{-1}\tilde{p}$  
\STATE Normalize the patch $\tilde{x} \gets (\tilde{p}/255 - 0.5)\cdot 5$
\STATE Compute the patch mean shift $s = \frac{1}{1521} \sum_{k=1}^{1521}\tilde{x}_{k} - \mathfrak{m}$ where $\tilde{x}_{k}$ is the $k$-th pixel in $\tilde{x}$
\STATE Shift the network's input mean $\forall k,\; \tilde{x}_{k} \gets \tilde{x}_{k} - s$ 
\STATE Run the generic neural network denoising $y = f_{\sigma^*}(\tilde{x}, \theta)$
\STATE Shift the network's output by the same amount $\forall k,\; y_{k} \gets y_{k}+ s$ in the opposite direction
\STATE Reverse the normalization $\widehat{\mathfrak{p}} \gets (y /5 + 0.5)\cdot 255 \cdot (\sigma^*\sigma^{-1})^{-1}$ 
\end{algorithmic}\label{lt}
\end{algorithm}

\begin{figure}[htb]
\centering
\subfloat[]{\label{all}\includegraphics[width = 3.0in, height = 1.8in]{Figures/singleNN}}\\
%\hspace{1cm}
\subfloat[]{\label{sole}\includegraphics[width = 3.0in, height = 1.8in]{Figures/singleNNsole}}
\caption{(\ref{all}) Performance discrepancy between dedicated neural networks and $\sigma^*$-indexed generic neural networks (using Algorithm \ref{lt}) at handling different noise levels. The horizontal axis marks various test Gaussian noise levels and the vertical axis the achieved RMSEs on $10^5$ random patches from BSD500. (\ref{sole}) singles out the best generic network with $\sigma^* = 25$.} \label{allsole}
\vspace{-3mm} 
\end{figure}

Fig.\ref{allsole} shows that the generic network built with $\sigma^* = 25$ is the best, which is hardly surprising because a neural network can learn the most about the underlying patch space when noise is weak. Moreover, except for the extremely noisy case, one sees no tangible difference in Fig.\ref{allsole} between a dedicated network and the best generic network. Also observe that the higher the $\sigma^*$, the worse the RMSEs at low noise levels, an expected phenomenon since a high $\sigma^*\sigma^{-1}$ exaggerates the patch-wide variation so much that resultant patches no longer remain in the natural patch space. 

The implication of this analysis is straightforward: training a network with the same architecture but at an even lower noise level may be rewarding. But it should also be said that too low a $\sigma^*$ is not likely to work well in a strong noise environment, as already observed in Fig.\ref{allsole}. Because the factor $\sigma^*\sigma^{-1}$ will then flatten all the meaningful patch-wide variations, leaving the network unable to tell one patch from another.

Put differently, what we have shown so far is that the trained deep network is not very different from its most informative section. Hence, if it had been trained on mean normalized patches, its performance would have been better thanks to a denser data distribution. Even with the current network, it may still be possible to improve Algorithm \ref{lt} with a variance dependent patch mean shift, resulting in an undulating section on the mean-variance plane. However, this would require additional knowledge of the training patch distribution. 

To conclude this section, we tested our simple algorithm with $\sigma^* = 25$ on some real images. The test images (Fig.\ref{suite}) come from the \textit{Image Processing On Line} (\texttt{IPOL}) website and the results are compiled in Tab.\ref{fullres}. Recall that even for $\sigma = 25$, our generic network is different from the dedicated one, in that ours involves one additional step of patch mean shift. The obtained results are thus consistent with Fig.\ref{s25}. 

\begin{table}
\centering 
\fontsize{2.3mm}{3.5mm}\selectfont
\caption{Comparison between dedicated neural networks and our algorithm (generic $\sigma^* = 25$)}
\vspace{-3mm} 
\begin{tabular}{cc}
\begin{tabular}{l*{2}{c}r}
\\[3pt]\hline
\textbf{$\sigma=25$} & \textbf{ dedicated} & \textbf{ generic}\\
\hline
\textbf{computer} & \textbf{7.99} & 8.10\\
\textbf{dice} & 2.77 & 2.77\\
\textbf{flower} & \textbf{4.59} & 4.63\\
\textbf{girl} & \textbf{3.38} & 3.41\\
\textbf{traffic} & \textbf{9.16} & 9.22\\
\textbf{valldemossa} & \textbf{11.85} & 11.99\\
\textsl{avg.} & \textbf{6.62} & 6.68\\
\hline
\end{tabular}
&
\begin{tabular}{l*{2}{c}r}
\\[3pt]\hline
\textbf{$\sigma=35$} & \textbf{ dedicated} & \textbf{ generic}\\
\hline
\textbf{computer} & \textbf{9.63} & 9.82\\
\textbf{dice} & \textbf{3.29} & 3.45\\
\textbf{flower} & \textbf{5.53} & 5.64\\
\textbf{girl} & \textbf{3.88} & 4.07\\
\textbf{traffic} & \textbf{10.81} & 10.94\\
\textbf{valldemossa} & \textbf{14.21} & 14.28\\
\textsl{avg.} & \textbf{7.89} & 8.03\\
\hline
\end{tabular}
\\
\begin{tabular}{l*{2}{c}r}
\\[3pt]\hline
\textbf{$\sigma=50$} & \textbf{ dedicated} & \textbf{ generic}\\
\hline
\textbf{computer} & \textbf{11.59} & 11.92\\
\textbf{dice} & \textbf{4.17} & 4.50\\
\textbf{flower} & \textbf{6.85} & 7.06\\
\textbf{girl} & \textbf{4.60} & 4.92\\
\textbf{traffic} & \textbf{12.83} & 13.07\\
\textbf{valldemossa} & \textbf{16.98} & 17.06\\
\textsl{avg.} & \textbf{9.50} & 9.75\\
\hline
\end{tabular}
&
\begin{tabular}{l*{2}{c}r}
\\[3pt]\hline
\textbf{$\sigma=65$} & \textbf{ dedicated} & \textbf{ generic}\\
\hline
\textbf{computer} & \textbf{13.16} & 13.82\\
\textbf{dice} & \textbf{4.83} & 5.39\\
\textbf{flower} & \textbf{7.89} & 8.20\\
\textbf{girl} & \textbf{5.28} & 5.74\\
\textbf{traffic} & \textbf{14.32} & 14.78\\
\textbf{valldemossa} & \textbf{18.67} & 18.99\\
\textsl{avg.} & \textbf{10.69} & 11.15\\
\hline
\end{tabular}
\\
\begin{tabular}{l*{2}{c}r}
\\[3pt]\hline
\textbf{$\sigma=75$} & \textbf{ dedicated} & \textbf{ generic}\\
\hline
\textbf{computer} & \textbf{14.10} & 14.78\\
\textbf{dice} & \textbf{5.48} & 6.14\\
\textbf{flower} & \textbf{8.57} & 8.96\\
\textbf{girl} & \textbf{5.66} & 6.20\\
\textbf{traffic} & \textbf{15.08} & 15.56\\
\textbf{valldemossa} & \textbf{19.97} & 20.34\\
\textsl{avg.} & \textbf{11.47} & 11.99\\
\hline
\end{tabular}
&
\begin{tabular}{l*{2}{c}r}
\\[3pt]\hline
\textbf{$\sigma=170$} & \textbf{ dedicated} & \textbf{ generic}\\
\hline
\textbf{computer} & \textbf{20.51} & 21.81\\
\textbf{dice} & \textbf{9.23} & 10.92\\
\textbf{flower} & \textbf{12.84} & 13.64\\
\textbf{girl} & \textbf{8.79} & 10.54\\
\textbf{traffic} & \textbf{20.71} & 21.72\\
\textbf{valldemossa} & \textbf{26.42} & 27.26\\
\textsl{avg.} & \textbf{16.41} & 17.64\\
\hline
\end{tabular}
\end{tabular}\label{fullres}
\vspace{-3mm} 
\end{table}

\begin{figure}[htb]
\centering
\vspace{-3mm} 
\subfloat[]{\label{computer}\includegraphics[width = 0.5in, height = 0.34in]{Figures/computer}}
\subfloat[]{\label{dice}\includegraphics[width = 0.5in, height = 0.34in]{Figures/dice}}
\subfloat[]{\label{flower}\includegraphics[width = 0.5in, height = 0.34in]{Figures/flower}}
\subfloat[]{\label{girl}\includegraphics[width = 0.5in, height = 0.34in]{Figures/girl}}
\subfloat[]{\label{traffic}\includegraphics[width = 0.5in, height = 0.34in]{Figures/traffic}}
\subfloat[]{\label{valldemossa}\includegraphics[width = 0.86in, height = 0.34in]{Figures/valldemossa}}
\caption[]{Test images for algorithm comparison (\ref{computer}) computer (\ref{dice}) dice (\ref{flower}) flower (\ref{girl}) girl (\ref{traffic}) traffic (\ref{valldemossa}) valldemossa. All images are of dimension $704\times469$ except for valldemossa ($769\times 338$). } \label{suite} 
\vspace{-3mm} 
\end{figure}

\section{Neural network domain reduction}

\begin{figure*}[htb]
\centering
\vspace{-3mm} 
\subfloat[]{\label{before}\includegraphics[width = 2.30in, height = 2in]{Figures/beforea2}}
%\hspace{1cm}
\subfloat[]{\label{after}\includegraphics[width = 2.30in, height = 2in]{Figures/aftera2}}
\subfloat[]{\label{aftern}\includegraphics[width = 2.30in, height = 2in]{Figures/aftera2n}}
\caption{(\ref{before}) Some random patches of dimension $39\times 39$ from the grayscale BSD500 (\ref{after}) their geometrically normalized versions (\ref{aftern}) the geometrically normalized noisy versions of (\ref{before}) with $\sigma = 25$ (using Algorithm \ref{pvr}). There is one noticeable noise induced error (the fourth from left in the second row). }  \label{flipRot}
\vspace{-3mm} 
\end{figure*}

This second issue is less task specific. Viewed as an ordinary function, one major source of a neural network's redundancy comes from its domain of definition \cite{lisani2013explore}. If we can both restrict its domain and keep all the training examples at the same time, a neural network shall be able to learn better for two reasons. First, the image patches' intrinsic geometry makes it essential for a good but rudimentary neural network to acquire a certain symmetry, although this property does not seem to follow naturally from the way a network is parametrized, thereby leading to a larger than necessary architecture and training pool. It is thus desirable to remove this degree of freedom from the network's inputs. Second, if we can somehow convert and bring training examples to a smaller area that would otherwise be scattered around in the domain, in a spirit akin to importance sampling in Monte Carlo, the network should learn better with more data (see the previous section). 


Fortunately, this is feasible, again due to the natural patch distribution being invariant to many linear operations. The two \textit{geometrically normalizing} operations proposed in Algorithm \ref{pvr} are such examples. They are reversible, non patch size specific, and thus can be applied without difficulty to a possibly smaller denoised patch as in our case. In addition, invariant to linear transforms by design, they are also insensitive to noise, provided that the patch they operate on is large enough. As shown in Fig.\ref{flipRot}, Algorithm \ref{pvr} is indeed robust. The inconsistency that occurs at times due to noise, however, is unlikely to change much the result in RMSE because it affects only those patches with insignificant intrinsic orientation. For an empirical validation of this hypothesis, we ran $10^5$ geometrically normalized random noisy patches ($\sigma = 25$) through the neural network trained with $\sigma^* = 25$ and applied the reversed operators $T_p^{-1}$ on the outputs. The resulting RMSE ($9.96$), compared with that ($9.95$) from a direct application of the neural network using the same noisy patches without geometrical normalization, fully confirmed our approach.  

\begin{algorithm}
\caption{Patch Geometrical Normalization}
\fontsize{3.0mm}{4.0mm}\selectfont
\begin{algorithmic}[1]
\STATE \textbf{Input:} a possibly noisy patch $p$ of dimension $39 \times 39$ 
\STATE \textbf{Output:} its geometrically normalized version $p_g$ and the associated geometrically normalizing operator $T_p: p \mapsto p_g$ 
\STATE Let $\mathfrak{1}$ be a column vector of $39$ ones and $\var(\cdot)$ be the variance of the argument vector's entries.  
\IF{ $\var (p \mathfrak{1}) > \var (p'\mathfrak{1})$}
\STATE $p_g \gets p'$ and $T_p \gets \mathfrak{t}$ with $\mathfrak{t}$ the standard matrix transposition
\ELSE
\STATE $p_g \gets p$ and $T_p \gets \mathfrak{I}$ with $\mathfrak{I}$ the identity
\ENDIF
\IF{the sum of the top half of $p_g$ is bigger than that of its bottom half}
\STATE Flip $p_g$ upside down and $T_p \gets \mathfrak{f} \circ T_p$ with $\mathfrak{f}$ the flipping operation 
\ELSE
\STATE $T_p \gets \mathfrak{I} \circ T_p$ 
\ENDIF 
\end{algorithmic}\label{pvr}
\end{algorithm}





% if have a single appendix:
%\appendix[Proof of the Zonklar Equations]
% or
%\appendix  % for no appendix heading
% do not use \section anymore after \appendix, only \section*
% is possibly needed

% use appendices with more than one appendix
% then use \section to start each appendix
% you must declare a \section before using any
% \subsection or using \label (\appendices by itself
% starts a section numbered zero.)
%

% use section* for acknowledgement
\section*{Acknowledgment}

This work was supported in part by CNES, ERC(AG Twelve Labours), and ONR N00014-97-1-0839.



% Can use something like this to put references on a page
% by themselves when using endfloat and the captionsoff option.
\ifCLASSOPTIONcaptionsoff
  \newpage
\fi



% trigger a \newpage just before the given reference
% number - used to balance the columns on the last page
% adjust value as needed - may need to be readjusted if
% the document is modified later
%\IEEEtriggeratref{8}
% The "triggered" command can be changed if desired:
%\IEEEtriggercmd{\enlargethispage{-5in}}

% references section

% can use a bibliography generated by BibTeX as a .bbl file
% BibTeX documentation can be easily obtained at:
% http://www.ctan.org/tex-archive/biblio/bibtex/contrib/doc/
% The IEEEtran BibTeX style support page is at:
% http://www.michaelshell.org/tex/ieeetran/bibtex/
\bibliographystyle{IEEEtran}
\bibliography{refs}
% argument is your BibTeX string definitions and bibliography database(s)
%\bibliography{IEEEabrv,../bib/paper}
%
% <OR> manually copy in the resultant .bbl file
% set second argument of \begin to the number of references
% (used to reserve space for the reference number labels box)

% biography section
% 
% If you have an EPS/PDF photo (graphicx package needed) extra braces are
% needed around the contents of the optional argument to biography to prevent
% the LaTeX parser from getting confused when it sees the complicated
% \includegraphics command within an optional argument. (You could create
% your own custom macro containing the \includegraphics command to make things
% simpler here.)
%\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{mshell}}]{Michael Shell}
% or if you just want to reserve a space for a photo:

%\begin{IEEEbiography}{Yi-Qing Wang}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/wang}}]{Yi-Qing Wang}
received the Bachelor's degree in Telecommunications from Fudan University, Shanghai, China in 2005, the diplôme d'ingénieur from Ecole Polytechnique, Palaiseau, France in 2008, the M.Sc. degree in probabilities from University Pierre et Marie Curie, Paris, France in 2009 and the M.Sc. degree in applied mathematics from ENS de Cachan, Cachan, France in 2012.

He is now a second year Ph.D. student at CMLA, ENS de Cachan. His research interests include machine learning and image processing.
\end{IEEEbiography}

% if you will not have a photo at all:
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/morel}}]{Jean-Michel Morel}
received the Ph.D. degree in applied mathematics from University Pierre et Marie Curie, Paris, France in 1980. He started his career in 1979 as assistant professor in Marseille Luminy, then moved in 1984 to University Paris-Dauphine where he was promoted professor in 1992.  He is Professor of Applied Mathematics at the Ecole Normale Supérieure de Cachan since 1997. His research is focused on the mathematical analysis of image processing. He received in 2013 the Grand Prix INRIA delivered by the French Academy of Science.
\end{IEEEbiography}

% insert where needed to balance the two columns on the last page with
% biographies
%\newpage

% You can push biographies down or up by placing
% a \vfill before or after them. The appropriate
% use of \vfill depends on what kind of text is
% on the last page and whether or not the columns
% are being equalized.

%\vfill

% Can be used to pull up biographies so that the bottom of the last one
% is flush with the other column.
%\enlargethispage{-5in}



% that's all folks
\end{document}


