@inproceedings{55e3b7766662445c9393e8e81b93fd83,
title = "Noisin: Unbiased Regularization for Recurrent Neural Networks",
abstract = "Recurrent neural networks (RNNS) are powerful models of sequential data. They have been successfully used in domains such as text and speech. However, RNNs are susceptible to over- fitting; regularization is important. In this paper we develop Noisin, a new method for regularizing RNNs. Noisin injects random noise into the hidden states of the RNN and then maximizes the corresponding marginal likelihood of the data. We show how Noisin applies to any RNN and we study many different types of noise. Noisin is unbiased-it preserves the underlying RNN on average. We characterize how Noisin regularizes its RNN both theoretically and empirically. On language modeling benchmarks, Noisin improves over dropout by as much as 12.2% on the Penn Treebank and 9.4% on the Wikitext-2 dataset. We also compared the state-of-the-art language model of Yang et al. 2017, both with and without Noisin. On the Penn Treebank, the method with Noisin more quickly reaches state- of-the-art performance.",
author = "Dieng, {Adji B.} and Rajcsh Ranganath and Jaan Altosaar and Blei, {David M.}",
note = "Funding Information: We thank the Princeton Institute for Computational Science and Engineering (PICSciE), the Office of Information Technology's High Performance Computing Center and Visualization Laboratory at Princeton University for the computational resources. This work was supported by ONR N00014-15-1-2209, ONR 133691-5102004, NIH 5100481- 5500001084, NSF CCF-1740833, the Alfred P. Sloan Foundation, the John Simon Guggenheim Foundation, Facebook, Amazon, and IBM. Publisher Copyright: {\textcopyright} 2018 35th International Conference on Machine Learning, ICML 2018. All rights reserved.; 35th International Conference on Machine Learning, ICML 2018 ; Conference date: 10-07-2018 Through 15-07-2018",
year = "2018",
language = "English (US)",
series = "35th International Conference on Machine Learning, ICML 2018",
publisher = "International Machine Learning Society (IMLS)",
pages = "2030--2039",
editor = "Andreas Krause and Jennifer Dy",
booktitle = "35th International Conference on Machine Learning, ICML 2018",
}