@inproceedings{60869a33224b4ff1b84a056c42a7535f,

title = "Analysis of the Relative Entropy Asymmetry in the Regularization of Empirical Risk Minimization",

abstract = "The effect of the relative entropy asymmetry is analyzed in the empirical risk minimization with relative entropy regularization (ERM-RER) problem. A novel regularization is introduced, coined Type-II regularization, that allows for solutions to the ERM-RER problem with a support that extends outside the support of the reference measure. The solution to the new ERM-RER Type-II problem is analytically characterized in terms of the Radon-Nikodym derivative of the reference measure with respect to the solution. The analysis of the solution unveils the following properties of relative entropy when it acts as a regularizer in the ERM-RER problem: i) relative entropy forces the support of the Type-II solution to collapse into the support of the reference measure, which introduces a strong inductive bias that dominates the evidence provided by the training data; ii) Type-II regularization is equivalent to classical relative entropy regularization with an appropriate transformation of the empirical risk function. Closed-form expressions of the expected empirical risk as a function of the regularization parameters are provided.",

keywords = "Empirical risk minimization, inductive bias, reference measure, regularization, relative entropy",

author = "Francisco Daunas and I{\~n}aki Esnaola and Perlaza, {Samir M.} and {Vincent Poor}, H.",

note = "Publisher Copyright: {\textcopyright} 2023 IEEE.; 2023 IEEE International Symposium on Information Theory, ISIT 2023 ; Conference date: 25-06-2023 Through 30-06-2023",

year = "2023",

doi = "10.1109/ISIT54713.2023.10206876",

language = "English (US)",

series = "IEEE International Symposium on Information Theory - Proceedings",

publisher = "Institute of Electrical and Electronics Engineers Inc.",

pages = "340--345",

booktitle = "2023 IEEE International Symposium on Information Theory, ISIT 2023",

address = "United States",

}