A Kernel Perspective for Regularizing Deep Neural Networks

A Kernel Perspective for Regularizing Deep Neural Networks. Bietti, A., Mialon, G., Chen, D., & Mairal, J. Technical Report arXiv:1810.00363, arXiv, May, 2019. arXiv:1810.00363 [cs, stat] type: article

Paper doi abstract bibtex

We propose a new point of view for regularizing deep neural networks by using the norm of a reproducing kernel Hilbert space (RKHS). Even though this norm cannot be computed, it admits upper and lower approximations leading to various practical strategies. Specifically, this perspective (i) provides a common umbrella for many existing regularization principles, including spectral norm and gradient penalties, or adversarial training, (ii) leads to new effective regularization penalties, and (iii) suggests hybrid strategies combining lower and upper bounds to get better approximations of the RKHS norm. We experimentally show this approach to be effective when learning on small datasets, or to obtain adversarially robust models.

@techreport{bietti_kernel_2019,
	title = {A {Kernel} {Perspective} for {Regularizing} {Deep} {Neural} {Networks}},
	url = {http://arxiv.org/abs/1810.00363},
	abstract = {We propose a new point of view for regularizing deep neural networks by using the norm of a reproducing kernel Hilbert space (RKHS). Even though this norm cannot be computed, it admits upper and lower approximations leading to various practical strategies. Specifically, this perspective (i) provides a common umbrella for many existing regularization principles, including spectral norm and gradient penalties, or adversarial training, (ii) leads to new effective regularization penalties, and (iii) suggests hybrid strategies combining lower and upper bounds to get better approximations of the RKHS norm. We experimentally show this approach to be effective when learning on small datasets, or to obtain adversarially robust models.},
	number = {arXiv:1810.00363},
	urldate = {2022-05-23},
	institution = {arXiv},
	author = {Bietti, Alberto and Mialon, Grégoire and Chen, Dexiong and Mairal, Julien},
	month = may,
	year = {2019},
	doi = {10.48550/arXiv.1810.00363},
	note = {arXiv:1810.00363 [cs, stat]
type: article},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

Downloads: 0

{"_id":"AxC3tnabxfLfDsjRv","bibbaseid":"bietti-mialon-chen-mairal-akernelperspectiveforregularizingdeepneuralnetworks-2019","author_short":["Bietti, A.","Mialon, G.","Chen, D.","Mairal, J."],"bibdata":{"bibtype":"techreport","type":"techreport","title":"A Kernel Perspective for Regularizing Deep Neural Networks","url":"http://arxiv.org/abs/1810.00363","abstract":"We propose a new point of view for regularizing deep neural networks by using the norm of a reproducing kernel Hilbert space (RKHS). Even though this norm cannot be computed, it admits upper and lower approximations leading to various practical strategies. Specifically, this perspective (i) provides a common umbrella for many existing regularization principles, including spectral norm and gradient penalties, or adversarial training, (ii) leads to new effective regularization penalties, and (iii) suggests hybrid strategies combining lower and upper bounds to get better approximations of the RKHS norm. We experimentally show this approach to be effective when learning on small datasets, or to obtain adversarially robust models.","number":"arXiv:1810.00363","urldate":"2022-05-23","institution":"arXiv","author":[{"propositions":[],"lastnames":["Bietti"],"firstnames":["Alberto"],"suffixes":[]},{"propositions":[],"lastnames":["Mialon"],"firstnames":["Grégoire"],"suffixes":[]},{"propositions":[],"lastnames":["Chen"],"firstnames":["Dexiong"],"suffixes":[]},{"propositions":[],"lastnames":["Mairal"],"firstnames":["Julien"],"suffixes":[]}],"month":"May","year":"2019","doi":"10.48550/arXiv.1810.00363","note":"arXiv:1810.00363 [cs, stat] type: article","keywords":"Computer Science - Machine Learning, Statistics - Machine Learning","bibtex":"@techreport{bietti_kernel_2019,\n\ttitle = {A {Kernel} {Perspective} for {Regularizing} {Deep} {Neural} {Networks}},\n\turl = {http://arxiv.org/abs/1810.00363},\n\tabstract = {We propose a new point of view for regularizing deep neural networks by using the norm of a reproducing kernel Hilbert space (RKHS). Even though this norm cannot be computed, it admits upper and lower approximations leading to various practical strategies. Specifically, this perspective (i) provides a common umbrella for many existing regularization principles, including spectral norm and gradient penalties, or adversarial training, (ii) leads to new effective regularization penalties, and (iii) suggests hybrid strategies combining lower and upper bounds to get better approximations of the RKHS norm. We experimentally show this approach to be effective when learning on small datasets, or to obtain adversarially robust models.},\n\tnumber = {arXiv:1810.00363},\n\turldate = {2022-05-23},\n\tinstitution = {arXiv},\n\tauthor = {Bietti, Alberto and Mialon, Grégoire and Chen, Dexiong and Mairal, Julien},\n\tmonth = may,\n\tyear = {2019},\n\tdoi = {10.48550/arXiv.1810.00363},\n\tnote = {arXiv:1810.00363 [cs, stat]\ntype: article},\n\tkeywords = {Computer Science - Machine Learning, Statistics - Machine Learning},\n}\n\n","author_short":["Bietti, A.","Mialon, G.","Chen, D.","Mairal, J."],"key":"bietti_kernel_2019","id":"bietti_kernel_2019","bibbaseid":"bietti-mialon-chen-mairal-akernelperspectiveforregularizingdeepneuralnetworks-2019","role":"author","urls":{"Paper":"http://arxiv.org/abs/1810.00363"},"keyword":["Computer Science - Machine Learning","Statistics - Machine Learning"],"metadata":{"authorlinks":{}},"html":""},"bibtype":"techreport","biburl":"https://bibbase.org/zotero/mxmplx","dataSources":["aXmRAq63YsH7a3ufx"],"keywords":["computer science - machine learning","statistics - machine learning"],"search_terms":["kernel","perspective","regularizing","deep","neural","networks","bietti","mialon","chen","mairal"],"title":"A Kernel Perspective for Regularizing Deep Neural Networks","year":2019}