2023
Adilova, Linara; Abourayya, Amr; Li, Jianning; Dada, Amin; Petzka, Henning; Egger, Jan; Kleesiek, Jens; Kamp, Michael
FAM: Relative Flatness Aware Minimization Proceedings Article
In: Proceedings of the ICML Workshop on Topology, Algebra, and Geometry in Machine Learning (TAG-ML), 2023.
Links | BibTeX | Tags: deep learning, flatness, generalization, machine learning, relative flatness, theory of deep learning
@inproceedings{adilova2023fam,
title = {FAM: Relative Flatness Aware Minimization},
author = {Linara Adilova and Amr Abourayya and Jianning Li and Amin Dada and Henning Petzka and Jan Egger and Jens Kleesiek and Michael Kamp},
url = {https://michaelkamp.org/wp-content/uploads/2023/06/fam_regularization.pdf},
year = {2023},
date = {2023-07-22},
urldate = {2023-07-22},
booktitle = {Proceedings of the ICML Workshop on Topology, Algebra, and Geometry in Machine Learning (TAG-ML)},
keywords = {deep learning, flatness, generalization, machine learning, relative flatness, theory of deep learning},
pubstate = {published},
tppubtype = {inproceedings}
}
2021
Petzka, Henning; Kamp, Michael; Adilova, Linara; Sminchisescu, Cristian; Boley, Mario
Relative Flatness and Generalization Proceedings Article
In: Advances in Neural Information Processing Systems, Curran Associates, Inc., 2021.
Abstract | BibTeX | Tags: deep learning, flatness, generalization, Hessian, learning theory, relative flatness, theory of deep learning
@inproceedings{petzka2021relative,
title = {Relative Flatness and Generalization},
author = {Henning Petzka and Michael Kamp and Linara Adilova and Cristian Sminchisescu and Mario Boley},
year = {2021},
date = {2021-12-07},
urldate = {2021-12-07},
booktitle = {Advances in Neural Information Processing Systems},
publisher = {Curran Associates, Inc.},
abstract = {Flatness of the loss curve is conjectured to be connected to the generalization ability of machine learning models, in particular neural networks. While it has been empirically observed that flatness measures consistently correlate strongly with generalization, it is still an open theoretical problem why and under which circumstances flatness is connected to generalization, in particular in light of reparameterizations that change certain flatness measures but leave generalization unchanged. We investigate the connection between flatness and generalization by relating it to the interpolation from representative data, deriving notions of representativeness, and feature robustness. The notions allow us to rigorously connect flatness and generalization and to identify conditions under which the connection holds. Moreover, they give rise to a novel, but natural relative flatness measure that correlates strongly with generalization, simplifies to ridge regression for ordinary least squares, and solves the reparameterization issue.},
keywords = {deep learning, flatness, generalization, Hessian, learning theory, relative flatness, theory of deep learning},
pubstate = {published},
tppubtype = {inproceedings}
}
Flatness of the loss curve is conjectured to be connected to the generalization ability of machine learning models, in particular neural networks. While it has been empirically observed that flatness measures consistently correlate strongly with generalization, it is still an open theoretical problem why and under which circumstances flatness is connected to generalization, in particular in light of reparameterizations that change certain flatness measures but leave generalization unchanged. We investigate the connection between flatness and generalization by relating it to the interpolation from representative data, deriving notions of representativeness, and feature robustness. The notions allow us to rigorously connect flatness and generalization and to identify conditions under which the connection holds. Moreover, they give rise to a novel, but natural relative flatness measure that correlates strongly with generalization, simplifies to ridge regression for ordinary least squares, and solves the reparameterization issue.
