@inproceedings{12536, abstract = {We consider the problem of estimating a rank-1 signal corrupted by structured rotationally invariant noise, and address the following question: how well do inference algorithms perform when the noise statistics is unknown and hence Gaussian noise is assumed? While the matched Bayes-optimal setting with unstructured noise is well understood, the analysis of this mismatched problem is only at its premises. In this paper, we make a step towards understanding the effect of the strong source of mismatch which is the noise statistics. Our main technical contribution is the rigorous analysis of a Bayes estimator and of an approximate message passing (AMP) algorithm, both of which incorrectly assume a Gaussian setup. The first result exploits the theory of spherical integrals and of low-rank matrix perturbations; the idea behind the second one is to design and analyze an artificial AMP which, by taking advantage of the flexibility in the denoisers, is able to "correct" the mismatch. Armed with these sharp asymptotic characterizations, we unveil a rich and often unexpected phenomenology. For example, despite AMP is in principle designed to efficiently compute the Bayes estimator, the former is outperformed by the latter in terms of mean-square error. We show that this performance gap is due to an incorrect estimation of the signal norm. In fact, when the SNR is large enough, the overlaps of the AMP and the Bayes estimator coincide, and they even match those of optimal estimators taking into account the structure of the noise.}, author = {Barbier, Jean and Hou, TianQi and Mondelli, Marco and Saenz, Manuel}, booktitle = {36th Annual Conference on Neural Information Processing Systems}, isbn = {9781713871088}, location = {New Orleans, LA, United States}, title = {{The price of ignorance: How much does it cost to forget noise structure in low-rank matrix estimation?}}, volume = {35}, year = {2022}, } @inproceedings{12537, abstract = {The Neural Tangent Kernel (NTK) has emerged as a powerful tool to provide memorization, optimization and generalization guarantees in deep neural networks. A line of work has studied the NTK spectrum for two-layer and deep networks with at least a layer with Ω(N) neurons, N being the number of training samples. Furthermore, there is increasing evidence suggesting that deep networks with sub-linear layer widths are powerful memorizers and optimizers, as long as the number of parameters exceeds the number of samples. Thus, a natural open question is whether the NTK is well conditioned in such a challenging sub-linear setup. In this paper, we answer this question in the affirmative. Our key technical contribution is a lower bound on the smallest NTK eigenvalue for deep networks with the minimum possible over-parameterization: the number of parameters is roughly Ω(N) and, hence, the number of neurons is as little as Ω(N−−√). To showcase the applicability of our NTK bounds, we provide two results concerning memorization capacity and optimization guarantees for gradient descent training.}, author = {Bombari, Simone and Amani, Mohammad Hossein and Mondelli, Marco}, booktitle = {36th Conference on Neural Information Processing Systems}, isbn = {9781713871088}, issn = {1049-5258}, location = {New Orleans, LA, United States}, pages = {7628--7640}, publisher = {Neural Information Processing Systems Foundation}, title = {{Memorization and optimization in deep neural networks with minimum over-parameterization}}, volume = {35}, year = {2022}, } @article{12538, abstract = {In this paper, we study the compression of a target two-layer neural network with N nodes into a compressed network with M