[{"author":[{"first_name":"Yihan","full_name":"Zhang, Yihan","last_name":"Zhang"},{"full_name":"Mondelli, Marco","last_name":"Mondelli","orcid":"0000-0002-3242-7020","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco"},{"first_name":"Ramji","full_name":"Venkataramanan, Ramji","last_name":"Venkataramanan"}],"corr_author":"1","oa":1,"intvolume":" 8","publisher":"SIAM","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","has_accepted_license":"1","status":"public","quality_controlled":"1","year":"2026","title":"Precise asymptotics for spectral methods in mixed generalized linear models","mathsc":["62E20","62J05","62J12"],"date_updated":"2026-08-12T06:23:07Z","doi":"10.1137/24m1702854","PlanS_conform":"1","_id":"22228","external_id":{"arxiv":["2211.11368"]},"date_created":"2026-06-30T13:03:41Z","publication_status":"published","article_processing_charge":"Yes (in subscription journal)","month":"06","OA_place":"publisher","OA_type":"hybrid","department":[{"_id":"MaMo"}],"fulldoi":"https://doi.org/10.1137/24m1702854","das_tickbox":"0","arxiv":1,"date_published":"2026-06-01T00:00:00Z","type":"journal_article","page":"411-439","ddc":["000"],"publication":"SIAM Journal on Mathematics of Data Science","scopus_import":"1","article_type":"original","day":"01","abstract":[{"text":"In a mixed generalized linear model, the goal is to learn multiple signals from unlabeled observations: each sample comes from exactly one signal, but it is not known which one. We consider the prototypical problem of estimating two statistically independent signals in a mixed generalized linear model with Gaussian covariates. Spectral methods are a popular class of estimators which output the top two eigenvectors of a suitable data-dependent matrix. However, despite the wide applicability, their design is still obtained via heuristic considerations, and the number of samples 𝑛 needed to guarantee recovery is superlinear in the signal dimension 𝑑. In this paper, we develop exact asymptotics on spectral methods in the challenging proportional regime in which 𝑛,𝑑 grow large and their ratio converges to a finite constant. This allows us optimize the design of the spectral method, and combine it with a simple linear estimator, to minimize the estimation error. Our characterization exploits a mix of tools from random matrices, free probability, and the theory of approximate message passing algorithms. Numerical simulations for mixed linear regression and phase retrieval demonstrate the advantage enabled by our analysis over existing designs of spectral methods.","lang":"eng"}],"supplementarymaterial":"no","volume":8,"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"researchdata_availability":"no","publication_identifier":{"eissn":["2577-0187"]},"citation":{"ieee":"Y. Zhang, M. Mondelli, and R. Venkataramanan, “Precise asymptotics for spectral methods in mixed generalized linear models,” SIAM Journal on Mathematics of Data Science, vol. 8, no. 2. SIAM, pp. 411–439, 2026.","chicago":"Zhang, Yihan, Marco Mondelli, and Ramji Venkataramanan. “Precise Asymptotics for Spectral Methods in Mixed Generalized Linear Models.” SIAM Journal on Mathematics of Data Science. SIAM, 2026. https://doi.org/10.1137/24m1702854.","short":"Y. Zhang, M. Mondelli, R. Venkataramanan, SIAM Journal on Mathematics of Data Science 8 (2026) 411–439.","apa":"Zhang, Y., Mondelli, M., & Venkataramanan, R. (2026). Precise asymptotics for spectral methods in mixed generalized linear models. SIAM Journal on Mathematics of Data Science. SIAM. https://doi.org/10.1137/24m1702854","mla":"Zhang, Yihan, et al. “Precise Asymptotics for Spectral Methods in Mixed Generalized Linear Models.” SIAM Journal on Mathematics of Data Science, vol. 8, no. 2, SIAM, 2026, pp. 411–39, doi:10.1137/24m1702854.","ista":"Zhang Y, Mondelli M, Venkataramanan R. 2026. Precise asymptotics for spectral methods in mixed generalized linear models. SIAM Journal on Mathematics of Data Science. 8(2), 411–439.","ama":"Zhang Y, Mondelli M, Venkataramanan R. Precise asymptotics for spectral methods in mixed generalized linear models. SIAM Journal on Mathematics of Data Science. 2026;8(2):411-439. doi:10.1137/24m1702854"},"keyword":["spectral estimator","generalized linear models","mixed regression","high-dimensional asymptotics","random matrix theory","approximate message passing (AMP)"],"file_date_updated":"2026-07-01T06:22:15Z","file":[{"date_created":"2026-07-01T06:22:15Z","file_name":"2026_SIAMJourmathDataScience_Zhang.pdf","checksum":"5cfd350dc64d1476063e959316dbff65","content_type":"application/pdf","date_updated":"2026-07-01T06:22:15Z","access_level":"open_access","success":1,"creator":"dernst","file_size":1210346,"relation":"main_file","file_id":"22230"}],"acknowledgement":"The first and second authors were partially supported by the 2019 Lopez-Loreta prize.","issue":"2","language":[{"iso":"eng"}],"oa_version":"Published Version","tmp":{"short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png"}},{"type":"dissertation","date_published":"2026-10-08T00:00:00Z","related_material":{"record":[{"relation":"part_of_dissertation","id":"14459","status":"public"},{"relation":"part_of_dissertation","id":"12540","status":"public"},{"status":"public","id":"17469","relation":"part_of_dissertation"}]},"fulldoi":"https://doi.org/10.15479/AT-ISTA-23072","department":[{"_id":"MaMo"},{"_id":"GradSch"}],"OA_place":"publisher","article_processing_charge":"No","month":"10","publication_status":"published","date_created":"2026-10-07T11:20:26Z","supervisor":[{"orcid":"0000-0002-3242-7020","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","full_name":"Mondelli, Marco","last_name":"Mondelli"}],"_id":"23072","degree_awarded":"PhD","doi":"10.15479/AT-ISTA-23072","acknowledged_ssus":[{"_id":"ScienComp"}],"year":"2026","date_updated":"2026-10-12T13:04:49Z","title":"Compression and inference in high dimensions: An approximate message passing perspective","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","has_accepted_license":"1","status":"public","publisher":"Institute of Science and Technology Austria","corr_author":"1","author":[{"last_name":"Kögler","full_name":"Kögler, Kevin","id":"94ec913c-dc85-11ea-9058-e5051ab2428b","first_name":"Kevin"}],"oa":1,"oa_version":"Published Version","language":[{"iso":"eng"}],"acknowledgement":"This project was partially supported by the 2019 Lopez-Loreta Prize and\r\nthe ERC INF2 project (project number 101161364). Furthermore, the candidate acknowledges\r\nthe support from the Scientific Service Units of the Institute of Science and Technology Austria\r\nthrough resources provided by Scientific Computing.","file":[{"date_updated":"2026-10-09T12:22:55Z","access_level":"closed","creator":"kkoegler","relation":"source_file","file_id":"23084","file_size":8504109,"date_created":"2026-10-09T11:20:09Z","checksum":"5fb2ff02dc9d8c659b2fb5e0802b06ff","file_name":"2026_Koegler_Kevin_Thesis.zip","content_type":"application/zip"},{"relation":"main_file","file_id":"23085","file_size":6692605,"creator":"kkoegler","success":1,"access_level":"open_access","date_updated":"2026-10-09T13:25:49Z","content_type":"application/pdf","checksum":"a6df3b04c97628a1703f63356fb56f96","file_name":"2026_Koegler_Kevin_Thesis.pdf","date_created":"2026-10-09T13:25:49Z"}],"file_date_updated":"2026-10-09T13:25:49Z","citation":{"ama":"Kögler K. Compression and inference in high dimensions: An approximate message passing perspective. 2026. doi:10.15479/AT-ISTA-23072","ista":"Kögler K. 2026. Compression and inference in high dimensions: An approximate message passing perspective. Institute of Science and Technology Austria.","mla":"Kögler, Kevin. Compression and Inference in High Dimensions: An Approximate Message Passing Perspective. Institute of Science and Technology Austria, 2026, doi:10.15479/AT-ISTA-23072.","apa":"Kögler, K. (2026). Compression and inference in high dimensions: An approximate message passing perspective. Institute of Science and Technology Austria. https://doi.org/10.15479/AT-ISTA-23072","short":"K. Kögler, Compression and Inference in High Dimensions: An Approximate Message Passing Perspective, Institute of Science and Technology Austria, 2026.","chicago":"Kögler, Kevin. “Compression and Inference in High Dimensions: An Approximate Message Passing Perspective.” Institute of Science and Technology Austria, 2026. https://doi.org/10.15479/AT-ISTA-23072.","ieee":"K. Kögler, “Compression and inference in high dimensions: An approximate message passing perspective,” Institute of Science and Technology Austria, 2026."},"publication_identifier":{"issn":["2663-337X"]},"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"alternative_title":["ISTA Thesis"],"abstract":[{"lang":"eng","text":"Inferring signal from large quantities of data is one of the most important problems in today’s\r\nworld, with neural networks emerging as the most powerful solution. While there have been\r\nmassive breakthroughs in the ML space, there are two main drawbacks. The first is that\r\nmost of these models are “black boxes”, with very little understanding of why they work,\r\nwhich makes it hard to find the correct architecture for a given problem. The architecture is\r\ncrucial, as the widely used training algorithms are “local” and thus rely on the loss landscape\r\nbeing benign enough to find a global optimum. This brings us to the second problem: while\r\nthere is a lot of evidence that this can be overcome with large enough data, this is less clear\r\nfor datasets which are noisy, do not have enough samples, or have a non-standard structure.\r\nHistorically, on a smaller scale, the alternative has been simple (Bayesian) statistical methods,\r\nwhich also have the advantage of being interpretable and adaptable. However, many of the\r\nclassical results cannot be easily extended to the high-dimensional regime in which modern\r\ndata lives.\r\nIn this work we want to consider two different settings relating to the above. The first is\r\napproximate message passing (AMP), an iterative algorithm for high-dimensional (generalized)\r\nlinear models that easily lends itself to Bayesian setups, and which in many cases is provably\r\nBayes-optimal. The second is an autoencoder with a shallow encoder, for which we can\r\nanalyze the weights along the training trajectory of gradient descent. Surprisingly, we will find\r\na connection between the two, as we recover certain structures of AMP.\r\nWe will first introduce the AMP framework by studying an extension that can be applied\r\nto a wider variety of data: rotationally invariant generalized approximate message passing\r\n(RI-GAMP). This shows that one can extend the AMP framework to a much wider class of\r\nalgorithms, relevant to more practical settings. The results are then also confirmed on real\r\ndata that go beyond the theoretical setting.\r\nWe then move to the setup of an autoencoder with a shallow encoder and a Bayesian prior in\r\nthe high-dimensional regime, in which the input and compression dimensions are large and\r\nproportional; this is also a first step towards opening one of the black boxes above, as we\r\ncan say precisely when and why the network works. While autoencoders are widely used in\r\nmodern ML architectures, most fundamental research in this direction has focused on extreme\r\nasymptotic regimes, where the autoencoder is either severely over- or under-parameterized,\r\nand which do not extend to the proportional high-dimensional regime. We show that for\r\nGaussian priors the network finds the global optimum, and these theoretical findings extrapolate\r\nstrikingly well to real image data. Next we extend this analysis to more structured priors and\r\na deeper decoder. For sparse Gaussian inputs in the training regime studied here, the shallow\r\ndecoder does not exploit sparsity and yields the same dynamics as in the Gaussian setting.\r\nWe then show how a deeper decoder can break this behavior, and interestingly, if we treat the\r\nproblem as an AMP problem, the network eventually learns the same decoding as AMP. This\r\ntoo extrapolates to real image data.\r\nLastly, we will show that the AMP setup has real practical use, by using a practical variant,\r\ngenetic vector approximate message passing (gVAMP), for predicting plasma protein levels\r\nfrom genetic information using the UKB GWAS data. This is a perfect example of the problems\r\nin the high-dimensional regime: approximately 8.4 million dimensions and 40,000 samples,\r\nwhere the high noise, low sample count, and large size make it hard for ML methods to work.\r\nTo get the optimal prediction accuracy we consider a two-step inference. We first consider a\r\npre-processed version of the data, of about 880,000 dimensions, on which we run the gVAMP\r\nalgorithm; this allows us to find predictive regions, taking into account joint effects between\r\ndifferent regions, and for most proteins we end up selecting only about 0.1% of the dimensions.\r\nWe then consider the analogue of this small subset, on which we can now run a standard tree\r\nalgorithm (XGB) that no longer suffers from the aforementioned problems. This pipeline yields\r\nbetter prediction accuracy than recent nonlinear deep learning approaches applied to this data,\r\nas we are able to handle the pre-processing in a principled Bayesian way."}],"day":"08","doi_confirm":"1","ddc":["519"],"page":"222"},{"related_material":{"record":[{"relation":"part_of_dissertation","id":"21488","status":"public"}]},"date_published":"2026-07-11T00:00:00Z","type":"dissertation","fulldoi":"https://doi.org/10.15479/AT-ISTA-22258","das_tickbox":"1","publication_status":"published","department":[{"_id":"GradSch"},{"_id":"MaRo"},{"_id":"MaMo"}],"OA_place":"publisher","article_processing_charge":"No","month":"07","date_created":"2026-07-10T13:27:20Z","supervisor":[{"last_name":"Robinson","full_name":"Robinson, Matthew Richard","id":"E5D42276-F5DA-11E9-8E24-6303E6697425","first_name":"Matthew Richard","orcid":"0000-0001-8982-8813"},{"orcid":"0000-0002-3242-7020","first_name":"Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","full_name":"Mondelli, Marco","last_name":"Mondelli"}],"degree_awarded":"PhD","_id":"22258","status":"public","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","has_accepted_license":"1","doi":"10.15479/AT-ISTA-22258","title":"From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics","year":"2026","date_updated":"2026-10-20T09:03:21Z","acknowledged_ssus":[{"_id":"ScienComp"}],"corr_author":"1","oa":1,"author":[{"full_name":"Depope, Al","last_name":"Depope","first_name":"Al","id":"0b77531d-dbcd-11ea-9d1d-a8eee0bf3830"}],"publisher":"Institute of Science and Technology Austria","oa_version":"Published Version","acknowledgement":"This work was supported in part by the Swiss National Science Foundation through the\r\nEccellenza Grant \"Improving estimation and prediction of common complex disease risk\"\r\n(grant number PCEGP3_181181); the European Research Council through the grant\r\n\"Inference in High Dimensions: Light-speed Algorithms and Information Limits\" (grant\r\nnumber 101161364); and the Fondation Jean-Jacques et Felicia Lopez-Loreta through the\r\nPrix Lopez-Loretta 2019.\r\n","language":[{"iso":"eng"}],"file_date_updated":"2026-07-13T14:56:41Z","file":[{"date_created":"2026-07-13T14:52:19Z","checksum":"9ab386790515628d957a194f30a7ccb4","file_name":"2026_Depope_Al_Thesis.pdf","content_type":"application/pdf","date_updated":"2026-07-13T14:52:19Z","access_level":"open_access","creator":"adepope","file_id":"22316","relation":"main_file","file_size":25109878},{"content_type":"application/zip","checksum":"8ed8fb63f76a695d5b6fec35343f4b90","file_name":"2026_Depope_Al_Thesis.zip","date_created":"2026-07-13T14:56:41Z","file_id":"22317","relation":"source_file","file_size":1203199939,"creator":"adepope","access_level":"closed","date_updated":"2026-07-13T14:56:41Z"}],"publication_identifier":{"issn":["2663-337X"]},"keyword":["Approximate Message Passing","GWAS","Genomics","Proteomics","Survival modeling"],"citation":{"short":"A. Depope, From Sparse Selection to Risk Prediction: Approximate Message Passing for Proteomic Survival Models and Large-Scale Genomics, Institute of Science and Technology Austria, 2026.","apa":"Depope, A. (2026). From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics. Institute of Science and Technology Austria. https://doi.org/10.15479/AT-ISTA-22258","ieee":"A. Depope, “From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics,” Institute of Science and Technology Austria, 2026.","chicago":"Depope, Al. “From Sparse Selection to Risk Prediction: Approximate Message Passing for Proteomic Survival Models and Large-Scale Genomics.” Institute of Science and Technology Austria, 2026. https://doi.org/10.15479/AT-ISTA-22258.","ama":"Depope A. From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics. 2026. doi:10.15479/AT-ISTA-22258","mla":"Depope, Al. From Sparse Selection to Risk Prediction: Approximate Message Passing for Proteomic Survival Models and Large-Scale Genomics. Institute of Science and Technology Austria, 2026, doi:10.15479/AT-ISTA-22258.","ista":"Depope A. 2026. From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics. Institute of Science and Technology Austria."},"project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"},{"_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf","grant_number":"101161364","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits"},{"grant_number":"PCEGP3_181181","name":"Improving estimation and prediction of common complex disease risk","_id":"9B8D11D6-BA93-11EA-9121-9846C619BF3A"}],"day":"11","alternative_title":["ISTA Thesis"],"abstract":[{"lang":"eng","text":"Uncovering the genetic architecture of complex traits and pinpointing causal molecular drivers require the ability to distinguish true signals from noise within massive, high-dimensional omics datasets. To extract meaningful biological insights from these datasets, such as identifying causal genetic variants and proteins, scalable and accurate inference methods are essential. To this end, this thesis develops novel Bayesian inference frameworks based on Vector Approximate Message Passing and demonstrates their effectiveness in the modeling of disease onset times and quantitative physical and clinical measures.\r\n\r\nFirst, we introduce gVAMP, a Bayesian framework tailored for Genome-Wide Association Studies that enables the joint modeling of quantitative complex traits across millions of genetic variants. gVAMP demonstrates superior accuracy in variable selection and out-of-sample polygenic risk prediction compared to state-of-the-art approaches. We model human height using 17 million whole-genome sequence variants from the UK Biobank, incorporating a vast number of rare variants and revealing novel associations. gVAMP achieves a prediction accuracy of approximately 46% for human height, representing the highest reported performance for this trait to date. \r\n\r\nSecond, we present vampW, a Bayesian framework for survival analysis applied to proteomic data. By effectively handling right-censoring and complex protein dependencies within the UK Biobank Pharma Proteomics Project dataset, vampW identifies 219 protein associations across 24 disease outcomes, the majority of which are not among the top marginal discoveries. We further adjust protein levels for exponential age effects, yielding 1,308 associations and highlighting the sensitivity of the analysis to the chosen age-correction methodology. Finally, vampW improves upon the variable selection capabilities of the commonly used (penalized) variants of the Cox proportional hazards model and delivers state-of-the-art out-of-sample prediction of disease onset times.\r\n\r\nCollectively, these methods provide powerful tools for dissecting the genetic architecture of complex traits and the proteomic drivers of disease onset. Furthermore, by delivering accurate polygenic risk scores and precise predictions of onset times, this work advances the capabilities of personalized medicine and clinical risk stratification."}],"doi_confirm":"1","page":"169","ddc":["576","610","006"]},{"researchdata_availability":"yes","project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"},{"grant_number":"101161364","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf"},{"_id":"9B8D11D6-BA93-11EA-9121-9846C619BF3A","grant_number":"PCEGP3_181181","name":"Improving estimation and prediction of common complex disease risk"}],"abstract":[{"lang":"eng","text":"Human height is a model for the genetic analysis of complex traits, and recent studies suggest the presence of thousands of common genetic variant associations and hundreds of low-frequency/rare variants. Here, we develop a new algorithmic paradigm based on approximate message passing (genomic vector approximate message passing [gVAMP]) for identifying DNA sequence variants associated with complex traits and common diseases in large-scale whole-genome sequencing (WGS) data. We show that gVAMP accurately localizes associations to variants with the correct frequency and position in the DNA, outperforming existing fine-mapping methods in selecting the appropriate genetic variants within WGS data. We then apply gVAMP to jointly model the relationship of tens of millions of WGS variants with human height in hundreds of thousands of UK Biobank individuals. We identify 59 rare variants and gene burden scores alongside many hundreds of DNA regions containing common variant associations and show that understanding the genetic basis of complex traits will require the joint analysis of hundreds of millions of variables measured on millions of people. The polygenic risk scores obtained from gVAMP have high accuracy (including a prediction accuracy of ∌46% for human height) and outperform current methods for downstream tasks such as mixed linear model association testing across 13 UK Biobank traits. In conclusion, gVAMP offers a scalable foundation for a wider range of analyses in WGS data."}],"supplementarymaterial":"yes","volume":6,"scopus_import":"1","article_type":"original","day":"13","publication":"Cell Genomics","ddc":["000","570"],"oa_version":"Published Version","tmp":{"image":"/images/cc_by_nc_nd.png","legal_code_url":"https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode","short":"CC BY-NC-ND (4.0)","name":"Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International (CC BY-NC-ND 4.0)"},"language":[{"iso":"eng"}],"acknowledgement":"We thank Malgorzata Borczyk for creating the gene burden scores. We thank Robin Beaumont, Amedeo Roberto Esposito, Gareth Hawkes, Philip Schniter, Matthew Stephens, Pragya Sur, Peter Visscher, Michael Weedon, and Harry Wright for providing valuable suggestions and comments on earlier versions of the work. This project was funded by a Lopez-Loreta Prize to M.M., an SNSF Eccellenza Grant to M.R.R. (PCEGP3-181181), an ERC Starting Grant to M.M. (INF2, project number 101161364), and core funding from ISTA. High-performance computing was supported by the Scientific Service Units (SSU) of ISTA through resources provided by Scientific Computing (SciComp). We would like to acknowledge the participants and investigators of the UK Biobank study. We gratefully acknowledge the All of Us participants for their contributions, without whom this research would not have been possible. We also thank the National Institutes of Health All of Us Research Program for making available the participant data (and/or samples and/or cohort) examined in this study.","issue":"5","file":[{"success":1,"access_level":"open_access","date_updated":"2026-07-28T07:06:26Z","relation":"main_file","file_id":"22596","file_size":3736705,"creator":"dernst","checksum":"6b59686f8d9733add4f23d23f3dd9df0","file_name":"2026_CellGenomics_Depope.pdf","date_created":"2026-07-28T07:06:26Z","content_type":"application/pdf"}],"file_date_updated":"2026-07-28T07:06:26Z","citation":{"ama":"Depope A, Bajzik J, Mondelli M, Robinson MR. Joint modeling of whole-genome sequencing data for human height via approximate message passing. Cell Genomics. 2026;6(5). doi:10.1016/j.xgen.2026.101162","mla":"Depope, Al, et al. “Joint Modeling of Whole-Genome Sequencing Data for Human Height via Approximate Message Passing.” Cell Genomics, vol. 6, no. 5, 101162, Cell Press, 2026, doi:10.1016/j.xgen.2026.101162.","ista":"Depope A, Bajzik J, Mondelli M, Robinson MR. 2026. Joint modeling of whole-genome sequencing data for human height via approximate message passing. Cell Genomics. 6(5), 101162.","short":"A. Depope, J. Bajzik, M. Mondelli, M.R. Robinson, Cell Genomics 6 (2026).","apa":"Depope, A., Bajzik, J., Mondelli, M., & Robinson, M. R. (2026). Joint modeling of whole-genome sequencing data for human height via approximate message passing. Cell Genomics. Cell Press. https://doi.org/10.1016/j.xgen.2026.101162","ieee":"A. Depope, J. Bajzik, M. Mondelli, and M. R. Robinson, “Joint modeling of whole-genome sequencing data for human height via approximate message passing,” Cell Genomics, vol. 6, no. 5. Cell Press, 2026.","chicago":"Depope, Al, Jakub Bajzik, Marco Mondelli, and Matthew Richard Robinson. “Joint Modeling of Whole-Genome Sequencing Data for Human Height via Approximate Message Passing.” Cell Genomics. Cell Press, 2026. https://doi.org/10.1016/j.xgen.2026.101162."},"publication_identifier":{"eissn":["2666-979X"]},"_id":"21488","year":"2026","date_updated":"2026-10-20T09:03:22Z","title":"Joint modeling of whole-genome sequencing data for human height via approximate message passing","doi":"10.1016/j.xgen.2026.101162","has_accepted_license":"1","status":"public","quality_controlled":"1","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","intvolume":" 6","publisher":"Cell Press","author":[{"id":"0b77531d-dbcd-11ea-9d1d-a8eee0bf3830","first_name":"Al","full_name":"Depope, Al","last_name":"Depope"},{"first_name":"Jakub","id":"b995e25b-8c4b-11ed-a6d8-f71b7bcd6122","last_name":"Bajzik","full_name":"Bajzik, Jakub"},{"first_name":"Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","orcid":"0000-0002-3242-7020","last_name":"Mondelli","full_name":"Mondelli, Marco"},{"first_name":"Matthew Richard","id":"E5D42276-F5DA-11E9-8E24-6303E6697425","orcid":"0000-0001-8982-8813","last_name":"Robinson","full_name":"Robinson, Matthew Richard"}],"corr_author":"1","oa":1,"DOAJ_listed":"1","type":"journal_article","date_published":"2026-05-13T00:00:00Z","pmid":1,"related_material":{"record":[{"relation":"dissertation_contains","id":"22258","status":"public"}],"link":[{"url":"https://ista.ac.at/en/news/big-data-and-human-height/","relation":"press_release","description":"News on ISTA website"}]},"das_tickbox":"1","fulldoi":"https://doi.org/10.1016/j.xgen.2026.101162","month":"05","OA_place":"publisher","article_processing_charge":"Yes","OA_type":"gold","department":[{"_id":"MaMo"},{"_id":"MaRo"}],"publication_status":"published","date_created":"2026-03-23T15:10:03Z","dataavailabilitystatement":"This project uses the UK Biobank data under project number 35520. UK Biobank genotypic and phenotypic data are available through a formal request at http://www.ukbiobank.ac.uk. It also uses genotypic and phenotypic data from the All of Us study, which are also available through a formal request at https://www.researchallofus.org/data-tools/data-access/. All summary statistic estimates are released publicly on Dryad: https://doi.org/10.5061/dryad.cz8w9gjjc.\r\n‱\r\nThe gVAMP code developed in this work is open source and has been deposited on GitHub, where it is publicly available at https://github.com/medical-genomics-group/gVAMP, and the code used to generate the data in the manuscript are available from Zenodo https://doi.org/10.5281/zenodo.17935521. The URLs of other software used are listed in the key resources table.","external_id":{"pmid":["41713425"]},"article_number":"101162"},{"file_date_updated":"2025-02-03T08:27:59Z","file":[{"checksum":"52c5f72d80ffc928542469114fcdb62b","file_name":"2025_PhysReviewResearch_Barbier.pdf","date_created":"2025-02-03T08:27:59Z","content_type":"application/pdf","success":1,"access_level":"open_access","date_updated":"2025-02-03T08:27:59Z","relation":"main_file","file_id":"18988","file_size":702543,"creator":"dernst"}],"publication_identifier":{"issn":["2643-1564"]},"citation":{"short":"J. Barbier, F. Camilli, Y. Xu, M. Mondelli, Physical Review Research 7 (2025).","apa":"Barbier, J., Camilli, F., Xu, Y., & Mondelli, M. (2025). Information limits and Thouless-Anderson-Palmer equations for spiked matrix models with structured noise. Physical Review Research. American Physical Society. https://doi.org/10.1103/PhysRevResearch.7.013081","ieee":"J. Barbier, F. Camilli, Y. Xu, and M. Mondelli, “Information limits and Thouless-Anderson-Palmer equations for spiked matrix models with structured noise,” Physical Review Research, vol. 7. American Physical Society, 2025.","chicago":"Barbier, Jean, Francesco Camilli, Yizhou Xu, and Marco Mondelli. “Information Limits and Thouless-Anderson-Palmer Equations for Spiked Matrix Models with Structured Noise.” Physical Review Research. American Physical Society, 2025. https://doi.org/10.1103/PhysRevResearch.7.013081.","ama":"Barbier J, Camilli F, Xu Y, Mondelli M. Information limits and Thouless-Anderson-Palmer equations for spiked matrix models with structured noise. Physical Review Research. 2025;7. doi:10.1103/PhysRevResearch.7.013081","mla":"Barbier, Jean, et al. “Information Limits and Thouless-Anderson-Palmer Equations for Spiked Matrix Models with Structured Noise.” Physical Review Research, vol. 7, 013081, American Physical Society, 2025, doi:10.1103/PhysRevResearch.7.013081.","ista":"Barbier J, Camilli F, Xu Y, Mondelli M. 2025. Information limits and Thouless-Anderson-Palmer equations for spiked matrix models with structured noise. Physical Review Research. 7, 013081."},"oa_version":"Published Version","tmp":{"short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png"},"acknowledgement":"J.B., F.C., and Y.X. were funded by the European Union (ERC, CHORAL, Project No. 101039794). Views and opinions expressed are however those of the authors only and do not necessarily reflect those of the European Union or the European Research Council. Neither the European Union nor the granting authority can be held responsible for them. M.M. was supported by the 2019 Lopez-Loreta Prize. J.B. acknowledges discussions with TianQi Hou at the initial stage of the project, as well as with Antoine Bodin.","language":[{"iso":"eng"}],"publication":"Physical Review Research","ddc":["530"],"project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"}],"scopus_import":"1","article_type":"original","day":"22","abstract":[{"lang":"eng","text":"We consider a prototypical problem of Bayesian inference for a structured spiked model: a low-rank signal is corrupted by additive noise. While both information-theoretic and algorithmic limits are well understood when the noise is a Gaussian Wigner matrix, the more realistic case of structured noise still remains challenging. To capture the structure while maintaining mathematical tractability, a line of work has focused on rotationally invariant noise. However, existing studies either provide suboptimal algorithms or are limited to a special class of noise ensembles. In this paper, using tools from statistical physics (replica method) and random matrix theory (generalized spherical integrals) we establish the characterization of the information-theoretic limits for a noise matrix drawn from a general trace ensemble. Remarkably, our analysis unveils the asymptotic equivalence between the rotationally invariant model and a surrogate Gaussian one. Finally, we show how to saturate the predicted statistical limits using an efficient algorithm inspired by the theory of adaptive Thouless-Anderson-Palmer (TAP) equations."}],"volume":7,"publication_status":"published","article_processing_charge":"Yes","month":"01","OA_place":"publisher","OA_type":"gold","department":[{"_id":"MaMo"}],"external_id":{"arxiv":["2405.20993"]},"article_number":"013081","date_created":"2025-02-02T23:01:54Z","arxiv":1,"related_material":{"link":[{"url":"https://github.com/xu-yz19/spiked-matrix-models-with-structured-noise","relation":"software"}]},"DOAJ_listed":"1","type":"journal_article","date_published":"2025-01-22T00:00:00Z","fulldoi":"https://doi.org/10.1103/PhysRevResearch.7.013081","APC_amount":"3272,21 EUR","author":[{"first_name":"Jean","last_name":"Barbier","full_name":"Barbier, Jean"},{"first_name":"Francesco","full_name":"Camilli, Francesco","last_name":"Camilli"},{"first_name":"Yizhou","full_name":"Xu, Yizhou","last_name":"Xu"},{"full_name":"Mondelli, Marco","last_name":"Mondelli","orcid":"0000-0002-3242-7020","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco"}],"oa":1,"intvolume":" 7","publisher":"American Physical Society","_id":"18986","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","status":"public","has_accepted_license":"1","quality_controlled":"1","title":"Information limits and Thouless-Anderson-Palmer equations for spiked matrix models with structured noise","year":"2025","date_updated":"2026-05-06T12:57:36Z","doi":"10.1103/PhysRevResearch.7.013081"},{"author":[{"last_name":"Zhang","full_name":"Zhang, Yihan","id":"2ce5da42-b2ea-11eb-bba5-9f264e9d002c","first_name":"Yihan","orcid":"0000-0002-6465-6258"},{"first_name":"Hong Chang","full_name":"Ji, Hong Chang","last_name":"Ji"},{"first_name":"Ramji","full_name":"Venkataramanan, Ramji","last_name":"Venkataramanan"},{"id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","orcid":"0000-0002-3242-7020","last_name":"Mondelli","full_name":"Mondelli, Marco"}],"corr_author":"1","oa":1,"publisher":"EMS Press","intvolume":" 8","quality_controlled":"1","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","has_accepted_license":"1","status":"public","doi":"10.4171/MSL/52","title":"Spectral estimators for structured generalized linear models via approximate message passing","date_updated":"2025-12-09T13:53:31Z","year":"2025","_id":"20734","PlanS_conform":"1","date_created":"2025-12-07T23:02:02Z","publication_status":"published","department":[{"_id":"MaMo"}],"OA_type":"diamond","OA_place":"publisher","month":"09","article_processing_charge":"No","fulldoi":"https://doi.org/10.4171/MSL/52","date_published":"2025-09-02T00:00:00Z","type":"journal_article","page":"193-304","publication":"Mathematical Statistics and Learning","ddc":["000"],"article_type":"original","day":"02","scopus_import":"1","volume":8,"abstract":[{"text":"We consider the problem of parameter estimation in a high-dimensional generalized linear model. Spectral methods obtained via the principal eigenvector of a suitable data-dependent matrix provide a simple yet surprisingly effective solution. However, despite their wide use, a rigorous performance characterization, as well as a principled way to preprocess the data, are available only for unstructured (i.i.d. Gaussian and Haar orthogonal) designs. In contrast, real-world data matrices are highly structured and exhibit non-trivial correlations. To address the problem, we consider correlated Gaussian designs capturing the anisotropic nature of the features via a covariance matrix ÎŁ. Our main result is a precise asymptotic characterization of the performance of spectral estimators. This allows us to identify the optimal preprocessing that minimizes the number of samples needed for parameter estimation. Surprisingly, such preprocessing is universal across a broad set of designs, which partly addresses a conjecture on optimal spectral estimators for rotationally invariant models. Our principled approach vastly improves upon previous heuristic methods, including for designs common in computational imaging and genetics. The proposed methodology, based on approximate message passing, is broadly applicable and opens the way to the precise characterization of spiked matrices and of the corresponding spectral methods in a variety of settings.","lang":"eng"}],"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"publication_identifier":{"eissn":["2520-2324"],"issn":["2520-2316"]},"citation":{"ama":"Zhang Y, Ji HC, Venkataramanan R, Mondelli M. Spectral estimators for structured generalized linear models via approximate message passing. Mathematical Statistics and Learning. 2025;8(3-4):193-304. doi:10.4171/MSL/52","ista":"Zhang Y, Ji HC, Venkataramanan R, Mondelli M. 2025. Spectral estimators for structured generalized linear models via approximate message passing. Mathematical Statistics and Learning. 8(3–4), 193–304.","mla":"Zhang, Yihan, et al. “Spectral Estimators for Structured Generalized Linear Models via Approximate Message Passing.” Mathematical Statistics and Learning, vol. 8, no. 3–4, EMS Press, 2025, pp. 193–304, doi:10.4171/MSL/52.","apa":"Zhang, Y., Ji, H. C., Venkataramanan, R., & Mondelli, M. (2025). Spectral estimators for structured generalized linear models via approximate message passing. Mathematical Statistics and Learning. EMS Press. https://doi.org/10.4171/MSL/52","short":"Y. Zhang, H.C. Ji, R. Venkataramanan, M. Mondelli, Mathematical Statistics and Learning 8 (2025) 193–304.","chicago":"Zhang, Yihan, Hong Chang Ji, Ramji Venkataramanan, and Marco Mondelli. “Spectral Estimators for Structured Generalized Linear Models via Approximate Message Passing.” Mathematical Statistics and Learning. EMS Press, 2025. https://doi.org/10.4171/MSL/52.","ieee":"Y. Zhang, H. C. Ji, R. Venkataramanan, and M. Mondelli, “Spectral estimators for structured generalized linear models via approximate message passing,” Mathematical Statistics and Learning, vol. 8, no. 3–4. EMS Press, pp. 193–304, 2025."},"file_date_updated":"2025-12-09T13:50:03Z","file":[{"content_type":"application/pdf","date_created":"2025-12-09T13:50:03Z","checksum":"55a1bd9c1b6b0198c42504fb94f4ad4c","file_name":"2025_MathStatLearning_Zhang.pdf","creator":"dernst","relation":"main_file","file_id":"20752","file_size":1379626,"date_updated":"2025-12-09T13:50:03Z","success":1,"access_level":"open_access"}],"issue":"3-4","acknowledgement":"This work was done when Y. Z. and H. C. J. were at the Institute of Science and Technology Austria. Y. Z. thanks Hugo Latourelle-Vigeant for bringing [53] to the authors’ attention.\r\nY. Z. and M. M. are partially supported by the 2019 Lopez-Loreta Prize and by the Interdisciplinary Projects Committee (IPC) at ISTA. H. C. J. is supported by the ERC Advanced Grant “RMTBeyond” No. 101020331.","language":[{"iso":"eng"}],"tmp":{"short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png"},"oa_version":"Published Version"},{"department":[{"_id":"MaMo"}],"OA_type":"hybrid","article_processing_charge":"Yes (in subscription journal)","OA_place":"publisher","month":"04","publication_status":"published","date_created":"2025-04-27T22:02:13Z","article_number":"e2423072122","external_id":{"isi":["001471214000001"],"pmid":["40215275"],"arxiv":["2410.14787"]},"type":"journal_article","date_published":"2025-04-15T00:00:00Z","related_material":{"record":[{"status":"public","id":"22857","relation":"dissertation_contains"}]},"arxiv":1,"pmid":1,"fulldoi":"https://doi.org/10.1073/pnas.2423072122","intvolume":" 122","publisher":"National Academy of Sciences","author":[{"id":"ca726dda-de17-11ea-bc14-f9da834f63aa","first_name":"Simone","full_name":"Bombari, Simone","last_name":"Bombari"},{"first_name":"Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","orcid":"0000-0002-3242-7020","last_name":"Mondelli","full_name":"Mondelli, Marco"}],"corr_author":"1","oa":1,"APC_amount":"2754,32 EUR","_id":"19627","isi":1,"doi":"10.1073/pnas.2423072122","date_updated":"2026-09-21T13:07:01Z","title":"Privacy for free in the overparameterized regime","year":"2025","has_accepted_license":"1","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","status":"public","quality_controlled":"1","file":[{"access_level":"open_access","success":1,"date_updated":"2025-05-05T07:27:54Z","file_size":2328320,"file_id":"19648","relation":"main_file","creator":"dernst","file_name":"2025_PNAS_Bombari.pdf","checksum":"1ac6f78e368d35a0cafb4d2d9bd63443","date_created":"2025-05-05T07:27:54Z","content_type":"application/pdf"}],"file_date_updated":"2025-05-05T07:27:54Z","citation":{"ieee":"S. Bombari and M. Mondelli, “Privacy for free in the overparameterized regime,” Proceedings of the National Academy of Sciences, vol. 122, no. 15. National Academy of Sciences, 2025.","chicago":"Bombari, Simone, and Marco Mondelli. “Privacy for Free in the Overparameterized Regime.” Proceedings of the National Academy of Sciences. National Academy of Sciences, 2025. https://doi.org/10.1073/pnas.2423072122.","apa":"Bombari, S., & Mondelli, M. (2025). Privacy for free in the overparameterized regime. Proceedings of the National Academy of Sciences. National Academy of Sciences. https://doi.org/10.1073/pnas.2423072122","short":"S. Bombari, M. Mondelli, Proceedings of the National Academy of Sciences 122 (2025).","ista":"Bombari S, Mondelli M. 2025. Privacy for free in the overparameterized regime. Proceedings of the National Academy of Sciences. 122(15), e2423072122.","mla":"Bombari, Simone, and Marco Mondelli. “Privacy for Free in the Overparameterized Regime.” Proceedings of the National Academy of Sciences, vol. 122, no. 15, e2423072122, National Academy of Sciences, 2025, doi:10.1073/pnas.2423072122.","ama":"Bombari S, Mondelli M. Privacy for free in the overparameterized regime. Proceedings of the National Academy of Sciences. 2025;122(15). doi:10.1073/pnas.2423072122"},"publication_identifier":{"eissn":["1091-6490"],"issn":["0027-8424"]},"tmp":{"short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png"},"oa_version":"Published Version","language":[{"iso":"eng"}],"issue":"15","acknowledgement":"This research was funded in whole, or in part, by the Austrian Science Fund (FWF) Grant number COE 12. For the purpose of open access, the author has applied a CC BY public copyright license to any Author Accepted Manuscript version arising from this submission. The authors were also supported by the 2019 Lopez-Loreta prize, and Simone Bombari was supported by a Google PhD fellowship. We thank Diyuan Wu, Edwige Cyffers, Francesco Pedrotti, Inbar Seroussi, Nikita P. Kalinin, Pietro Pelliconi, Roodabeh Safavi, Yizhe Zhu, and Zhichao Wang for helpful discussions.","publication":"Proceedings of the National Academy of Sciences","ddc":["000"],"project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"},{"_id":"92099302-16d5-11f0-9cad-f9a785f54fbd","name":"Trustworthy Deep Learning Theory: Private Over-Parameterized Models and Robust LLMs"},{"_id":"74caaef7-b034-11f1-8f2d-e0e993bb422e","grant_number":"COE12","name":"Bilateral Artificial Intelligence (Mondelli)"}],"volume":122,"abstract":[{"lang":"eng","text":"Differentially private gradient descent (DP-GD) is a popular algorithm to train deep learning models with provable guarantees on the privacy of the training data. In the last decade, the problem of understanding its performance cost with respect to standard GD has received remarkable attention from the research community, which formally derived upper bounds on the excess population risk RP in different learning settings. However, existing bounds typically degrade with over-parameterization, i.e., as the number of parameters p gets larger than the number of training samples n -- a regime which is ubiquitous in current deep-learning practice. As a result, the lack of theoretical insights leaves practitioners without clear guidance, leading some to reduce the effective number of trainable parameters to improve performance, while others use larger models to achieve better results through scale. In this work, we show that in the popular random features model with quadratic loss, for any sufficiently large p , privacy can be obtained for free, i.e., |RP|=o(1) , not only when the privacy parameter Δ has constant order, but also in the strongly private setting Δ=o(1) . This challenges the common wisdom that over-parameterization inherently hinders performance in private learning."}],"article_type":"original","day":"15","scopus_import":"1"},{"user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","has_accepted_license":"1","status":"public","quality_controlled":"1","title":"Neural collapse versus low-rank bias: Is deep neural collapse really optimal?","date_updated":"2026-10-02T11:34:35Z","acknowledged_ssus":[{"_id":"ScienComp"}],"year":"2024","_id":"18891","oa":1,"corr_author":"1","author":[{"last_name":"SĂșkenĂ­k","full_name":"SĂșkenĂ­k, Peter","id":"d64d6a8d-eb8e-11eb-b029-96fd216dec3c","first_name":"Peter"},{"first_name":"Christoph","id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0001-8622-7887","last_name":"Lampert","full_name":"Lampert, Christoph"},{"orcid":"0000-0002-3242-7020","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","full_name":"Mondelli, Marco","last_name":"Mondelli"}],"intvolume":" 37","publisher":"Neural Information Processing Systems Foundation","das_tickbox":"0","arxiv":1,"date_published":"2024-12-01T00:00:00Z","type":"conference","conference":{"location":"Vancouver, Canada","start_date":"2024-12-16","name":"NeurIPS: Neural Information Processing Systems","end_date":"2024-12-16"},"external_id":{"arxiv":["2405.14468"]},"date_created":"2025-01-27T11:15:18Z","publication_status":"published","department":[{"_id":"GradSch"},{"_id":"MaMo"},{"_id":"ChLa"}],"OA_type":"gold","article_processing_charge":"No","OA_place":"publisher","month":"12","day":"01","alternative_title":["Advances in Neural Information Processing Systems"],"volume":37,"supplementarymaterial":"no","abstract":[{"text":"Deep neural networks (DNNs) exhibit a surprising structure in their final layer\r\nknown as neural collapse (NC), and a growing body of works has currently investigated the propagation of neural collapse to earlier layers of DNNs – a phenomenon\r\ncalled deep neural collapse (DNC). However, existing theoretical results are restricted to special cases: linear models, only two layers or binary classification.\r\nIn contrast, we focus on non-linear models of arbitrary depth in multi-class classification and reveal a surprising qualitative shift. As soon as we go beyond two\r\nlayers or two classes, DNC stops being optimal for the deep unconstrained features\r\nmodel (DUFM) – the standard theoretical framework for the analysis of collapse.\r\nThe main culprit is a low-rank bias of multi-layer regularization schemes: this bias\r\nleads to optimal solutions of even lower rank than the neural collapse. We support\r\nour theoretical findings with experiments on both DUFM and real data, which show\r\nthe emergence of the low-rank structure in the solution found by gradient descent.","lang":"eng"}],"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"researchdata_availability":"no","publication":"38th Annual Conference on Neural Information Processing Systems","ddc":["000"],"acknowledgement":"Marco Mondelli is partially supported by the 2019 Lopez-Loreta prize. This research was supported by the Scientific Service Units (SSU) of ISTA through resources provided by Scientific Computing (SciComp).","language":[{"iso":"eng"}],"tmp":{"short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png"},"oa_version":"Published Version","citation":{"ista":"SĂșkenĂ­k P, Lampert C, Mondelli M. 2024. Neural collapse versus low-rank bias: Is deep neural collapse really optimal? 38th Annual Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 37.","mla":"SĂșkenĂ­k, Peter, et al. “Neural Collapse versus Low-Rank Bias: Is Deep Neural Collapse Really Optimal?” 38th Annual Conference on Neural Information Processing Systems, vol. 37, Neural Information Processing Systems Foundation, 2024.","ama":"SĂșkenĂ­k P, Lampert C, Mondelli M. Neural collapse versus low-rank bias: Is deep neural collapse really optimal? In: 38th Annual Conference on Neural Information Processing Systems. Vol 37. Neural Information Processing Systems Foundation; 2024.","ieee":"P. SĂșkenĂ­k, C. Lampert, and M. Mondelli, “Neural collapse versus low-rank bias: Is deep neural collapse really optimal?,” in 38th Annual Conference on Neural Information Processing Systems, Vancouver, Canada, 2024, vol. 37.","chicago":"SĂșkenĂ­k, Peter, Christoph Lampert, and Marco Mondelli. “Neural Collapse versus Low-Rank Bias: Is Deep Neural Collapse Really Optimal?” In 38th Annual Conference on Neural Information Processing Systems, Vol. 37. Neural Information Processing Systems Foundation, 2024.","apa":"SĂșkenĂ­k, P., Lampert, C., & Mondelli, M. (2024). Neural collapse versus low-rank bias: Is deep neural collapse really optimal? In 38th Annual Conference on Neural Information Processing Systems (Vol. 37). Vancouver, Canada: Neural Information Processing Systems Foundation.","short":"P. SĂșkenĂ­k, C. Lampert, M. Mondelli, in:, 38th Annual Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2024."},"file_date_updated":"2025-02-04T08:11:25Z","file":[{"content_type":"application/pdf","checksum":"b7b79f1ea3ac1e9e11b3d91faaeb0780","file_name":"2024_NeurIPS_Sukenik.pdf","date_created":"2025-02-04T08:11:25Z","relation":"main_file","file_id":"18989","file_size":1784118,"creator":"dernst","success":1,"access_level":"open_access","date_updated":"2025-02-04T08:11:25Z"}]},{"citation":{"ista":"Pedrotti F, Maas J, Mondelli M. 2024. Improved convergence of score-based diffusion models via prediction-correction. Transactions on Machine Learning Research. , TMLR, .","mla":"Pedrotti, Francesco, et al. “Improved Convergence of Score-Based Diffusion Models via Prediction-Correction.” Transactions on Machine Learning Research, 2024.","ama":"Pedrotti F, Maas J, Mondelli M. Improved convergence of score-based diffusion models via prediction-correction. In: Transactions on Machine Learning Research. ; 2024.","chicago":"Pedrotti, Francesco, Jan Maas, and Marco Mondelli. “Improved Convergence of Score-Based Diffusion Models via Prediction-Correction.” In Transactions on Machine Learning Research, 2024.","ieee":"F. Pedrotti, J. Maas, and M. Mondelli, “Improved convergence of score-based diffusion models via prediction-correction,” in Transactions on Machine Learning Research, 2024.","apa":"Pedrotti, F., Maas, J., & Mondelli, M. (2024). Improved convergence of score-based diffusion models via prediction-correction. In Transactions on Machine Learning Research.","short":"F. Pedrotti, J. Maas, M. Mondelli, in:, Transactions on Machine Learning Research, 2024."},"publication_identifier":{"issn":["2835-8856"]},"file":[{"date_updated":"2025-01-27T12:19:44Z","access_level":"open_access","success":1,"creator":"dernst","file_size":780315,"relation":"main_file","file_id":"18898","date_created":"2025-01-27T12:19:44Z","file_name":"2024_TMLR_Pedrotti.pdf","checksum":"76a1fd5afd8ee6f7ae0e5912d7dbf6b4","content_type":"application/pdf"}],"file_date_updated":"2025-01-27T12:19:44Z","language":[{"iso":"eng"}],"acknowledgement":"Francesco Pedrotti and Jan Maas acknowledge support by the Austrian Science Fund (FWF) project 10.55776/F65. Marco Mondelli acknowledges support by the 2019 Lopez-Loreta prize.\r\n","oa_version":"Published Version","tmp":{"short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png"},"publication":"Transactions on Machine Learning Research","ddc":["000"],"abstract":[{"lang":"eng","text":"Score-based generative models (SGMs) are powerful tools to sample from complex data distributions. Their underlying idea is to (i) run a forward process for time T1 by adding noise to the data, (ii) estimate its score function, and (iii) use such estimate to run a reverse process. As the reverse process is initialized with the stationary distribution of the forward one, the existing analysis paradigm requires T1→∞. This is however problematic: from a theoretical viewpoint, for a given precision of the score approximation, the convergence guarantee fails as T1 diverges; from a practical viewpoint, a large T1 increases computational costs and leads to error propagation. This paper addresses the issue by considering a version of the popular predictor-corrector scheme: after running the forward process, we first estimate the final distribution via an inexact Langevin dynamics and then revert the process. Our key technical contribution is to provide convergence guarantees which require to run the forward process only for a fixed finite time T1. Our bounds exhibit a mild logarithmic dependence on the input dimension and the subgaussian norm of the target distribution, have minimal assumptions on the data, and require only to control the L2 loss on the score approximation, which is the quantity minimized in practice."}],"alternative_title":["TMLR"],"supplementarymaterial":"no","scopus_import":"1","day":"01","researchdata_availability":"no","project":[{"_id":"fc31cba2-9c52-11eb-aca3-ff467d239cd2","grant_number":"F6504","name":"Taming Complexity in Partial Differential Systems"},{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"}],"date_created":"2025-01-27T12:18:05Z","external_id":{"arxiv":["2305.14164"]},"article_processing_charge":"No","OA_place":"publisher","month":"06","department":[{"_id":"JaMa"},{"_id":"MaMo"}],"OA_type":"gold","publication_status":"published","das_tickbox":"0","date_published":"2024-06-01T00:00:00Z","type":"conference","arxiv":1,"related_material":{"record":[{"status":"public","relation":"earlier_version","id":"17350"}]},"oa":1,"author":[{"first_name":"Francesco","id":"d3ac8ac6-dc8d-11ea-abe3-e2a9628c4c3c","full_name":"Pedrotti, Francesco","last_name":"Pedrotti"},{"orcid":"0000-0002-0845-1338","first_name":"Jan","id":"4C5696CE-F248-11E8-B48F-1D18A9856A87","full_name":"Maas, Jan","last_name":"Maas"},{"first_name":"Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","orcid":"0000-0002-3242-7020","last_name":"Mondelli","full_name":"Mondelli, Marco"}],"corr_author":"1","year":"2024","date_updated":"2026-10-02T12:22:41Z","title":"Improved convergence of score-based diffusion models via prediction-correction","status":"public","has_accepted_license":"1","quality_controlled":"1","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","_id":"18897"},{"external_id":{"arxiv":["2305.14164"]},"date_created":"2024-07-31T07:56:40Z","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2305.14164"}],"publication_status":"draft","article_processing_charge":"No","month":"06","OA_place":"repository","department":[{"_id":"JaMa"},{"_id":"MaMo"}],"fulldoi":"https://doi.org/10.48550/arXiv.2305.14164","das_tickbox":"0","arxiv":1,"related_material":{"record":[{"relation":"dissertation_contains","id":"17336","status":"public"},{"status":"public","relation":"later_version","id":"18897"}]},"date_published":"2024-06-06T00:00:00Z","type":"preprint","author":[{"last_name":"Pedrotti","full_name":"Pedrotti, Francesco","first_name":"Francesco","id":"d3ac8ac6-dc8d-11ea-abe3-e2a9628c4c3c"},{"last_name":"Maas","full_name":"Maas, Jan","id":"4C5696CE-F248-11E8-B48F-1D18A9856A87","first_name":"Jan","orcid":"0000-0002-0845-1338"},{"full_name":"Mondelli, Marco","last_name":"Mondelli","orcid":"0000-0002-3242-7020","first_name":"Marco","id":"27EB676C-8706-11E9-9510-7717E6697425"}],"oa":1,"corr_author":"1","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","status":"public","year":"2024","date_updated":"2026-10-02T12:22:41Z","title":"Improved convergence of score-based diffusion models via prediction-correction","doi":"10.48550/arXiv.2305.14164","_id":"17350","citation":{"short":"F. Pedrotti, J. Maas, M. Mondelli, ArXiv (n.d.).","apa":"Pedrotti, F., Maas, J., & Mondelli, M. (n.d.). Improved convergence of score-based diffusion models via prediction-correction. arXiv. https://doi.org/10.48550/arXiv.2305.14164","ieee":"F. Pedrotti, J. Maas, and M. Mondelli, “Improved convergence of score-based diffusion models via prediction-correction,” arXiv. .","chicago":"Pedrotti, Francesco, Jan Maas, and Marco Mondelli. “Improved Convergence of Score-Based Diffusion Models via Prediction-Correction.” ArXiv, n.d. https://doi.org/10.48550/arXiv.2305.14164.","ama":"Pedrotti F, Maas J, Mondelli M. Improved convergence of score-based diffusion models via prediction-correction. arXiv. doi:10.48550/arXiv.2305.14164","mla":"Pedrotti, Francesco, et al. “Improved Convergence of Score-Based Diffusion Models via Prediction-Correction.” ArXiv, doi:10.48550/arXiv.2305.14164.","ista":"Pedrotti F, Maas J, Mondelli M. Improved convergence of score-based diffusion models via prediction-correction. arXiv, 10.48550/arXiv.2305.14164."},"language":[{"iso":"eng"}],"oa_version":"Preprint","publication":"arXiv","day":"06","abstract":[{"lang":"eng","text":"Score-based generative models (SGMs) are powerful tools to sample from\r\ncomplex data distributions. Their underlying idea is to (i) run a forward\r\nprocess for time $T_1$ by adding noise to the data, (ii) estimate its score\r\nfunction, and (iii) use such estimate to run a reverse process. As the reverse\r\nprocess is initialized with the stationary distribution of the forward one, the\r\nexisting analysis paradigm requires $T_1\\to\\infty$. This is however\r\nproblematic: from a theoretical viewpoint, for a given precision of the score\r\napproximation, the convergence guarantee fails as $T_1$ diverges; from a\r\npractical viewpoint, a large $T_1$ increases computational costs and leads to\r\nerror propagation. This paper addresses the issue by considering a version of\r\nthe popular predictor-corrector scheme: after running the forward process, we\r\nfirst estimate the final distribution via an inexact Langevin dynamics and then\r\nrevert the process. Our key technical contribution is to provide convergence\r\nguarantees which require to run the forward process only for a fixed finite\r\ntime $T_1$. Our bounds exhibit a mild logarithmic dependence on the input\r\ndimension and the subgaussian norm of the target distribution, have minimal\r\nassumptions on the data, and require only to control the $L^2$ loss on the\r\nscore approximation, which is the quantity minimized in practice."}],"supplementarymaterial":"no","project":[{"grant_number":"F6504","name":"Taming Complexity in Partial Differential Systems","_id":"fc31cba2-9c52-11eb-aca3-ff467d239cd2"},{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"researchdata_availability":"no"},{"arxiv":1,"type":"conference","date_published":"2024-12-01T00:00:00Z","fulldoi":"https://doi.org/10.52202/079017-4156","das_tickbox":"0","main_file_link":[{"open_access":"1","url":"https://doi.org/10.52202/079017-4156"}],"publication_status":"published","OA_place":"publisher","article_processing_charge":"No","month":"12","department":[{"_id":"GradSch"},{"_id":"MaMo"}],"OA_type":"free access","conference":{"location":"Vancouver, Canada","start_date":"2024-12-16","name":"NeurIPS: Neural Information Processing Systems","end_date":"2024-12-16"},"external_id":{"arxiv":["2402.13728"]},"date_created":"2025-01-27T11:11:40Z","_id":"18890","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","status":"public","quality_controlled":"1","date_updated":"2026-10-15T09:39:46Z","year":"2024","title":"Average gradient outer product as a mechanism for deep neural collapse","doi":"10.52202/079017-4156","oa":1,"corr_author":"1","author":[{"full_name":"Beaglehole, Daniel","last_name":"Beaglehole","first_name":"Daniel"},{"full_name":"SĂșkenĂ­k, Peter","last_name":"SĂșkenĂ­k","id":"d64d6a8d-eb8e-11eb-b029-96fd216dec3c","first_name":"Peter"},{"id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","orcid":"0000-0002-3242-7020","last_name":"Mondelli","full_name":"Mondelli, Marco"},{"full_name":"Belkin, Mikhail","last_name":"Belkin","first_name":"Mikhail"}],"intvolume":" 37","publisher":"Neural Information Processing Systems Foundation","oa_version":"Published Version","acknowledgement":"We acknowledge support from the National Science Foundation (NSF) and the Simons Foundation for the Collaboration on the Theoretical Foundations of Deep Learning through awards DMS-2031883 and #814639 as well as the TILOS institute (NSF CCF-2112665). This work used the programs (1) XSEDE (Extreme science and engineering discovery environment) which is supported by NSF grant numbers ACI-1548562, and (2) ACCESS (Advanced cyberinfrastructure coordination ecosystem: services & support) which is supported by NSF grants numbers #2138259, #2138286, #2138307, #2137603, and #2138296. Specifically, we used the resources from SDSC Expanse GPU compute nodes, and NCSA Delta system, via allocations TG-CIS220009. Marco Mondelli is supported by the 2019 Lopez-Loreta prize. We also acknowledge useful feedback from anonymous reviewers. ","language":[{"iso":"eng"}],"publication_identifier":{"eissn":["1049-5258"]},"citation":{"short":"D. Beaglehole, P. SĂșkenĂ­k, M. Mondelli, M. Belkin, in:, 38th Annual Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2024.","apa":"Beaglehole, D., SĂșkenĂ­k, P., Mondelli, M., & Belkin, M. (2024). Average gradient outer product as a mechanism for deep neural collapse. In 38th Annual Conference on Neural Information Processing Systems (Vol. 37). Vancouver, Canada: Neural Information Processing Systems Foundation. https://doi.org/10.52202/079017-4156","ieee":"D. Beaglehole, P. SĂșkenĂ­k, M. Mondelli, and M. Belkin, “Average gradient outer product as a mechanism for deep neural collapse,” in 38th Annual Conference on Neural Information Processing Systems, Vancouver, Canada, 2024, vol. 37.","chicago":"Beaglehole, Daniel, Peter SĂșkenĂ­k, Marco Mondelli, and Mikhail Belkin. “Average Gradient Outer Product as a Mechanism for Deep Neural Collapse.” In 38th Annual Conference on Neural Information Processing Systems, Vol. 37. Neural Information Processing Systems Foundation, 2024. https://doi.org/10.52202/079017-4156.","ama":"Beaglehole D, SĂșkenĂ­k P, Mondelli M, Belkin M. Average gradient outer product as a mechanism for deep neural collapse. In: 38th Annual Conference on Neural Information Processing Systems. Vol 37. Neural Information Processing Systems Foundation; 2024. doi:10.52202/079017-4156","mla":"Beaglehole, Daniel, et al. “Average Gradient Outer Product as a Mechanism for Deep Neural Collapse.” 38th Annual Conference on Neural Information Processing Systems, vol. 37, Neural Information Processing Systems Foundation, 2024, doi:10.52202/079017-4156.","ista":"Beaglehole D, SĂșkenĂ­k P, Mondelli M, Belkin M. 2024. Average gradient outer product as a mechanism for deep neural collapse. 38th Annual Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 37."},"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"researchdata_availability":"no","scopus_import":"1","day":"01","abstract":[{"text":"Deep Neural Collapse (DNC) refers to the surprisingly rigid structure of the data representations in the final layers of Deep Neural Networks (DNNs). Though the phenomenon has been measured in a variety of settings, its emergence is typically explained via data-agnostic approaches, such as the unconstrained features model. In this work, we introduce a data-dependent setting where DNC forms due to feature learning through the average gradient outer product (AGOP). The AGOP is defined with respect to a learned predictor and is equal to the uncentered covariance matrix of its input-output gradients averaged over the training dataset. The Deep Recursive Feature Machine (Deep RFM) is a method that constructs a neural network by iteratively mapping the data with the AGOP and applying an untrained random feature map. We demonstrate empirically that DNC occurs in Deep RFM across standard settings as a consequence of the projection with the AGOP matrix computed at each layer. Further, we theoretically explain DNC in Deep RFM in an asymptotic setting and as a result of kernel learning. We then provide evidence that this mechanism holds for neural networks more generally. In particular, we show that the right singular vectors and values of the weights can be responsible for the majority of within-class variability collapse for DNNs trained in the feature learning regime. As observed in recent work, this singular structure is highly correlated with that of the AGOP.","lang":"eng"}],"volume":37,"supplementarymaterial":"no","alternative_title":["Advances in Neural Information Processing Systems"],"publication":"38th Annual Conference on Neural Information Processing Systems"},{"doi":"10.1109/TIT.2024.3367767","title":"Concentration without independence via information measures","year":"2024","date_updated":"2026-10-16T10:39:43Z","status":"public","quality_controlled":"1","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","_id":"15172","isi":1,"publisher":"IEEE","intvolume":" 70","author":[{"full_name":"Esposito, Amedeo Roberto","last_name":"Esposito","id":"9583e921-e1ad-11ec-9862-cef099626dc9","first_name":"Amedeo Roberto"},{"first_name":"Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","orcid":"0000-0002-3242-7020","last_name":"Mondelli","full_name":"Mondelli, Marco"}],"corr_author":"1","oa":1,"das_tickbox":"0","fulldoi":"https://doi.org/10.1109/TIT.2024.3367767","date_published":"2024-06-01T00:00:00Z","type":"journal_article","related_material":{"record":[{"status":"public","relation":"earlier_version","id":"14922"}]},"arxiv":1,"date_created":"2024-03-24T23:01:00Z","external_id":{"isi":["001230181100001"],"arxiv":["2303.07245"]},"department":[{"_id":"MaMo"}],"article_processing_charge":"No","month":"06","publication_status":"published","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2303.07245"}],"volume":70,"supplementarymaterial":"yes","abstract":[{"text":"We propose a novel approach to concentration for non-independent random variables. The main idea is to “pretend” that the random variables are independent and pay a multiplicative price measuring how far they are from actually being independent. This price is encapsulated in the Hellinger integral between the joint and the product of the marginals, which is then upper bounded leveraging tensorisation properties. Our bounds represent a natural generalisation of concentration inequalities in the presence of dependence: we recover exactly the classical bounds (McDiarmid’s inequality) when the random variables are independent. Furthermore, in a “large deviations” regime, we obtain the same decay in the probability as for the independent case, even when the random variables display non-trivial dependencies. To show this, we consider a number of applications of interest. First, we provide a bound for Markov chains with finite state space. Then, we consider the Simple Symmetric Random Walk, which is a non-contracting Markov chain, and a non-Markovian setting in which the stochastic process depends on its entire past. To conclude, we propose an application to Markov Chain Monte Carlo methods, where our approach leads to an improved lower bound on the minimum burn-in period required to reach a certain accuracy. In all of these settings, we provide a regime of parameters in which our bound fares better than what the state of the art can provide.","lang":"eng"}],"day":"01","article_type":"original","scopus_import":"1","researchdata_availability":"no","project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"}],"publication":"IEEE Transactions on Information Theory","page":"3823-3839","language":[{"iso":"eng"}],"issue":"6","oa_version":"Preprint","citation":{"ama":"Esposito AR, Mondelli M. Concentration without independence via information measures. IEEE Transactions on Information Theory. 2024;70(6):3823-3839. doi:10.1109/TIT.2024.3367767","mla":"Esposito, Amedeo Roberto, and Marco Mondelli. “Concentration without Independence via Information Measures.” IEEE Transactions on Information Theory, vol. 70, no. 6, IEEE, 2024, pp. 3823–39, doi:10.1109/TIT.2024.3367767.","ista":"Esposito AR, Mondelli M. 2024. Concentration without independence via information measures. IEEE Transactions on Information Theory. 70(6), 3823–3839.","short":"A.R. Esposito, M. Mondelli, IEEE Transactions on Information Theory 70 (2024) 3823–3839.","apa":"Esposito, A. R., & Mondelli, M. (2024). Concentration without independence via information measures. IEEE Transactions on Information Theory. IEEE. https://doi.org/10.1109/TIT.2024.3367767","ieee":"A. R. Esposito and M. Mondelli, “Concentration without independence via information measures,” IEEE Transactions on Information Theory, vol. 70, no. 6. IEEE, pp. 3823–3839, 2024.","chicago":"Esposito, Amedeo Roberto, and Marco Mondelli. “Concentration without Independence via Information Measures.” IEEE Transactions on Information Theory. IEEE, 2024. https://doi.org/10.1109/TIT.2024.3367767."},"publication_identifier":{"eissn":["1557-9654"],"issn":["0018-9448"]}},{"supplementarymaterial":"no","abstract":[{"text":"Efficient utilization of large-scale biobank data is crucial for inferring the genetic basis of disease and predicting health outcomes from the DNA. Yet we lack efficient, accurate methods that scale to data where electronic health records are linked to whole genome sequence information. To address this issue, our paper develops a new algorithmic paradigm based on Approximate Message Passing (AMP), which is specifically tailored for genomic prediction and association testing. Our method yields comparable out-of-sample prediction accuracy to the state of the art on UK Biobank traits, whilst dramatically improving computational complexity, with a 8x-speed up in the run time. In addition, AMP theory provides a joint association testing framework, which outperforms the currently used REGENIE method, in roughly a third of the compute time. This first, truly large-scale application of the AMP framework lays the foundations for a far wider range of statistical analyses for hundreds of millions of variables measured on millions of people.","lang":"eng"}],"day":"19","scopus_import":"1","researchdata_availability":"no","project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"},{"_id":"9B8D11D6-BA93-11EA-9121-9846C619BF3A","grant_number":"PCEGP3_181181","name":"Improving estimation and prediction of common complex disease risk"}],"publication":"2024 IEEE International Conference on Acoustics, Speech, and Signal Processing","page":"13151-13155","language":[{"iso":"eng"}],"acknowledgement":"This work was supported by a Lopez-Loreta Prize to MM, an SNSF Eccellenza Grant to MRR (PCEGP3-181181), and core funding from ISTA. The authors thank Philip Schniter, Matthew Stephens and Pragya Sur for valuable suggestions on an early version of the work. The authors acknowledge the participants and investigators of the UK Biobank study. High-performance\r\ncomputing was supported by the Scientific Service Units (SSU) of IST Austria through resources provided by Scientific Computing (SciComp).","oa_version":"Submitted Version","citation":{"ama":"Depope A, Mondelli M, Robinson MR. Inference of genetic effects via approximate message passing. In: 2024 IEEE International Conference on Acoustics, Speech, and Signal Processing. IEEE; 2024:13151-13155. doi:10.1109/ICASSP48485.2024.10447198","ista":"Depope A, Mondelli M, Robinson MR. 2024. Inference of genetic effects via approximate message passing. 2024 IEEE International Conference on Acoustics, Speech, and Signal Processing. ICASSP: International Conference on Acoustics, Speech and Signal Processing, 13151–13155.","mla":"Depope, Al, et al. “Inference of Genetic Effects via Approximate Message Passing.” 2024 IEEE International Conference on Acoustics, Speech, and Signal Processing, IEEE, 2024, pp. 13151–55, doi:10.1109/ICASSP48485.2024.10447198.","apa":"Depope, A., Mondelli, M., & Robinson, M. R. (2024). Inference of genetic effects via approximate message passing. In 2024 IEEE International Conference on Acoustics, Speech, and Signal Processing (pp. 13151–13155). Seoul, Korea: IEEE. https://doi.org/10.1109/ICASSP48485.2024.10447198","short":"A. Depope, M. Mondelli, M.R. Robinson, in:, 2024 IEEE International Conference on Acoustics, Speech, and Signal Processing, IEEE, 2024, pp. 13151–13155.","ieee":"A. Depope, M. Mondelli, and M. R. Robinson, “Inference of genetic effects via approximate message passing,” in 2024 IEEE International Conference on Acoustics, Speech, and Signal Processing, Seoul, Korea, 2024, pp. 13151–13155.","chicago":"Depope, Al, Marco Mondelli, and Matthew Richard Robinson. “Inference of Genetic Effects via Approximate Message Passing.” In 2024 IEEE International Conference on Acoustics, Speech, and Signal Processing, 13151–55. IEEE, 2024. https://doi.org/10.1109/ICASSP48485.2024.10447198."},"publication_identifier":{"issn":["1520-6149"],"isbn":["9798350344851"]},"doi":"10.1109/ICASSP48485.2024.10447198","year":"2024","acknowledged_ssus":[{"_id":"ScienComp"}],"date_updated":"2026-10-16T10:42:55Z","title":"Inference of genetic effects via approximate message passing","status":"public","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","quality_controlled":"1","isi":1,"_id":"17147","publisher":"IEEE","oa":1,"corr_author":"1","author":[{"last_name":"Depope","full_name":"Depope, Al","first_name":"Al","id":"0b77531d-dbcd-11ea-9d1d-a8eee0bf3830"},{"orcid":"0000-0002-3242-7020","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","full_name":"Mondelli, Marco","last_name":"Mondelli"},{"id":"E5D42276-F5DA-11E9-8E24-6303E6697425","first_name":"Matthew Richard","orcid":"0000-0001-8982-8813","last_name":"Robinson","full_name":"Robinson, Matthew Richard"}],"das_tickbox":"0","fulldoi":"https://doi.org/10.1109/ICASSP48485.2024.10447198","date_published":"2024-04-19T00:00:00Z","type":"conference","date_created":"2024-06-16T22:01:07Z","conference":{"start_date":"2024-04-14","location":"Seoul, Korea","end_date":"2024-04-19","name":"ICASSP: International Conference on Acoustics, Speech and Signal Processing"},"external_id":{"isi":["001396233806078"]},"OA_type":"green","department":[{"_id":"MaMo"},{"_id":"MaRo"}],"OA_place":"repository","month":"04","article_processing_charge":"No","main_file_link":[{"url":"https://openreview.net/forum?id=aQYCDxfZV0","open_access":"1"}],"publication_status":"published"},{"OA_place":"repository","article_processing_charge":"No","month":"07","OA_type":"green","department":[{"_id":"MaMo"}],"publication_status":"published","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2402.02969"}],"date_created":"2025-01-30T07:35:49Z","external_id":{"arxiv":["2402.02969"]},"conference":{"name":"ICML: International Conference on Machine Learning","end_date":"2024-07-27","start_date":"2024-07-21","location":"Vienna, Austria"},"date_published":"2024-07-30T00:00:00Z","type":"conference","arxiv":1,"related_material":{"record":[{"status":"public","id":"22857","relation":"dissertation_contains"}]},"das_tickbox":"0","intvolume":" 235","publisher":"ML Research Press","author":[{"full_name":"Bombari, Simone","last_name":"Bombari","first_name":"Simone","id":"ca726dda-de17-11ea-bc14-f9da834f63aa"},{"id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","orcid":"0000-0002-3242-7020","last_name":"Mondelli","full_name":"Mondelli, Marco"}],"oa":1,"corr_author":"1","_id":"18973","year":"2024","date_updated":"2026-10-16T10:42:17Z","title":"Towards understanding the word sensitivity of attention layers: A study via random features","status":"public","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","quality_controlled":"1","citation":{"short":"S. Bombari, M. Mondelli, in:, 41st International Conference on Machine Learning, ML Research Press, 2024, pp. 4300–4328.","apa":"Bombari, S., & Mondelli, M. (2024). Towards understanding the word sensitivity of attention layers: A study via random features. In 41st International Conference on Machine Learning (Vol. 235, pp. 4300–4328). Vienna, Austria: ML Research Press.","ieee":"S. Bombari and M. Mondelli, “Towards understanding the word sensitivity of attention layers: A study via random features,” in 41st International Conference on Machine Learning, Vienna, Austria, 2024, vol. 235, pp. 4300–4328.","chicago":"Bombari, Simone, and Marco Mondelli. “Towards Understanding the Word Sensitivity of Attention Layers: A Study via Random Features.” In 41st International Conference on Machine Learning, 235:4300–4328. ML Research Press, 2024.","ama":"Bombari S, Mondelli M. Towards understanding the word sensitivity of attention layers: A study via random features. In: 41st International Conference on Machine Learning. Vol 235. ML Research Press; 2024:4300-4328.","mla":"Bombari, Simone, and Marco Mondelli. “Towards Understanding the Word Sensitivity of Attention Layers: A Study via Random Features.” 41st International Conference on Machine Learning, vol. 235, ML Research Press, 2024, pp. 4300–28.","ista":"Bombari S, Mondelli M. 2024. Towards understanding the word sensitivity of attention layers: A study via random features. 41st International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 235, 4300–4328."},"publication_identifier":{"eissn":["2640-3498"]},"oa_version":"Preprint","language":[{"iso":"eng"}],"acknowledgement":"The authors were partially supported by the 2019 LopezLoreta prize, and they would like to thank Mohammad Hossein Amani, Lorenzo Beretta, and Clement Rebuffel for helpful discussions.","publication":"41st International Conference on Machine Learning","page":"4300-4328","researchdata_availability":"no","project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"abstract":[{"text":"Understanding the reasons behind the exceptional success of transformers requires a better analysis of why attention layers are suitable for NLP tasks. In particular, such tasks require predictive models to capture contextual meaning which often depends on one or few words, even if the sentence is long. Our work studies this key property, dubbed word sensitivity (WS), in the prototypical setting of random features. We show that attention layers enjoy high WS, namely, there exists a vector in the space of embeddings that largely perturbs the random attention features map. The argument critically exploits the role of the softmax in the attention layer, highlighting its benefit compared to other activations (e.g., ReLU). In contrast, the WS of standard random features is of order 1/n−−√, n being the number of words in the textual sample, and thus it decays with the length of the context. We then translate these results on the word sensitivity into generalization bounds: due to their low WS, random features provably cannot learn to distinguish between two sentences that differ only in a single word; in contrast, due to their high WS, random attention features have higher generalization capabilities. We validate our theoretical results with experimental evidence over the BERT-Base word embeddings of the imdb review dataset.","lang":"eng"}],"alternative_title":["PMLR"],"volume":235,"supplementarymaterial":"no","scopus_import":"1","day":"30"},{"arxiv":1,"related_material":{"record":[{"status":"public","id":"22857","relation":"dissertation_contains"}]},"date_published":"2024-07-30T00:00:00Z","type":"conference","das_tickbox":"0","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2305.12100"}],"publication_status":"published","OA_place":"repository","article_processing_charge":"No","month":"07","OA_type":"green","department":[{"_id":"MaMo"}],"conference":{"end_date":"2024-07-27","name":"ICML: International Conference on Machine Learning","start_date":"2024-07-21","location":"Vienna, Austria"},"external_id":{"arxiv":["2305.12100"]},"date_created":"2025-01-30T07:29:47Z","_id":"18972","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","status":"public","quality_controlled":"1","title":"How spurious features are memorized: Precise analysis for random and NTK features","date_updated":"2026-10-16T10:42:37Z","year":"2024","corr_author":"1","oa":1,"author":[{"id":"ca726dda-de17-11ea-bc14-f9da834f63aa","first_name":"Simone","full_name":"Bombari, Simone","last_name":"Bombari"},{"full_name":"Mondelli, Marco","last_name":"Mondelli","orcid":"0000-0002-3242-7020","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco"}],"publisher":"ML Research Press","intvolume":" 235","oa_version":"Preprint","acknowledgement":"The authors were partially supported by the 2019 LopezLoreta prize, and they would like to thank (in alphabetical order) Grigorios Chrysos, Simone Maria Giancola, Mahyar\r\nJafari Nodeh, Christoph Lampert, Marco Miani, GuanWen Qiu, and Peter Sukenık for helpful discussions.","language":[{"iso":"eng"}],"publication_identifier":{"eissn":["2640-3498"]},"citation":{"apa":"Bombari, S., & Mondelli, M. (2024). How spurious features are memorized: Precise analysis for random and NTK features. In 41st International Conference on Machine Learning (Vol. 235, pp. 4267–4299). Vienna, Austria: ML Research Press.","short":"S. Bombari, M. Mondelli, in:, 41st International Conference on Machine Learning, ML Research Press, 2024, pp. 4267–4299.","ieee":"S. Bombari and M. Mondelli, “How spurious features are memorized: Precise analysis for random and NTK features,” in 41st International Conference on Machine Learning, Vienna, Austria, 2024, vol. 235, pp. 4267–4299.","chicago":"Bombari, Simone, and Marco Mondelli. “How Spurious Features Are Memorized: Precise Analysis for Random and NTK Features.” In 41st International Conference on Machine Learning, 235:4267–99. ML Research Press, 2024.","ama":"Bombari S, Mondelli M. How spurious features are memorized: Precise analysis for random and NTK features. In: 41st International Conference on Machine Learning. Vol 235. ML Research Press; 2024:4267-4299.","ista":"Bombari S, Mondelli M. 2024. How spurious features are memorized: Precise analysis for random and NTK features. 41st International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 235, 4267–4299.","mla":"Bombari, Simone, and Marco Mondelli. “How Spurious Features Are Memorized: Precise Analysis for Random and NTK Features.” 41st International Conference on Machine Learning, vol. 235, ML Research Press, 2024, pp. 4267–99."},"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"researchdata_availability":"no","scopus_import":"1","day":"30","abstract":[{"text":"Deep learning models are known to overfit and memorize spurious features in the training dataset. While numerous empirical studies have aimed at understanding this phenomenon, a rigorous theoretical framework to quantify it is still missing. In this paper, we consider spurious features that are uncorrelated with the learning task, and we provide a precise characterization of how they are memorized via two separate terms: (i) the stability of the model with respect to individual training samples, and (ii) the feature alignment between the spurious pattern and the full sample. While the first term is well established in learning theory and it is connected to the generalization error in classical work, the second one is, to the best of our knowledge, novel. Our key technical result gives a precise characterization of the feature alignment for the two prototypical settings of random features (RF) and neural tangent kernel (NTK) regression. We prove that the memorization of spurious features weakens as the generalization capability increases and, through the analysis of the feature alignment, we unveil the role of the model and of its activation function. Numerical experiments show the predictive power of our theory on standard datasets (MNIST, CIFAR-10).","lang":"eng"}],"volume":235,"supplementarymaterial":"yes","alternative_title":["PMLR"],"page":"4267-4299","publication":"41st International Conference on Machine Learning"},{"language":[{"iso":"eng"}],"oa_version":"Published Version","publication_identifier":{"issn":["2663-337X"]},"citation":{"ieee":"A. Shevchenko, “High-dimensional limits in artificial neural networks,” Institute of Science and Technology Austria, 2024.","chicago":"Shevchenko, Alexander. “High-Dimensional Limits in Artificial Neural Networks.” Institute of Science and Technology Austria, 2024. https://doi.org/10.15479/at:ista:17465.","apa":"Shevchenko, A. (2024). High-dimensional limits in artificial neural networks. Institute of Science and Technology Austria. https://doi.org/10.15479/at:ista:17465","short":"A. Shevchenko, High-Dimensional Limits in Artificial Neural Networks, Institute of Science and Technology Austria, 2024.","ista":"Shevchenko A. 2024. High-dimensional limits in artificial neural networks. Institute of Science and Technology Austria.","mla":"Shevchenko, Alexander. High-Dimensional Limits in Artificial Neural Networks. Institute of Science and Technology Austria, 2024, doi:10.15479/at:ista:17465.","ama":"Shevchenko A. High-dimensional limits in artificial neural networks. 2024. doi:10.15479/at:ista:17465"},"file_date_updated":"2024-10-05T22:30:05Z","file":[{"content_type":"application/pdf","embargo":"2024-10-04","date_created":"2024-09-02T09:23:32Z","checksum":"da6dd3166078934577f6af93d27000e2","file_name":"thesis_a2b.pdf","creator":"ashevche","relation":"main_file","file_id":"17482","file_size":4468610,"date_updated":"2024-10-05T22:30:05Z","access_level":"open_access"},{"embargo_to":"open_access","creator":"ashevche","file_size":15930999,"file_id":"17483","relation":"source_file","date_updated":"2024-10-05T22:30:05Z","access_level":"closed","content_type":"application/zip","date_created":"2024-09-02T09:23:46Z","file_name":"Thesis Alex - ISTA.zip","checksum":"76a39ef252239560923cdda4ce0a31a4"}],"day":"29","abstract":[{"lang":"eng","text":"In the modern age of machine learning, artificial neural networks have become an integral part\r\nof many practical systems. One of the key ingredients of the success of the deep learning\r\napproach is recent computational advances which allowed the training of models with billions\r\nof parameters on large-scale data. Such over-parameterized and data-hungry regimes pose a\r\nchallenge for the theoretical analysis of modern models since “classical” statistical wisdom\r\nis no longer applicable. In this view, it is paramount to extend or develop new machinery\r\nthat will allow tackling the neural network analysis under new challenging asymptotic regimes,\r\nwhich is the focus of this thesis.\r\nLarge neural network systems are usually optimized via “local” search algorithms, such\r\nas stochastic gradient descent (SGD). However, given the high-dimensional nature of the\r\nparameter space, it is a priori not clear why such a crude “local” approach works so remarkably\r\nwell in practice. We take a step towards demystifying this phenomenon by showing that\r\nthe landscape of the SGD training dynamics exhibits a few beneficial properties for the\r\noptimization. First, we show that along the SGD trajectory an over-parameterized network\r\nis dropout stable. The emergence of dropout stability allows to conclude that the minima\r\nfound by SGD are connected via a continuous path of small loss. This in turn means that\r\nthe high-dimensional landscape of the neural network optimization problem is provably not so\r\nunfavourable to gradient-based training, due to mode connectivity. Next, we show that SGD\r\nfor an over-parameterized network tends to find solutions that are functionally more “simple”.\r\nThis in turn means that the SGD minima are more robust, since a less complicated solution\r\nwill less likely overfit the data. More formally, for a prototypical example of a wide two-layer\r\nReLU network on a 1d regression task we show that the SGD algorithm is implicitly selective in\r\nits choice of an interpolating solution. Namely, at convergence the neural network implements\r\na piece-wise linear function with the number of linear regions depending only on the amount\r\nof training data. This is in contrast to a “smooth”-like behaviour which one would expect\r\ngiven such a severe over-parameterization of the model.\r\nDiverging from the generic supervised setting of classification and regression problems, we\r\nanalyze an auto-encoder model that is commonly used for representation learning and data\r\ncompression. Despite the wide applicability of the auto-encoding paradigm, the theoretical\r\nunderstanding of their behaviour is limited even in the simplistic shallow case. The related\r\nwork is restricted to extreme asymptotic regimes in which the auto-encoder is either severely\r\nover-parameterized or under-parameterized. In contrast, we provide a tight characterization\r\nfor the 1-bit compression of Gaussian signals in the challenging proportional regime, i.e., the\r\ninput dimension and the size of the compressed representation obey the same asymptotics.\r\nWe also show that gradient-based methods are able to find a globally optimal solution and\r\nthat the predictions made for Gaussian data extrapolate beyond - to the case of compression\r\nof natural images. Next, we relax the Gaussian assumption and study more structured input\r\nsources. We show that the shallow model is sometimes agnostic to the structure of the data\r\nvii\r\nwhich results in a Gaussian-like behaviour. We prove that making the decoding component\r\nslightly less shallow is already enough to escape the “curse” of Gaussian performance.\r\n"}],"supplementarymaterial":"no","alternative_title":["ISTA Thesis"],"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"},{"grant_number":"W1260-N35","name":"Vienna Graduate School on Computational Optimization","_id":"9B9290DE-BA93-11EA-9121-9846C619BF3A"}],"researchdata_availability":"no","page":"232","ddc":["519"],"doi_confirm":"1","fulldoi":"https://doi.org/10.15479/at:ista:17465","das_tickbox":"0","related_material":{"record":[{"id":"11420","relation":"part_of_dissertation","status":"public"},{"relation":"part_of_dissertation","id":"14459","status":"public"},{"relation":"part_of_dissertation","id":"9198","status":"public"},{"status":"public","relation":"part_of_dissertation","id":"17469"}]},"date_published":"2024-08-29T00:00:00Z","type":"dissertation","supervisor":[{"id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","orcid":"0000-0002-3242-7020","last_name":"Mondelli","full_name":"Mondelli, Marco"},{"orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh"}],"date_created":"2024-08-28T15:14:25Z","publication_status":"published","article_processing_charge":"No","OA_place":"repository","month":"08","department":[{"_id":"GradSch"},{"_id":"DaAl"},{"_id":"MaMo"}],"status":"public","has_accepted_license":"1","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","year":"2024","acknowledged_ssus":[{"_id":"ScienComp"}],"date_updated":"2026-10-02T11:20:32Z","title":"High-dimensional limits in artificial neural networks","doi":"10.15479/at:ista:17465","degree_awarded":"PhD","_id":"17465","oa":1,"corr_author":"1","author":[{"last_name":"Shevchenko","full_name":"Shevchenko, Aleksandr","id":"F2B06EC2-C99E-11E9-89F0-752EE6697425","first_name":"Aleksandr"}],"publisher":"Institute of Science and Technology Austria"},{"related_material":{"record":[{"status":"public","relation":"dissertation_contains","id":"23072"},{"status":"public","relation":"dissertation_contains","id":"17465"}]},"arxiv":1,"type":"conference","date_published":"2024-07-01T00:00:00Z","das_tickbox":"0","main_file_link":[{"url":"https://proceedings.mlr.press/v235/kogler24a.html","open_access":"1"}],"publication_status":"published","department":[{"_id":"DaAl"},{"_id":"MaMo"}],"article_processing_charge":"No","month":"07","external_id":{"arxiv":["2402.05013"]},"conference":{"name":"ICML: International Conference on Machine Learning","end_date":"2024-07-27","start_date":"2024-07-21","location":"Vienna, Austria"},"date_created":"2024-08-29T11:47:57Z","_id":"17469","quality_controlled":"1","status":"public","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","year":"2024","date_updated":"2026-11-08T23:31:11Z","title":"Compression of structured data with autoencoders: Provable benefit of nonlinearities and depth","oa":1,"author":[{"first_name":"Kevin","id":"94ec913c-dc85-11ea-9058-e5051ab2428b","full_name":"Kögler, Kevin","last_name":"Kögler"},{"id":"F2B06EC2-C99E-11E9-89F0-752EE6697425","first_name":"Aleksandr","last_name":"Shevchenko","full_name":"Shevchenko, Aleksandr"},{"first_name":"Hamed","full_name":"Hassani, Hamed","last_name":"Hassani"},{"full_name":"Mondelli, Marco","last_name":"Mondelli","orcid":"0000-0002-3242-7020","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco"}],"corr_author":"1","intvolume":" 235","publisher":"ML Research Press","oa_version":"Published Version","acknowledgement":"Kevin Kogler, Alexander Shevchenko and Marco Mondelli are supported by the 2019 Lopez-Loreta Prize. Hamed\r\nHassani acknowledges the support by the NSF CIF award (1910056) and the NSF Institute for CORE Emerging Methods in Data Science (EnCORE).","language":[{"iso":"eng"}],"citation":{"ama":"Kögler K, Shevchenko A, Hassani H, Mondelli M. Compression of structured data with autoencoders: Provable benefit of nonlinearities and depth. In: Proceedings of the 41st International Conference on Machine Learning. Vol 235. ML Research Press; 2024:24964-25015.","mla":"Kögler, Kevin, et al. “Compression of Structured Data with Autoencoders: Provable Benefit of Nonlinearities and Depth.” Proceedings of the 41st International Conference on Machine Learning, vol. 235, ML Research Press, 2024, pp. 24964–5015.","ista":"Kögler K, Shevchenko A, Hassani H, Mondelli M. 2024. Compression of structured data with autoencoders: Provable benefit of nonlinearities and depth. Proceedings of the 41st International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 235, 24964–25015.","short":"K. Kögler, A. Shevchenko, H. Hassani, M. Mondelli, in:, Proceedings of the 41st International Conference on Machine Learning, ML Research Press, 2024, pp. 24964–25015.","apa":"Kögler, K., Shevchenko, A., Hassani, H., & Mondelli, M. (2024). Compression of structured data with autoencoders: Provable benefit of nonlinearities and depth. In Proceedings of the 41st International Conference on Machine Learning (Vol. 235, pp. 24964–25015). Vienna, Austria: ML Research Press.","chicago":"Kögler, Kevin, Alexander Shevchenko, Hamed Hassani, and Marco Mondelli. “Compression of Structured Data with Autoencoders: Provable Benefit of Nonlinearities and Depth.” In Proceedings of the 41st International Conference on Machine Learning, 235:24964–15. ML Research Press, 2024.","ieee":"K. Kögler, A. Shevchenko, H. Hassani, and M. Mondelli, “Compression of structured data with autoencoders: Provable benefit of nonlinearities and depth,” in Proceedings of the 41st International Conference on Machine Learning, Vienna, Austria, 2024, vol. 235, pp. 24964–25015."},"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"researchdata_availability":"no","day":"01","scopus_import":"1","alternative_title":["PMLR"],"volume":235,"supplementarymaterial":"no","abstract":[{"text":"Autoencoders are a prominent model in many empirical branches of machine learning and lossy data compression. However, basic theoretical questions remain unanswered even in a shallow two-layer setting. In particular, to what degree does a shallow autoencoder capture the structure of the underlying data distribution? For the prototypical case of the 1-bit compression of sparse Gaussian data, we prove that gradient descent converges to a solution that completely disregards the sparse structure of the input. Namely, the performance of the algorithm is the same as if it was compressing a Gaussian source - with no sparsity. For general data distributions, we give evidence of a phase transition phenomenon in the shape of the gradient descent minimizer, as a function of the data sparsity: below the critical sparsity level, the minimizer is a rotation taken uniformly at random (just like in the compression of non-sparse data); above the critical sparsity, the minimizer is the identity (up to a permutation). Finally, by exploiting a connection with approximate message passing algorithms, we show how to improve upon Gaussian performance for the compression of sparse data: adding a denoising function to a shallow architecture already reduces the loss provably, and a suitable multi-layer decoder leads to a further improvement. We validate our findings on image datasets, such as CIFAR-10 and MNIST.","lang":"eng"}],"page":"24964-25015","ddc":["000"],"publication":"Proceedings of the 41st International Conference on Machine Learning"},{"fulldoi":"https://doi.org/10.1073/pnas.2302028120","type":"journal_article","date_published":"2023-07-25T00:00:00Z","pmid":1,"related_material":{"link":[{"relation":"software","url":"https://github.com/fcamilli95/Structured-PCA-"}]},"date_created":"2023-07-30T22:01:02Z","external_id":{"pmid":["37463204"],"isi":["001121663500001"]},"article_number":"e2302028120","month":"07","article_processing_charge":"Yes (in subscription journal)","department":[{"_id":"MaMo"}],"publication_status":"published","year":"2023","date_updated":"2025-09-09T12:41:50Z","title":"Fundamental limits in structured principal component analysis and how to reach them","doi":"10.1073/pnas.2302028120","quality_controlled":"1","has_accepted_license":"1","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","status":"public","isi":1,"_id":"13315","publisher":"National Academy of Sciences","intvolume":" 120","author":[{"first_name":"Jean","last_name":"Barbier","full_name":"Barbier, Jean"},{"full_name":"Camilli, Francesco","last_name":"Camilli","first_name":"Francesco"},{"full_name":"Mondelli, Marco","last_name":"Mondelli","orcid":"0000-0002-3242-7020","first_name":"Marco","id":"27EB676C-8706-11E9-9510-7717E6697425"},{"last_name":"SĂĄenz","full_name":"SĂĄenz, Manuel","first_name":"Manuel"}],"oa":1,"language":[{"iso":"eng"}],"acknowledgement":"J.B. was funded by the European Union (ERC, CHORAL, project number 101039794). Views and opinions expressed are however those of the author(s) only and do not necessarily reflect those of the European Union or the European Research Council. Neither the European Union nor the granting authority can be held responsible for them. M.M. was supported by the 2019 Lopez-Loreta Prize. We would like to thank the reviewers for the insightful comments and, in particular, for suggesting the BAMP-inspired denoisers leading to AMP-AP.","issue":"30","oa_version":"Published Version","tmp":{"short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png"},"citation":{"ama":"Barbier J, Camilli F, Mondelli M, SĂĄenz M. Fundamental limits in structured principal component analysis and how to reach them. Proceedings of the National Academy of Sciences of the United States of America. 2023;120(30). doi:10.1073/pnas.2302028120","mla":"Barbier, Jean, et al. “Fundamental Limits in Structured Principal Component Analysis and How to Reach Them.” Proceedings of the National Academy of Sciences of the United States of America, vol. 120, no. 30, e2302028120, National Academy of Sciences, 2023, doi:10.1073/pnas.2302028120.","ista":"Barbier J, Camilli F, Mondelli M, SĂĄenz M. 2023. Fundamental limits in structured principal component analysis and how to reach them. Proceedings of the National Academy of Sciences of the United States of America. 120(30), e2302028120.","short":"J. Barbier, F. Camilli, M. Mondelli, M. SĂĄenz, Proceedings of the National Academy of Sciences of the United States of America 120 (2023).","apa":"Barbier, J., Camilli, F., Mondelli, M., & SĂĄenz, M. (2023). Fundamental limits in structured principal component analysis and how to reach them. Proceedings of the National Academy of Sciences of the United States of America. National Academy of Sciences. https://doi.org/10.1073/pnas.2302028120","ieee":"J. Barbier, F. Camilli, M. Mondelli, and M. SĂĄenz, “Fundamental limits in structured principal component analysis and how to reach them,” Proceedings of the National Academy of Sciences of the United States of America, vol. 120, no. 30. National Academy of Sciences, 2023.","chicago":"Barbier, Jean, Francesco Camilli, Marco Mondelli, and Manuel SĂĄenz. “Fundamental Limits in Structured Principal Component Analysis and How to Reach Them.” Proceedings of the National Academy of Sciences of the United States of America. National Academy of Sciences, 2023. https://doi.org/10.1073/pnas.2302028120."},"publication_identifier":{"eissn":["1091-6490"]},"file":[{"checksum":"1fc06228afdb3aa80cf8e7766bcf9dc5","file_name":"2023_PNAS_Barbier.pdf","date_created":"2023-07-31T07:30:48Z","content_type":"application/pdf","success":1,"access_level":"open_access","date_updated":"2023-07-31T07:30:48Z","relation":"main_file","file_id":"13323","file_size":995933,"creator":"dernst"}],"file_date_updated":"2023-07-31T07:30:48Z","abstract":[{"lang":"eng","text":"How do statistical dependencies in measurement noise influence high-dimensional inference? To answer this, we study the paradigmatic spiked matrix model of principal components analysis (PCA), where a rank-one matrix is corrupted by additive noise. We go beyond the usual independence assumption on the noise entries, by drawing the noise from a low-order polynomial orthogonal matrix ensemble. The resulting noise correlations make the setting relevant for applications but analytically challenging. We provide characterization of the Bayes optimal limits of inference in this model. If the spike is rotation invariant, we show that standard spectral PCA is optimal. However, for more general priors, both PCA and the existing approximate message-passing algorithm (AMP) fall short of achieving the information-theoretic limits, which we compute using the replica method from statistical physics. We thus propose an AMP, inspired by the theory of adaptive Thouless–Anderson–Palmer equations, which is empirically observed to saturate the conjectured theoretical limit. This AMP comes with a rigorous state evolution analysis tracking its performance. Although we focus on specific noise distributions, our methodology can be generalized to a wide class of trace matrix ensembles at the cost of more involved expressions. Finally, despite the seemingly strong assumption of rotation-invariant noise, our theory empirically predicts algorithmic performance on real data, pointing at strong universality properties."}],"volume":120,"scopus_import":"1","article_type":"original","day":"25","project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"}],"publication":"Proceedings of the National Academy of Sciences of the United States of America","ddc":["000"]},{"external_id":{"arxiv":["2210.06819"]},"date_created":"2024-02-02T11:21:56Z","publication_status":"published","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2210.06819"}],"month":"02","article_processing_charge":"No","department":[{"_id":"MaMo"}],"arxiv":1,"type":"conference","date_published":"2023-02-28T00:00:00Z","author":[{"last_name":"Wu","full_name":"Wu, Diyuan","id":"1a5914c2-896a-11ed-bdf8-fb80621a0635","first_name":"Diyuan"},{"last_name":"Kungurtsev","full_name":"Kungurtsev, Vyacheslav","first_name":"Vyacheslav"},{"last_name":"Mondelli","full_name":"Mondelli, Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","orcid":"0000-0002-3242-7020"}],"corr_author":"1","oa":1,"publisher":"ML Research Press","has_accepted_license":"1","status":"public","quality_controlled":"1","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","title":"Mean-field analysis for heavy ball methods: Dropout-stability, connectivity, and global convergence","year":"2023","date_updated":"2026-06-18T17:41:36Z","_id":"14924","citation":{"short":"D. Wu, V. Kungurtsev, M. Mondelli, in:, Transactions on Machine Learning Research, ML Research Press, 2023.","apa":"Wu, D., Kungurtsev, V., & Mondelli, M. (2023). Mean-field analysis for heavy ball methods: Dropout-stability, connectivity, and global convergence. In Transactions on Machine Learning Research. ML Research Press.","ieee":"D. Wu, V. Kungurtsev, and M. Mondelli, “Mean-field analysis for heavy ball methods: Dropout-stability, connectivity, and global convergence,” in Transactions on Machine Learning Research, 2023.","chicago":"Wu, Diyuan, Vyacheslav Kungurtsev, and Marco Mondelli. “Mean-Field Analysis for Heavy Ball Methods: Dropout-Stability, Connectivity, and Global Convergence.” In Transactions on Machine Learning Research. ML Research Press, 2023.","ama":"Wu D, Kungurtsev V, Mondelli M. Mean-field analysis for heavy ball methods: Dropout-stability, connectivity, and global convergence. In: Transactions on Machine Learning Research. ML Research Press; 2023.","mla":"Wu, Diyuan, et al. “Mean-Field Analysis for Heavy Ball Methods: Dropout-Stability, Connectivity, and Global Convergence.” Transactions on Machine Learning Research, ML Research Press, 2023.","ista":"Wu D, Kungurtsev V, Mondelli M. 2023. Mean-field analysis for heavy ball methods: Dropout-stability, connectivity, and global convergence. Transactions on Machine Learning Research. , TMLR, ."},"acknowledgement":"D. Wu and M. Mondelli are partially supported by the 2019 Lopez-Loreta Prize. V. Kungurtsev was supported by the OP VVV project CZ.02.1.01/0.0/0.0/16_019/0000765 \"Research Center for Informatics\".","language":[{"iso":"eng"}],"oa_version":"Published Version","tmp":{"short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png"},"ddc":["000"],"publication":"Transactions on Machine Learning Research","day":"28","abstract":[{"lang":"eng","text":"The stochastic heavy ball method (SHB), also known as stochastic gradient descent (SGD) with Polyak's momentum, is widely used in training neural networks. However, despite the remarkable success of such algorithm in practice, its theoretical characterization remains limited. In this paper, we focus on neural networks with two and three layers and provide a rigorous understanding of the properties of the solutions found by SHB: \\emph{(i)} stability after dropping out part of the neurons, \\emph{(ii)} connectivity along a low-loss path, and \\emph{(iii)} convergence to the global optimum.\r\nTo achieve this goal, we take a mean-field view and relate the SHB dynamics to a certain partial differential equation in the limit of large network widths. This mean-field perspective has inspired a recent line of work focusing on SGD while, in contrast, our paper considers an algorithm with momentum. More specifically, after proving existence and uniqueness of the limit differential equations, we show convergence to the global optimum and give a quantitative bound between the mean-field limit and the SHB dynamics of a finite-width network. Armed with this last bound, we are able to establish the dropout-stability and connectivity of SHB solutions."}],"alternative_title":["TMLR"],"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}]},{"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","status":"public","quality_controlled":"1","year":"2023","date_updated":"2026-07-07T06:35:57Z","title":"Deep neural collapse is provably optimal for the deep unconstrained features model","_id":"14921","corr_author":"1","author":[{"id":"d64d6a8d-eb8e-11eb-b029-96fd216dec3c","first_name":"Peter","last_name":"SĂșkenĂ­k","full_name":"SĂșkenĂ­k, Peter"},{"last_name":"Mondelli","full_name":"Mondelli, Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","orcid":"0000-0002-3242-7020"},{"full_name":"Lampert, Christoph","last_name":"Lampert","orcid":"0000-0001-8622-7887","id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","first_name":"Christoph"}],"oa":1,"publisher":"Neural Information Processing Systems Foundation","das_tickbox":"1","arxiv":1,"date_published":"2023-12-15T00:00:00Z","type":"conference","conference":{"location":"New Orleans, LA, United States","start_date":"2023-12-10","name":"NeurIPS: Neural Information Processing Systems","end_date":"2023-12-16"},"external_id":{"arxiv":["2305.13165"]},"date_created":"2024-02-02T11:17:41Z","publication_status":"published","main_file_link":[{"open_access":"1","url":" https://doi.org/10.48550/arXiv.2305.13165"}],"department":[{"_id":"MaMo"},{"_id":"ChLa"}],"month":"12","article_processing_charge":"No","day":"15","alternative_title":["Advances in Neural Information Processing Systems"],"abstract":[{"lang":"eng","text":"Neural collapse (NC) refers to the surprising structure of the last layer of deep neural networks in the terminal phase of gradient descent training. Recently, an increasing amount of experimental evidence has pointed to the propagation of NC to earlier layers of neural networks. However, while the NC in the last layer is well studied theoretically, much less is known about its multi-layered counterpart - deep neural collapse (DNC). In particular, existing work focuses either on linear layers or only on the last two layers at the price of an extra assumption. Our paper fills this gap by generalizing the established analytical framework for NC - the unconstrained features model - to multiple non-linear layers. Our key technical contribution is to show that, in a deep unconstrained features model, the unique global optimum for binary classification exhibits all the properties typical of DNC. This explains the existing experimental evidence of DNC. We also empirically show that (i) by optimizing deep unconstrained features models via gradient descent, the resulting solution agrees well with our theory, and (ii) trained networks recover the unconstrained features suitable for the occurrence of DNC, thus supporting the validity of this modeling principle."}],"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"publication":"37th Conference on Neural Information Processing Systems","acknowledgement":"M. M. is partially supported by the 2019 Lopez-Loreta Prize. The authors would like to thank Eugenia Iofinova, Bernd Prach and Simone Bombari for valuable feedback on the manuscript.","language":[{"iso":"eng"}],"oa_version":"Preprint","publication_identifier":{"eissn":["1049-5258"]},"citation":{"mla":"SĂșkenĂ­k, Peter, et al. “Deep Neural Collapse Is Provably Optimal for the Deep Unconstrained Features Model.” 37th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2023.","ista":"SĂșkenĂ­k P, Mondelli M, Lampert C. 2023. Deep neural collapse is provably optimal for the deep unconstrained features model. 37th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, .","ama":"SĂșkenĂ­k P, Mondelli M, Lampert C. Deep neural collapse is provably optimal for the deep unconstrained features model. In: 37th Conference on Neural Information Processing Systems. Neural Information Processing Systems Foundation; 2023.","chicago":"SĂșkenĂ­k, Peter, Marco Mondelli, and Christoph Lampert. “Deep Neural Collapse Is Provably Optimal for the Deep Unconstrained Features Model.” In 37th Conference on Neural Information Processing Systems. Neural Information Processing Systems Foundation, 2023.","ieee":"P. SĂșkenĂ­k, M. Mondelli, and C. Lampert, “Deep neural collapse is provably optimal for the deep unconstrained features model,” in 37th Conference on Neural Information Processing Systems, New Orleans, LA, United States, 2023.","short":"P. SĂșkenĂ­k, M. Mondelli, C. Lampert, in:, 37th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2023.","apa":"SĂșkenĂ­k, P., Mondelli, M., & Lampert, C. (2023). Deep neural collapse is provably optimal for the deep unconstrained features model. In 37th Conference on Neural Information Processing Systems. New Orleans, LA, United States: Neural Information Processing Systems Foundation."}}]