[{"external_id":{"arxiv":["2509.22214"]},"file":[{"creator":"cchlebak","content_type":"application/pdf","date_updated":"2026-09-11T12:07:48Z","relation":"main_file","checksum":"a3dcba6649b7496124fa0ae68bf2c4cb","access_level":"open_access","file_size":7869510,"date_created":"2026-09-11T12:07:48Z","file_name":"2026_ICLR_Iurada.pdf","file_id":"22907","success":1}],"intvolume":"      2026","_id":"22894","status":"public","oa_version":"Published Version","das_tickbox":"0","day":"26","arxiv":1,"year":"2026","acknowledgement":"M.M. is funded by the European Union (ERC, INF2\r\n, project number 101161364). S.B. was supported\r\nby a Google PhD fellowship. L.I. acknowledges the grant received from the European Union NextGenerationEU (Piano Nazionale di Ripresa E Resilienza (PNRR)) DM 351 on Trustworthy AI. T.T. &\r\nL.I. acknowledge the EU project ELSA - European Lighthouse on Secure and Safe AI. This study was\r\ncarried out within the FAIR - Future Artificial Intelligence Research and received funding from the\r\nEuropean Union Next-GenerationEU (PIANO NAZIONALE DI RIPRESA E RESILIENZA (PNRR)\r\n– MISSIONE 4 COMPONENTE 2, INVESTIMENTO 1.3 – D.D. 1555 11/10/2022, PE00000013).\r\nThis manuscript reflects only the authors’ views and opinions, neither the European Union nor the\r\nEuropean Commission can be considered responsible for them. The authors would like to thank\r\nYizhe Zhu for helpful discussions.","citation":{"ama":"Iurada L, Bombari S, Tommasi T, Mondelli M. A law of data reconstruction for random features (and beyond). In: <i>14th International Conference on Learning Representations</i>. Vol 2026. OpenReview; 2026:145275-145314.","ista":"Iurada L, Bombari S, Tommasi T, Mondelli M. 2026. A law of data reconstruction for random features (and beyond). 14th International Conference on Learning Representations. ICLR: International Conference on Learning Representations  vol. 2026, 145275–145314.","ieee":"L. Iurada, S. Bombari, T. Tommasi, and M. Mondelli, “A law of data reconstruction for random features (and beyond),” in <i>14th International Conference on Learning Representations</i>, Rio de Janeiro, Brazil, 2026, vol. 2026, pp. 145275–145314.","apa":"Iurada, L., Bombari, S., Tommasi, T., &#38; Mondelli, M. (2026). A law of data reconstruction for random features (and beyond). In <i>14th International Conference on Learning Representations</i> (Vol. 2026, pp. 145275–145314). Rio de Janeiro, Brazil: OpenReview.","chicago":"Iurada, Leonardo, Simone Bombari, Tatiana Tommasi, and Marco Mondelli. “A Law of Data Reconstruction for Random Features (and Beyond).” In <i>14th International Conference on Learning Representations</i>, 2026:145275–314. OpenReview, 2026.","mla":"Iurada, Leonardo, et al. “A Law of Data Reconstruction for Random Features (and Beyond).” <i>14th International Conference on Learning Representations</i>, vol. 2026, OpenReview, 2026, pp. 145275–314.","short":"L. Iurada, S. Bombari, T. Tommasi, M. Mondelli, in:, 14th International Conference on Learning Representations, OpenReview, 2026, pp. 145275–145314."},"publisher":"OpenReview","ddc":["000"],"date_published":"2026-01-26T00:00:00Z","OA_type":"gold","has_accepted_license":"1","tmp":{"image":"/images/cc_by.png","short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)"},"OA_place":"publisher","date_updated":"2026-09-21T13:07:01Z","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","abstract":[{"text":"Large-scale deep learning models are known to memorize parts of the training\r\nset. In machine learning theory, memorization is often framed as interpolation or\r\nlabel fitting, and classical results show that this can be achieved when the number\r\nof parameters p in the model is larger than the number of training samples n. In\r\nthis work, we consider memorization from the perspective of data reconstruction,\r\ndemonstrating that this can be achieved when p is larger than dn, where d is\r\nthe dimensionality of the data. More specifically, we show that, in the random\r\nfeatures model, when p ≫ dn, the subspace spanned by the training samples in\r\nfeature space gives sufficient information to identify the individual samples in input\r\nspace. Our analysis suggests an optimization method to reconstruct the dataset\r\nfrom the model parameters, and we demonstrate that this method performs well on\r\nvarious architectures (random features, two-layer fully-connected and deep residual\r\nnetworks). Our results reveal a law of data reconstruction, according to which the\r\nentire training dataset can be recovered as p exceeds the threshold dn.\r\n","lang":"eng"}],"file_date_updated":"2026-09-11T12:07:48Z","date_created":"2026-09-09T13:31:46Z","article_processing_charge":"No","project":[{"grant_number":"101161364","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf"},{"_id":"92099302-16d5-11f0-9cad-f9a785f54fbd","name":"Trustworthy Deep Learning Theory: Private Over-Parameterized Models and Robust LLMs"}],"related_material":{"record":[{"status":"public","relation":"dissertation_contains","id":"22857"}]},"quality_controlled":"1","page":"145275-145314","department":[{"_id":"GradSch"},{"_id":"MaMo"}],"volume":2026,"language":[{"iso":"eng"}],"publication_status":"published","month":"01","license":"https://creativecommons.org/licenses/by/4.0/","publication_identifier":{"isbn":["9798331339678"]},"title":"A law of data reconstruction for random features (and beyond)","oa":1,"type":"conference","corr_author":"1","conference":{"end_date":"2026-04-27","name":"ICLR: International Conference on Learning Representations ","location":"Rio de Janeiro, Brazil","start_date":"2026-04-23"},"publication":"14th International Conference on Learning Representations","author":[{"last_name":"Iurada","full_name":"Iurada, Leonardo","first_name":"Leonardo"},{"first_name":"Simone","full_name":"Bombari, Simone","last_name":"Bombari","id":"ca726dda-de17-11ea-bc14-f9da834f63aa"},{"full_name":"Tommasi, Tatiana","last_name":"Tommasi","first_name":"Tatiana"},{"orcid":"0000-0002-3242-7020","first_name":"Marco","full_name":"Mondelli, Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","last_name":"Mondelli"}]},{"date_updated":"2026-10-16T10:42:36Z","alternative_title":["ISTA Thesis"],"user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","file_date_updated":"2026-09-10T10:03:48Z","abstract":[{"text":"Artificial intelligence and machine learning have undergone an unprecedented evolution in the past decade, motivating a research effort toward a theory able to capture the qualitative behavior of large-scale neural systems. A central puzzle has been the clear benefit of scaling architecture size and overfitting the training set in supervised learning tasks. This evidence, in apparent contradiction with classical statistical learning theory, pushed researchers to develop a new theory capturing the interplay between the algorithmic and architectural bias of training and the specific target function, differently from previous methods rooted in uniform stability.\r\nThis approach has enabled a grounded understanding of novel learning regimes, typically through formal limits where the number of training samples $n$, data dimensions $d$, and model parameters $p$ grow to infinity at different rates. \\\\\r\nIn this thesis, we follow this approach, focusing on the trustworthiness of high-dimensional models: properties that are difficult to control during training or deployment and often emerge under unpredictable or adversarial conditions. In such settings, it is crucial to formally ensure a priori the reliability of machine learning systems.\r\nFirst, we study data memorization, both as label fitting and as the storage of private information about training samples in trained parameters. We prove that $p = \\Omega(n)$ parameters are sufficient for a deep neural network to memorize a generic set of labels, and for a model to memorize spurious features across training data. We then give evidence that $p = \\Omega(dn)$ parameters are instead necessary for an adversary to reconstruct the full training set from the trained parameters.\r\nSecond, we study robustness, both to adversarial perturbations and to distribution shift. We first prove that $p = \\Omega(dn)$ parameters can be sufficient for a class of neural networks to overfit the training data while guaranteeing robustness to adversarial perturbations. Then, we focus on spurious correlations learning in high-dimensional regression, studying the effect of the ridge regularization parameter in the proportional regime $n = \\Theta(d)$, and connecting it via an equivalence argument to the role of over-parameterization $p = \\Omega(n)$ in neural networks. We also investigate the architectural bias of attention-based networks, showing that they are sensitive to the replacement of individual words in an embedded sentence, allowing them to generalize on sentences where the contextual meaning depends on one or few words.\r\nFinally, we study differentially private optimization in high-dimensional regimes. We prove that standard private gradient methods do not suffer in the over-parameterized regime $p = \\Omega(n)$, challenging the current wisdom based on stability-derived generalization bounds. We then consider linear regression in the proportional regime $n = \\Theta(d)$, showing that standard private gradient descent can achieve optimal rates under appropriate hyper-parameter scaling, such as sufficiently small gradient clipping constants, whose role is still debated in practice.","lang":"eng"}],"ddc":["519"],"has_accepted_license":"1","date_published":"2026-09-08T00:00:00Z","fulldoi":"https://doi.org/10.15479/AT-ISTA-22857","keyword":["machine learning","high-dimensional statistics","deep learning theory","privacy","memorization","robustness"],"OA_place":"publisher","_id":"22857","researchdata_availability":"no","status":"public","oa_version":"Published Version","day":"08","das_tickbox":"0","supervisor":[{"first_name":"Marco","orcid":"0000-0002-3242-7020","full_name":"Mondelli, Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","last_name":"Mondelli"}],"citation":{"ama":"Bombari S. Trustworthy machine learning in high dimensions. 2026. doi:<a href=\"https://doi.org/10.15479/AT-ISTA-22857\">10.15479/AT-ISTA-22857</a>","ista":"Bombari S. 2026. Trustworthy machine learning in high dimensions. Institute of Science and Technology Austria.","apa":"Bombari, S. (2026). <i>Trustworthy machine learning in high dimensions</i>. Institute of Science and Technology Austria. <a href=\"https://doi.org/10.15479/AT-ISTA-22857\">https://doi.org/10.15479/AT-ISTA-22857</a>","ieee":"S. Bombari, “Trustworthy machine learning in high dimensions,” Institute of Science and Technology Austria, 2026.","chicago":"Bombari, Simone. “Trustworthy Machine Learning in High Dimensions.” Institute of Science and Technology Austria, 2026. <a href=\"https://doi.org/10.15479/AT-ISTA-22857\">https://doi.org/10.15479/AT-ISTA-22857</a>.","mla":"Bombari, Simone. <i>Trustworthy Machine Learning in High Dimensions</i>. Institute of Science and Technology Austria, 2026, doi:<a href=\"https://doi.org/10.15479/AT-ISTA-22857\">10.15479/AT-ISTA-22857</a>.","short":"S. Bombari, Trustworthy Machine Learning in High Dimensions, Institute of Science and Technology Austria, 2026."},"acknowledgement":"This project was partially supported by the 2019 Lopez-Loreta prize,\r\nthe European Union (ERC, INF2\r\n, project number 101161364), the Austrian Science Fund\r\n(FWF) 10.55776/COE12, and a Google PhD fellowship in machine intelligence. Furthermore,\r\nthe candidate acknowledges the support from the Scientific Service Units of the Institute of\r\nScience and Technology Austria through resources provided by Scientific Computing.","year":"2026","publisher":"Institute of Science and Technology Austria","acknowledged_ssus":[{"_id":"ScienComp"}],"file":[{"date_updated":"2026-09-08T13:28:38Z","content_type":"application/zip","creator":"sbombari","relation":"source_file","checksum":"8eab99d6dc6e4476826bdfc7c00fdebb","access_level":"closed","file_name":"Thesis copy.zip","date_created":"2026-09-08T13:28:38Z","file_size":99154325,"file_id":"22859"},{"relation":"main_file","access_level":"open_access","checksum":"007033bafe4622ff4c2e758301354df1","creator":"sbombari","content_type":"application/pdf","date_updated":"2026-09-10T10:03:48Z","file_id":"22898","success":1,"date_created":"2026-09-10T10:03:48Z","file_size":12856211,"file_name":"2026_Bombari_Simone_Thesis.pdf"}],"corr_author":"1","author":[{"full_name":"Bombari, Simone","last_name":"Bombari","id":"ca726dda-de17-11ea-bc14-f9da834f63aa","first_name":"Simone"}],"publication_identifier":{"issn":["2663-337X"],"isbn":["978-3-99078-091-6"]},"oa":1,"title":"Trustworthy machine learning in high dimensions","doi":"10.15479/AT-ISTA-22857","type":"dissertation","doi_confirm":"1","page":"446","department":[{"_id":"GradSch"},{"_id":"MaMo"}],"language":[{"iso":"eng"}],"publication_status":"published","month":"09","date_created":"2026-09-08T13:40:08Z","supplementarymaterial":"no","article_processing_charge":"No","project":[{"_id":"92099302-16d5-11f0-9cad-f9a785f54fbd","name":"Trustworthy Deep Learning Theory: Private Over-Parameterized Models and Robust LLMs"},{"name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf","grant_number":"101161364"},{"grant_number":"COE12","_id":"74caaef7-b034-11f1-8f2d-e0e993bb422e","name":"Bilateral Artificial Intelligence (Mondelli)"}],"degree_awarded":"PhD","related_material":{"record":[{"status":"public","relation":"part_of_dissertation","id":"12537"},{"status":"public","id":"21324","relation":"part_of_dissertation"},{"status":"public","relation":"part_of_dissertation","id":"22894"},{"status":"public","id":"19627","relation":"part_of_dissertation"},{"id":"12859","relation":"part_of_dissertation","status":"public"},{"id":"18973","relation":"part_of_dissertation","status":"public"},{"status":"public","relation":"part_of_dissertation","id":"18972"}]}},{"related_material":{"record":[{"relation":"part_of_dissertation","id":"21488","status":"public"}]},"degree_awarded":"PhD","project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"},{"_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","grant_number":"101161364"},{"name":"Improving estimation and prediction of common complex disease risk","_id":"9B8D11D6-BA93-11EA-9121-9846C619BF3A","grant_number":"PCEGP3_181181"}],"date_created":"2026-07-10T13:27:20Z","article_processing_charge":"No","month":"07","language":[{"iso":"eng"}],"department":[{"_id":"GradSch"},{"_id":"MaRo"},{"_id":"MaMo"}],"publication_status":"published","page":"169","doi_confirm":"1","doi":"10.15479/AT-ISTA-22258","type":"dissertation","publication_identifier":{"issn":["2663-337X"]},"oa":1,"title":"From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics","author":[{"first_name":"Al","id":"0b77531d-dbcd-11ea-9d1d-a8eee0bf3830","last_name":"Depope","full_name":"Depope, Al"}],"corr_author":"1","file":[{"file_id":"22316","date_created":"2026-07-13T14:52:19Z","file_size":25109878,"file_name":"2026_Depope_Al_Thesis.pdf","access_level":"open_access","checksum":"9ab386790515628d957a194f30a7ccb4","relation":"main_file","creator":"adepope","content_type":"application/pdf","date_updated":"2026-07-13T14:52:19Z"},{"file_size":1203199939,"date_created":"2026-07-13T14:56:41Z","file_name":"2026_Depope_Al_Thesis.zip","file_id":"22317","creator":"adepope","date_updated":"2026-07-13T14:56:41Z","content_type":"application/zip","relation":"source_file","checksum":"8ed8fb63f76a695d5b6fec35343f4b90","access_level":"closed"}],"acknowledged_ssus":[{"_id":"ScienComp"}],"citation":{"apa":"Depope, A. (2026). <i>From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics</i>. Institute of Science and Technology Austria. <a href=\"https://doi.org/10.15479/AT-ISTA-22258\">https://doi.org/10.15479/AT-ISTA-22258</a>","ieee":"A. Depope, “From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics,” Institute of Science and Technology Austria, 2026.","ama":"Depope A. From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics. 2026. doi:<a href=\"https://doi.org/10.15479/AT-ISTA-22258\">10.15479/AT-ISTA-22258</a>","ista":"Depope A. 2026. From sparse selection to risk prediction: Approximate message passing for proteomic survival models and large-scale genomics. Institute of Science and Technology Austria.","short":"A. Depope, From Sparse Selection to Risk Prediction: Approximate Message Passing for Proteomic Survival Models and Large-Scale Genomics, Institute of Science and Technology Austria, 2026.","mla":"Depope, Al. <i>From Sparse Selection to Risk Prediction: Approximate Message Passing for Proteomic Survival Models and Large-Scale Genomics</i>. Institute of Science and Technology Austria, 2026, doi:<a href=\"https://doi.org/10.15479/AT-ISTA-22258\">10.15479/AT-ISTA-22258</a>.","chicago":"Depope, Al. “From Sparse Selection to Risk Prediction: Approximate Message Passing for Proteomic Survival Models and Large-Scale Genomics.” Institute of Science and Technology Austria, 2026. <a href=\"https://doi.org/10.15479/AT-ISTA-22258\">https://doi.org/10.15479/AT-ISTA-22258</a>."},"acknowledgement":"This work was supported in part by the Swiss National Science Foundation through the\r\nEccellenza Grant \"Improving estimation and prediction of common complex disease risk\"\r\n(grant number PCEGP3_181181); the European Research Council through the grant\r\n\"Inference in High Dimensions: Light-speed Algorithms and Information Limits\" (grant\r\nnumber 101161364); and the Fondation Jean-Jacques et Felicia Lopez-Loreta through the\r\nPrix Lopez-Loretta 2019.\r\n","year":"2026","publisher":"Institute of Science and Technology Austria","day":"11","das_tickbox":"1","supervisor":[{"first_name":"Matthew Richard","orcid":"0000-0001-8982-8813","id":"E5D42276-F5DA-11E9-8E24-6303E6697425","last_name":"Robinson","full_name":"Robinson, Matthew Richard"},{"first_name":"Marco","orcid":"0000-0002-3242-7020","full_name":"Mondelli, Marco","last_name":"Mondelli","id":"27EB676C-8706-11E9-9510-7717E6697425"}],"oa_version":"Published Version","_id":"22258","status":"public","OA_place":"publisher","fulldoi":"https://doi.org/10.15479/AT-ISTA-22258","keyword":["Approximate Message Passing","GWAS","Genomics","Proteomics","Survival modeling"],"ddc":["576","610","006"],"has_accepted_license":"1","date_published":"2026-07-11T00:00:00Z","file_date_updated":"2026-07-13T14:56:41Z","abstract":[{"text":"Uncovering the genetic architecture of complex traits and pinpointing causal molecular drivers require the ability to distinguish true signals from noise within massive, high-dimensional omics datasets. To extract meaningful biological insights from these datasets, such as identifying causal genetic variants and proteins, scalable and accurate inference methods are essential. To this end, this thesis develops novel Bayesian inference frameworks based on Vector Approximate Message Passing and demonstrates their effectiveness in the modeling of disease onset times and quantitative physical and clinical measures.\r\n\r\nFirst, we introduce gVAMP, a Bayesian framework tailored for Genome-Wide Association Studies that enables the joint modeling of quantitative complex traits across millions of genetic variants. gVAMP demonstrates superior accuracy in variable selection and out-of-sample polygenic risk prediction compared to state-of-the-art approaches. We model human height using 17 million whole-genome sequence variants from the UK Biobank, incorporating a vast number of rare variants and revealing novel associations. gVAMP achieves a prediction accuracy of approximately 46% for human height, representing the highest reported performance for this trait to date. \r\n\r\nSecond, we present vampW, a Bayesian framework for survival analysis applied to proteomic data. By effectively handling right-censoring and complex protein dependencies within the UK Biobank Pharma Proteomics Project dataset, vampW identifies 219 protein associations across 24 disease outcomes, the majority of which are not among the top marginal discoveries. We further adjust protein levels for exponential age effects, yielding 1,308 associations and highlighting the sensitivity of the analysis to the chosen age-correction methodology. Finally, vampW improves upon the variable selection capabilities of the commonly used (penalized) variants of the Cox proportional hazards model and delivers state-of-the-art out-of-sample prediction of disease onset times.\r\n\r\nCollectively, these methods provide powerful tools for dissecting the genetic architecture of complex traits and the proteomic drivers of disease onset. Furthermore, by delivering accurate polygenic risk scores and precise predictions of onset times, this work advances the capabilities of personalized medicine and clinical risk stratification.","lang":"eng"}],"user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","alternative_title":["ISTA Thesis"],"date_updated":"2026-10-20T09:03:21Z"},{"project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"},{"grant_number":"101161364","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf"},{"_id":"9B8D11D6-BA93-11EA-9121-9846C619BF3A","name":"Improving estimation and prediction of common complex disease risk","grant_number":"PCEGP3_181181"}],"related_material":{"record":[{"id":"22258","relation":"dissertation_contains","status":"public"}],"link":[{"url":"https://ista.ac.at/en/news/big-data-and-human-height/","description":"News on ISTA website","relation":"press_release"}]},"quality_controlled":"1","date_created":"2026-03-23T15:10:03Z","supplementarymaterial":"yes","article_processing_charge":"Yes","month":"05","license":"https://creativecommons.org/licenses/by-nc-nd/4.0/","department":[{"_id":"MaMo"},{"_id":"MaRo"}],"language":[{"iso":"eng"}],"volume":6,"publication_status":"published","article_number":"101162","pmid":1,"publication_identifier":{"eissn":["2666-979X"]},"oa":1,"title":"Joint modeling of whole-genome sequencing data for human height via approximate message passing","doi":"10.1016/j.xgen.2026.101162","type":"journal_article","publication":"Cell Genomics","author":[{"first_name":"Al","last_name":"Depope","id":"0b77531d-dbcd-11ea-9d1d-a8eee0bf3830","full_name":"Depope, Al"},{"id":"b995e25b-8c4b-11ed-a6d8-f71b7bcd6122","last_name":"Bajzik","full_name":"Bajzik, Jakub","first_name":"Jakub"},{"id":"27EB676C-8706-11E9-9510-7717E6697425","last_name":"Mondelli","full_name":"Mondelli, Marco","first_name":"Marco","orcid":"0000-0002-3242-7020"},{"id":"E5D42276-F5DA-11E9-8E24-6303E6697425","last_name":"Robinson","full_name":"Robinson, Matthew Richard","orcid":"0000-0001-8982-8813","first_name":"Matthew Richard"}],"corr_author":"1","scopus_import":"1","external_id":{"pmid":["41713425"]},"issue":"5","file":[{"creator":"dernst","date_updated":"2026-07-28T07:06:26Z","content_type":"application/pdf","relation":"main_file","access_level":"open_access","checksum":"6b59686f8d9733add4f23d23f3dd9df0","date_created":"2026-07-28T07:06:26Z","file_size":3736705,"file_name":"2026_CellGenomics_Depope.pdf","file_id":"22596","success":1}],"intvolume":"         6","dataavailabilitystatement":"This project uses the UK Biobank data under project number 35520. UK Biobank genotypic and phenotypic data are available through a formal request at http://www.ukbiobank.ac.uk. It also uses genotypic and phenotypic data from the All of Us study, which are also available through a formal request at https://www.researchallofus.org/data-tools/data-access/. All summary statistic estimates are released publicly on Dryad: https://doi.org/10.5061/dryad.cz8w9gjjc.\r\n•\r\nThe gVAMP code developed in this work is open source and has been deposited on GitHub, where it is publicly available at https://github.com/medical-genomics-group/gVAMP, and the code used to generate the data in the manuscript are available from Zenodo https://doi.org/10.5281/zenodo.17935521. The URLs of other software used are listed in the key resources table.","day":"13","das_tickbox":"1","year":"2026","acknowledgement":"We thank Malgorzata Borczyk for creating the gene burden scores. We thank Robin Beaumont, Amedeo Roberto Esposito, Gareth Hawkes, Philip Schniter, Matthew Stephens, Pragya Sur, Peter Visscher, Michael Weedon, and Harry Wright for providing valuable suggestions and comments on earlier versions of the work. This project was funded by a Lopez-Loreta Prize to M.M., an SNSF Eccellenza Grant to M.R.R. (PCEGP3-181181), an ERC Starting Grant to M.M. (INF2, project number 101161364), and core funding from ISTA. High-performance computing was supported by the Scientific Service Units (SSU) of ISTA through resources provided by Scientific Computing (SciComp). We would like to acknowledge the participants and investigators of the UK Biobank study. We gratefully acknowledge the All of Us participants for their contributions, without whom this research would not have been possible. We also thank the National Institutes of Health All of Us Research Program for making available the participant data (and/or samples and/or cohort) examined in this study.","citation":{"ama":"Depope A, Bajzik J, Mondelli M, Robinson MR. Joint modeling of whole-genome sequencing data for human height via approximate message passing. <i>Cell Genomics</i>. 2026;6(5). doi:<a href=\"https://doi.org/10.1016/j.xgen.2026.101162\">10.1016/j.xgen.2026.101162</a>","ista":"Depope A, Bajzik J, Mondelli M, Robinson MR. 2026. Joint modeling of whole-genome sequencing data for human height via approximate message passing. Cell Genomics. 6(5), 101162.","apa":"Depope, A., Bajzik, J., Mondelli, M., &#38; Robinson, M. R. (2026). Joint modeling of whole-genome sequencing data for human height via approximate message passing. <i>Cell Genomics</i>. Cell Press. <a href=\"https://doi.org/10.1016/j.xgen.2026.101162\">https://doi.org/10.1016/j.xgen.2026.101162</a>","ieee":"A. Depope, J. Bajzik, M. Mondelli, and M. R. Robinson, “Joint modeling of whole-genome sequencing data for human height via approximate message passing,” <i>Cell Genomics</i>, vol. 6, no. 5. Cell Press, 2026.","chicago":"Depope, Al, Jakub Bajzik, Marco Mondelli, and Matthew Richard Robinson. “Joint Modeling of Whole-Genome Sequencing Data for Human Height via Approximate Message Passing.” <i>Cell Genomics</i>. Cell Press, 2026. <a href=\"https://doi.org/10.1016/j.xgen.2026.101162\">https://doi.org/10.1016/j.xgen.2026.101162</a>.","short":"A. Depope, J. Bajzik, M. Mondelli, M.R. Robinson, Cell Genomics 6 (2026).","mla":"Depope, Al, et al. “Joint Modeling of Whole-Genome Sequencing Data for Human Height via Approximate Message Passing.” <i>Cell Genomics</i>, vol. 6, no. 5, 101162, Cell Press, 2026, doi:<a href=\"https://doi.org/10.1016/j.xgen.2026.101162\">10.1016/j.xgen.2026.101162</a>."},"publisher":"Cell Press","researchdata_availability":"yes","_id":"21488","status":"public","oa_version":"Published Version","OA_place":"publisher","DOAJ_listed":"1","ddc":["000","570"],"has_accepted_license":"1","OA_type":"gold","date_published":"2026-05-13T00:00:00Z","fulldoi":"https://doi.org/10.1016/j.xgen.2026.101162","article_type":"original","tmp":{"image":"/images/cc_by_nc_nd.png","short":"CC BY-NC-ND (4.0)","legal_code_url":"https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode","name":"Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International (CC BY-NC-ND 4.0)"},"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","file_date_updated":"2026-07-28T07:06:26Z","abstract":[{"text":"Human height is a model for the genetic analysis of complex traits, and recent studies suggest the presence of thousands of common genetic variant associations and hundreds of low-frequency/rare variants. Here, we develop a new algorithmic paradigm based on approximate message passing (genomic vector approximate message passing [gVAMP]) for identifying DNA sequence variants associated with complex traits and common diseases in large-scale whole-genome sequencing (WGS) data. We show that gVAMP accurately localizes associations to variants with the correct frequency and position in the DNA, outperforming existing fine-mapping methods in selecting the appropriate genetic variants within WGS data. We then apply gVAMP to jointly model the relationship of tens of millions of WGS variants with human height in hundreds of thousands of UK Biobank individuals. We identify 59 rare variants and gene burden scores alongside many hundreds of DNA regions containing common variant associations and show that understanding the genetic basis of complex traits will require the joint analysis of hundreds of millions of variables measured on millions of people. The polygenic risk scores obtained from gVAMP have high accuracy (including a prediction accuracy of ∼46% for human height) and outperform current methods for downstream tasks such as mixed linear model association testing across 13 UK Biobank traits. In conclusion, gVAMP offers a scalable foundation for a wider range of analyses in WGS data.","lang":"eng"}],"date_updated":"2026-10-20T09:03:22Z"},{"publication":"Proceedings of the 42nd International Conference on Machine Learning","author":[{"full_name":"Gozeten, Halil Alperen","last_name":"Gozeten","first_name":"Halil Alperen"},{"last_name":"Ildiz","full_name":"Ildiz, Muhammed Emrullah","first_name":"Muhammed Emrullah"},{"full_name":"Zhang, Xuechen","last_name":"Zhang","first_name":"Xuechen"},{"last_name":"Soltanolkotabi","full_name":"Soltanolkotabi, Mahdi","first_name":"Mahdi"},{"full_name":"Mondelli, Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","last_name":"Mondelli","orcid":"0000-0002-3242-7020","first_name":"Marco"},{"last_name":"Oymak","full_name":"Oymak, Samet","first_name":"Samet"}],"conference":{"start_date":"2025-07-13","location":"Vancouver, Canada","end_date":"2025-07-19","name":"ICML: International Conference on Machine Learning"},"title":"Test-time training provably improves transformers as in-context learners","oa":1,"pmid":1,"publication_identifier":{"eissn":["2640-3498"]},"type":"conference","month":"11","page":"20266-20295","publication_status":"published","language":[{"iso":"eng"}],"department":[{"_id":"MaMo"}],"volume":267,"project":[{"name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf","grant_number":"101161364"}],"quality_controlled":"1","article_processing_charge":"No","date_created":"2026-02-18T12:00:44Z","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","file_date_updated":"2026-02-19T08:15:48Z","abstract":[{"lang":"eng","text":"Test-time training (TTT) methods explicitly update the weights of a model to adapt to the specific test instance, and they have found success in a variety of settings, including most recently language modeling and reasoning. To demystify this success, we investigate a gradient-based TTT algorithm for in-context learning, where we train a transformer model on the in-context demonstrations provided in the test prompt. Specifically, we provide a comprehensive theoretical characterization of linear transformers when the update rule is a single gradient step. Our theory (i) delineates the role of alignment between pretraining distribution and target task, (ii) demystifies how TTT can alleviate distribution shift, and (iii) quantifies the sample complexity of TTT including how it can significantly reduce the eventual sample size required for in-context learning. As our empirical contribution, we study the benefits of TTT for TabPFN, a tabular foundation model. In line with our theory, we demonstrate that TTT significantly reduces the required sample size for tabular classification (3 to 5 times fewer) unlocking substantial inference efficiency with a negligible training cost."}],"alternative_title":["PMLR"],"date_updated":"2026-02-19T08:18:24Z","OA_place":"publisher","OA_type":"gold","has_accepted_license":"1","date_published":"2025-11-30T00:00:00Z","ddc":["000"],"tmp":{"image":"/images/cc_by.png","short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)"},"day":"30","publisher":"ML Research Press","year":"2025","citation":{"ieee":"H. A. Gozeten, M. E. Ildiz, X. Zhang, M. Soltanolkotabi, M. Mondelli, and S. Oymak, “Test-time training provably improves transformers as in-context learners,” in <i>Proceedings of the 42nd International Conference on Machine Learning</i>, Vancouver, Canada, 2025, vol. 267, pp. 20266–20295.","apa":"Gozeten, H. A., Ildiz, M. E., Zhang, X., Soltanolkotabi, M., Mondelli, M., &#38; Oymak, S. (2025). Test-time training provably improves transformers as in-context learners. In <i>Proceedings of the 42nd International Conference on Machine Learning</i> (Vol. 267, pp. 20266–20295). Vancouver, Canada: ML Research Press.","ista":"Gozeten HA, Ildiz ME, Zhang X, Soltanolkotabi M, Mondelli M, Oymak S. 2025. Test-time training provably improves transformers as in-context learners. Proceedings of the 42nd International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 267, 20266–20295.","ama":"Gozeten HA, Ildiz ME, Zhang X, Soltanolkotabi M, Mondelli M, Oymak S. Test-time training provably improves transformers as in-context learners. In: <i>Proceedings of the 42nd International Conference on Machine Learning</i>. Vol 267. ML Research Press; 2025:20266-20295.","short":"H.A. Gozeten, M.E. Ildiz, X. Zhang, M. Soltanolkotabi, M. Mondelli, S. Oymak, in:, Proceedings of the 42nd International Conference on Machine Learning, ML Research Press, 2025, pp. 20266–20295.","mla":"Gozeten, Halil Alperen, et al. “Test-Time Training Provably Improves Transformers as in-Context Learners.” <i>Proceedings of the 42nd International Conference on Machine Learning</i>, vol. 267, ML Research Press, 2025, pp. 20266–95.","chicago":"Gozeten, Halil Alperen, Muhammed Emrullah Ildiz, Xuechen Zhang, Mahdi Soltanolkotabi, Marco Mondelli, and Samet Oymak. “Test-Time Training Provably Improves Transformers as in-Context Learners.” In <i>Proceedings of the 42nd International Conference on Machine Learning</i>, 267:20266–95. ML Research Press, 2025."},"acknowledgement":"H.A.G., M.E.I., X.Z., and S.O. were supported in part by the NSF grants CCF2046816, CCF-2403075, CCF-2008020, and the Office of Naval Research grant N000142412289.\r\nM. M. is funded by the European Union (ERC, INF2 , project number 101161364). Views and opinions expressed are, however, those of the author(s) only and do not necessarily\r\nreflect those of the European Union or the European Research Council Executive Agency. Neither the European Union nor the granting authority can be held responsible for them. M.S. is supported by the Packard Fellowship in Science and Engineering, a Sloan Research Fellowship in Mathematics, an NSF-CAREER under award #1846369, DARPA FastNICS program, and NSF-CIF awards #1813877 and #2008443, and NIH DP2LM014564-01. The authors also\r\nacknowledge further support from Open Philanthropy, OpenAI, Amazon Research, Google Research, and Microsoft Research.","_id":"21325","status":"public","oa_version":"Published Version","external_id":{"pmid":["41321376"]},"intvolume":"       267","file":[{"date_updated":"2026-02-19T08:15:48Z","content_type":"application/pdf","creator":"dernst","checksum":"f774f8619a0d72f3975d9cb23942a1e9","access_level":"open_access","relation":"main_file","file_name":"2025_ICML_Gozeten.pdf","file_size":471176,"date_created":"2026-02-19T08:15:48Z","success":1,"file_id":"21336"}]},{"tmp":{"image":"/images/cc_by.png","short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)"},"has_accepted_license":"1","OA_type":"gold","date_published":"2025-07-01T00:00:00Z","ddc":["000"],"OA_place":"publisher","date_updated":"2026-02-19T09:03:53Z","alternative_title":["PMLR"],"file_date_updated":"2026-02-19T09:03:43Z","abstract":[{"text":"Multi-index models provide a popular framework to investigate the learnability of functions with low-dimensional structure and, also due to their connections with neural networks, they have been object of recent intensive study. In this paper, we focus on recovering the subspace spanned by the signals via spectral estimators – a family of methods routinely used in practice, often as a warm-start for iterative algorithms. Our main technical contribution is a precise asymptotic characterization of the performance of spectral methods, when sample size and input dimension grow proportionally and the dimension p of the space to recover is fixed. Specifically, we locate the top-p eigenvalues of the spectral matrix and establish the overlaps between the corresponding eigenvectors (which give the spectral estimators) and a basis of the signal subspace. Our analysis unveils a phase transition phenomenon in which, as the sample complexity grows, eigenvalues escape from the bulk of the spectrum and, when that happens, eigenvectors recover directions of the desired subspace. The precise characterization we put forward enables the optimization of the data preprocessing, thus allowing to identify the spectral estimator that requires the minimal sample size for weak recovery.","lang":"eng"}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","intvolume":"       291","file":[{"relation":"main_file","checksum":"19aa70ab4f57fb9067b6ebb99a5fd6f0","access_level":"open_access","creator":"dernst","content_type":"application/pdf","date_updated":"2026-02-19T09:03:43Z","file_id":"21339","success":1,"date_created":"2026-02-19T09:03:43Z","file_size":844611,"file_name":"2025_LearningTheory_Kovacevic.pdf"}],"external_id":{"arxiv":["2502.01583"]},"oa_version":"Published Version","status":"public","_id":"21328","publisher":"ML Research Press","acknowledgement":"This work was done when Y. Z. was at the Institute of Science and Technology Austria. Y. Z. and\r\nM. M. are funded by the European Union (ERC, INF2, project number 101161364). Views and\r\nopinions expressed are however those of the author(s) only and do not necessarily reflect those of the European Union or the European Research Council Executive Agency. Neither the European Union nor the granting authority can be held responsible for them. The authors would like to acknowledge (in alphabetical order) discussions with Yatin Dandi, Leonardo Defilippis and Bruno Loureiro concerning their parallel work (Defilippis et al., 2025).","citation":{"chicago":"Kovačević, Filip, Zhang Yihan, and Marco Mondelli. “Spectral Estimators for Multi-Index Models: Precise Asymptotics and Optimal Weak Recovery.” In <i>Proceedings of 38th Conference on Learning Theory</i>, 291:3354–3404. ML Research Press, 2025.","short":"F. Kovačević, Z. Yihan, M. Mondelli, in:, Proceedings of 38th Conference on Learning Theory, ML Research Press, 2025, pp. 3354–3404.","mla":"Kovačević, Filip, et al. “Spectral Estimators for Multi-Index Models: Precise Asymptotics and Optimal Weak Recovery.” <i>Proceedings of 38th Conference on Learning Theory</i>, vol. 291, ML Research Press, 2025, pp. 3354–404.","ista":"Kovačević F, Yihan Z, Mondelli M. 2025. Spectral estimators for multi-index models: Precise asymptotics and optimal weak recovery. Proceedings of 38th Conference on Learning Theory. COLT: Conference on Learning Theory, PMLR, vol. 291, 3354–3404.","ama":"Kovačević F, Yihan Z, Mondelli M. Spectral estimators for multi-index models: Precise asymptotics and optimal weak recovery. In: <i>Proceedings of 38th Conference on Learning Theory</i>. Vol 291. ML Research Press; 2025:3354-3404.","apa":"Kovačević, F., Yihan, Z., &#38; Mondelli, M. (2025). Spectral estimators for multi-index models: Precise asymptotics and optimal weak recovery. In <i>Proceedings of 38th Conference on Learning Theory</i> (Vol. 291, pp. 3354–3404). Lyon, France: ML Research Press.","ieee":"F. Kovačević, Z. Yihan, and M. Mondelli, “Spectral estimators for multi-index models: Precise asymptotics and optimal weak recovery,” in <i>Proceedings of 38th Conference on Learning Theory</i>, Lyon, France, 2025, vol. 291, pp. 3354–3404."},"year":"2025","day":"01","arxiv":1,"type":"conference","title":"Spectral estimators for multi-index models: Precise asymptotics and optimal weak recovery","oa":1,"publication_identifier":{"eissn":["2640-3498"]},"scopus_import":"1","conference":{"start_date":"2025-06-30","location":"Lyon, France","name":"COLT: Conference on Learning Theory","end_date":"2025-07-04"},"corr_author":"1","author":[{"last_name":"Kovačević","id":"d0258e7b-50b8-11ef-ad56-8b9f537b6b1b","full_name":"Kovačević, Filip","first_name":"Filip"},{"first_name":"Zhang","last_name":"Yihan","full_name":"Yihan, Zhang"},{"orcid":"0000-0002-3242-7020","first_name":"Marco","full_name":"Mondelli, Marco","last_name":"Mondelli","id":"27EB676C-8706-11E9-9510-7717E6697425"}],"publication":"Proceedings of 38th Conference on Learning Theory","article_processing_charge":"No","date_created":"2026-02-18T12:12:47Z","quality_controlled":"1","project":[{"grant_number":"101161364","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf"}],"publication_status":"published","volume":291,"department":[{"_id":"MaMo"}],"language":[{"iso":"eng"}],"page":"3354-3404","month":"07"},{"publication":"13th International Conference on Learning Representations","author":[{"first_name":"M.","full_name":"Emrullah Ildiz, M.","last_name":"Emrullah Ildiz"},{"first_name":"Halil Alperen","last_name":"Gozeten","full_name":"Gozeten, Halil Alperen"},{"full_name":"Taga, Ege Onur","last_name":"Taga","first_name":"Ege Onur"},{"id":"27EB676C-8706-11E9-9510-7717E6697425","last_name":"Mondelli","full_name":"Mondelli, Marco","orcid":"0000-0002-3242-7020","first_name":"Marco"},{"first_name":"Samet","full_name":"Oymak, Samet","last_name":"Oymak"}],"conference":{"start_date":"2025-04-24","end_date":"2025-04-28","name":"ICLR: International Conference on Learning Representations","location":"Singapore, Singapore"},"scopus_import":"1","publication_identifier":{"isbn":["9798331320850"]},"oa":1,"title":"High-dimensional analysis of knowledge distillation: Weak-to-Strong generalization and scaling laws","type":"conference","month":"04","page":"2967-3006","language":[{"iso":"eng"}],"department":[{"_id":"MaMo"}],"publication_status":"published","project":[{"_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","grant_number":"101161364"}],"quality_controlled":"1","date_created":"2025-07-20T22:02:02Z","article_processing_charge":"No","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","abstract":[{"text":"A growing number of machine learning scenarios rely on knowledge distillation where one uses the output of a surrogate model as labels to supervise the training of a target model. In this work, we provide a sharp characterization of this process for ridgeless, high-dimensional regression, under two settings: (i) model shift, where the surrogate model is arbitrary, and (ii) distribution shift, where the surrogate model is the solution of empirical risk minimization with out-of-distribution data. In both cases, we characterize the precise risk of the target model through non-asymptotic bounds in terms of sample size and data distribution under mild conditions. As a consequence, we identify the form of the optimal surrogate model, which reveals the benefits and limitations of discarding weak features in a data-dependent fashion. In the context of weak-to-strong (W2S) generalization, this has the interpretation that (i) W2S training, with the surrogate as the weak model, can provably outperform training with strong labels under the same data budget, but (ii) it is unable to improve the data scaling law. We validate our results on numerical experiments both on ridgeless regression and on neural network architectures.","lang":"eng"}],"file_date_updated":"2025-08-04T08:32:38Z","date_updated":"2025-08-04T08:33:58Z","OA_place":"publisher","ddc":["000"],"OA_type":"diamond","has_accepted_license":"1","date_published":"2025-04-01T00:00:00Z","tmp":{"image":"/images/cc_by.png","short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)"},"day":"01","arxiv":1,"year":"2025","citation":{"ieee":"M. Emrullah Ildiz, H. A. Gozeten, E. O. Taga, M. Mondelli, and S. Oymak, “High-dimensional analysis of knowledge distillation: Weak-to-Strong generalization and scaling laws,” in <i>13th International Conference on Learning Representations</i>, Singapore, Singapore, 2025, pp. 2967–3006.","apa":"Emrullah Ildiz, M., Gozeten, H. A., Taga, E. O., Mondelli, M., &#38; Oymak, S. (2025). High-dimensional analysis of knowledge distillation: Weak-to-Strong generalization and scaling laws. In <i>13th International Conference on Learning Representations</i> (pp. 2967–3006). Singapore, Singapore: ICLR.","ista":"Emrullah Ildiz M, Gozeten HA, Taga EO, Mondelli M, Oymak S. 2025. High-dimensional analysis of knowledge distillation: Weak-to-Strong generalization and scaling laws. 13th International Conference on Learning Representations. ICLR: International Conference on Learning Representations, 2967–3006.","ama":"Emrullah Ildiz M, Gozeten HA, Taga EO, Mondelli M, Oymak S. High-dimensional analysis of knowledge distillation: Weak-to-Strong generalization and scaling laws. In: <i>13th International Conference on Learning Representations</i>. ICLR; 2025:2967-3006.","mla":"Emrullah Ildiz, M., et al. “High-Dimensional Analysis of Knowledge Distillation: Weak-to-Strong Generalization and Scaling Laws.” <i>13th International Conference on Learning Representations</i>, ICLR, 2025, pp. 2967–3006.","short":"M. Emrullah Ildiz, H.A. Gozeten, E.O. Taga, M. Mondelli, S. Oymak, in:, 13th International Conference on Learning Representations, ICLR, 2025, pp. 2967–3006.","chicago":"Emrullah Ildiz, M., Halil Alperen Gozeten, Ege Onur Taga, Marco Mondelli, and Samet Oymak. “High-Dimensional Analysis of Knowledge Distillation: Weak-to-Strong Generalization and Scaling Laws.” In <i>13th International Conference on Learning Representations</i>, 2967–3006. ICLR, 2025."},"acknowledgement":"M.E.I., H.A.G., E.O.T., S.O. are supported by the NSF grants CCF-2046816, CCF-2403075, the Office of Naval Research grant N000142412289, an OpenAI Agentic AI Systems grant, and gifts by Open Philanthropy and Google Research. M. M. is funded by the European Union (ERC, INF2, project number 101161364). Views and opinions expressed are however those of the author(s) only and do not necessarily reflect those of the European Union or the European Research Council Executive Agency. Neither the European Union nor the granting authority can be held responsible for them.","publisher":"ICLR","_id":"20033","status":"public","oa_version":"Published Version","external_id":{"arxiv":["2410.18837"]},"file":[{"date_updated":"2025-08-04T08:32:38Z","content_type":"application/pdf","creator":"dernst","relation":"main_file","checksum":"5a38b093ebb4ee4eb662ea142621a5ca","access_level":"open_access","file_name":"2025_ICLR_Ildiz.pdf","date_created":"2025-08-04T08:32:38Z","file_size":528171,"file_id":"20112","success":1}]},{"tmp":{"image":"/images/cc_by.png","short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)"},"has_accepted_license":"1","OA_type":"diamond","date_published":"2025-04-01T00:00:00Z","ddc":["000"],"OA_place":"publisher","date_updated":"2025-08-04T08:47:00Z","abstract":[{"lang":"eng","text":"Deep neural networks (DNNs) at convergence consistently represent the training data in the last layer via a geometric structure referred to as neural collapse. This empirical evidence has spurred a line of theoretical research aimed at proving the emergence of neural collapse, mostly focusing on the unconstrained features model. Here, the features of the penultimate layer are free variables, which makes the model data-agnostic and puts into question its ability to capture DNN training. Our work addresses the issue, moving away from unconstrained features and\r\nstudying DNNs that end with at least two linear layers. We first prove generic guarantees on neural collapse that assume (i) low training error and balancedness of linear layers (for within-class variability collapse), and (ii) bounded conditioning of the features before the linear part (for orthogonality of class-means, and their alignment with weight matrices). The balancedness refers to the fact that W⊤ℓ+1Wℓ+1 ≈ WℓW⊤ℓfor any pair of consecutive weight matrices of the linear part, and the bounded conditioning requires a well-behaved ratio between largest and smallest non-zero singular values of the features. We then show that such assumptions hold for gradient descent training with weight decay: (i) for networks with a wide first layer, we prove low training error and balancedness, and (ii) for solutions that are either nearly optimal or stable under large learning rates, we additionally prove the bounded conditioning. Taken together, our results are the first to show neural collapse in the end-to-end training of DNNs."}],"file_date_updated":"2025-08-04T08:45:43Z","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","file":[{"relation":"main_file","checksum":"59c48c173887139647cc9839c0801136","access_level":"open_access","creator":"dernst","content_type":"application/pdf","date_updated":"2025-08-04T08:45:43Z","file_id":"20114","success":1,"file_size":1337236,"date_created":"2025-08-04T08:45:43Z","file_name":"2025_ICLR_Jacot.pdf"}],"external_id":{"arxiv":["2410.04887"]},"oa_version":"Published Version","status":"public","_id":"20035","publisher":"ICLR","citation":{"apa":"Jacot, A., Súkeník, P., Wang, Z., &#38; Mondelli, M. (2025). Wide neural networks trained with weight decay provably exhibit neural collapse. In <i>13th International Conference on Learning Representations</i> (pp. 1905–1931). Singapore, Singapore: ICLR.","ieee":"A. Jacot, P. Súkeník, Z. Wang, and M. Mondelli, “Wide neural networks trained with weight decay provably exhibit neural collapse,” in <i>13th International Conference on Learning Representations</i>, Singapore, Singapore, 2025, pp. 1905–1931.","ista":"Jacot A, Súkeník P, Wang Z, Mondelli M. 2025. Wide neural networks trained with weight decay provably exhibit neural collapse. 13th International Conference on Learning Representations. ICLR: International Conference on Learning Representations, 1905–1931.","ama":"Jacot A, Súkeník P, Wang Z, Mondelli M. Wide neural networks trained with weight decay provably exhibit neural collapse. In: <i>13th International Conference on Learning Representations</i>. ICLR; 2025:1905-1931.","short":"A. Jacot, P. Súkeník, Z. Wang, M. Mondelli, in:, 13th International Conference on Learning Representations, ICLR, 2025, pp. 1905–1931.","mla":"Jacot, Arthur, et al. “Wide Neural Networks Trained with Weight Decay Provably Exhibit Neural Collapse.” <i>13th International Conference on Learning Representations</i>, ICLR, 2025, pp. 1905–31.","chicago":"Jacot, Arthur, Peter Súkeník, Zihan Wang, and Marco Mondelli. “Wide Neural Networks Trained with Weight Decay Provably Exhibit Neural Collapse.” In <i>13th International Conference on Learning Representations</i>, 1905–31. ICLR, 2025."},"year":"2025","acknowledgement":"M. M. and P. S. are funded by the European Union (ERC, INF2, project number 101161364). Views and opinions expressed are however those of the author(s) only and do not necessarily reflect those of the European Union or the European Research Council Executive Agency. Neither the European Union nor the granting authority can be held responsible for them.","arxiv":1,"day":"01","type":"conference","oa":1,"title":"Wide neural networks trained with weight decay provably exhibit neural collapse","publication_identifier":{"isbn":["9798331320850"]},"scopus_import":"1","conference":{"name":"ICLR: International Conference on Learning Representations","end_date":"2025-04-28","location":"Singapore, Singapore","start_date":"2025-04-24"},"corr_author":"1","author":[{"first_name":"Arthur","last_name":"Jacot","full_name":"Jacot, Arthur"},{"first_name":"Peter","full_name":"Súkeník, Peter","id":"d64d6a8d-eb8e-11eb-b029-96fd216dec3c","last_name":"Súkeník"},{"full_name":"Wang, Zihan","last_name":"Wang","first_name":"Zihan"},{"last_name":"Mondelli","id":"27EB676C-8706-11E9-9510-7717E6697425","full_name":"Mondelli, Marco","first_name":"Marco","orcid":"0000-0002-3242-7020"}],"publication":"13th International Conference on Learning Representations","article_processing_charge":"No","date_created":"2025-07-20T22:02:02Z","quality_controlled":"1","project":[{"name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf","grant_number":"101161364"}],"publication_status":"published","language":[{"iso":"eng"}],"department":[{"_id":"MaMo"}],"page":"1905-1931","month":"04"},{"das_tickbox":"0","arxiv":1,"day":"02","publisher":"Neural Information Processing Systems Foundation","citation":{"chicago":"Súkeník, Peter, Christoph Lampert, and Marco Mondelli. “Neural Collapse Is Globally Optimal in Deep Regularized ResNets and Transformers.” In <i>39th Conference on Neural Information Processing Systems</i>, 38:48646–77. Neural Information Processing Systems Foundation, 2025. <a href=\"https://doi.org/10.52202/085713-1450\">https://doi.org/10.52202/085713-1450</a>.","mla":"Súkeník, Peter, et al. “Neural Collapse Is Globally Optimal in Deep Regularized ResNets and Transformers.” <i>39th Conference on Neural Information Processing Systems</i>, vol. 38, Neural Information Processing Systems Foundation, 2025, pp. 48646–77, doi:<a href=\"https://doi.org/10.52202/085713-1450\">10.52202/085713-1450</a>.","short":"P. Súkeník, C. Lampert, M. Mondelli, in:, 39th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2025, pp. 48646–48677.","ama":"Súkeník P, Lampert C, Mondelli M. Neural collapse is globally optimal in deep regularized ResNets and transformers. In: <i>39th Conference on Neural Information Processing Systems</i>. Vol 38. Neural Information Processing Systems Foundation; 2025:48646-48677. doi:<a href=\"https://doi.org/10.52202/085713-1450\">10.52202/085713-1450</a>","ista":"Súkeník P, Lampert C, Mondelli M. 2025. Neural collapse is globally optimal in deep regularized ResNets and transformers. 39th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 38, 48646–48677.","apa":"Súkeník, P., Lampert, C., &#38; Mondelli, M. (2025). Neural collapse is globally optimal in deep regularized ResNets and transformers. In <i>39th Conference on Neural Information Processing Systems</i> (Vol. 38, pp. 48646–48677). San Diego, CA, United States: Neural Information Processing Systems Foundation. <a href=\"https://doi.org/10.52202/085713-1450\">https://doi.org/10.52202/085713-1450</a>","ieee":"P. Súkeník, C. Lampert, and M. Mondelli, “Neural collapse is globally optimal in deep regularized ResNets and transformers,” in <i>39th Conference on Neural Information Processing Systems</i>, San Diego, CA, United States, 2025, vol. 38, pp. 48646–48677."},"year":"2025","acknowledgement":"M. M. and P. S. are funded by the European Union (ERC, INF2\r\n, project number 101161364). Views\r\nand opinions expressed are however those of the author(s) only and do not necessarily reflect those\r\nof the European Union or the European Research Council Executive Agency. Neither the European\r\nUnion nor the granting authority can be held responsible for them. This research was supported\r\nby the Scientific Service Units (SSU) of ISTA through resources provided by Scientific Computing (SciComp).","_id":"22825","researchdata_availability":"no","status":"public","oa_version":"Published Version","external_id":{"arxiv":["2505.15239"]},"intvolume":"        38","acknowledged_ssus":[{"_id":"ScienComp"}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","abstract":[{"lang":"eng","text":"The empirical emergence of neural collapse—a surprising symmetry in the feature representations of the training data in the penultimate layer of deep neural\r\nnetworks—has spurred a line of theoretical research aimed at its understanding.\r\nHowever, existing work focuses on data-agnostic models or, when data structure is\r\ntaken into account, it remains limited to multi-layer perceptrons. Our paper fills\r\nboth these gaps by analyzing modern architectures in a data-aware regime: we\r\nprove that global optima of deep regularized transformers and residual networks\r\n(ResNets) with LayerNorm trained with cross entropy or mean squared error loss\r\nare approximately collapsed, and the approximation gets tighter as the depth grows.\r\nMore generally, we formally reduce any end-to-end large-depth ResNet or transformer training into an equivalent unconstrained features model, thus justifying its\r\nwide use in the literature even beyond data-agnostic settings. Our theoretical results\r\nare supported by experiments on computer vision and language datasets showing\r\nthat, as the depth grows, neural collapse indeed becomes more prominent."}],"alternative_title":["Advances in Neural Information Processing Systems"],"date_updated":"2026-09-10T08:11:46Z","OA_place":"publisher","OA_type":"free access","date_published":"2025-12-02T00:00:00Z","ddc":["000"],"fulldoi":"https://doi.org/10.52202/085713-1450","month":"12","page":"48646-48677","publication_status":"published","language":[{"iso":"eng"}],"department":[{"_id":"MaMo"},{"_id":"GradSch"},{"_id":"ChLa"}],"volume":38,"project":[{"_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","grant_number":"101161364"}],"quality_controlled":"1","article_processing_charge":"No","date_created":"2026-09-06T22:01:59Z","supplementarymaterial":"no","publication":"39th Conference on Neural Information Processing Systems","author":[{"id":"d64d6a8d-eb8e-11eb-b029-96fd216dec3c","last_name":"Súkeník","full_name":"Súkeník, Peter","first_name":"Peter"},{"orcid":"0000-0001-8622-7887","first_name":"Christoph","id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","last_name":"Lampert","full_name":"Lampert, Christoph"},{"last_name":"Mondelli","id":"27EB676C-8706-11E9-9510-7717E6697425","full_name":"Mondelli, Marco","first_name":"Marco","orcid":"0000-0002-3242-7020"}],"conference":{"start_date":"2025-12-02","name":"NeurIPS: Neural Information Processing Systems","end_date":"2025-12-07","location":"San Diego, CA, United States"},"corr_author":"1","scopus_import":"1","oa":1,"title":"Neural collapse is globally optimal in deep regularized ResNets and transformers","publication_identifier":{"isbn":["9798331338275"],"issn":["1049-5258"]},"type":"conference","doi":"10.52202/085713-1450","main_file_link":[{"open_access":"1","url":"https://doi.org/10.52202/085713-1450"}]},{"month":"07","language":[{"iso":"eng"}],"department":[{"_id":"MaMo"}],"volume":267,"publication_status":"published","page":"4839-4873","quality_controlled":"1","related_material":{"record":[{"id":"22857","relation":"dissertation_contains","status":"public"}]},"project":[{"grant_number":"101161364","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits","_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf"},{"_id":"92099302-16d5-11f0-9cad-f9a785f54fbd","name":"Trustworthy Deep Learning Theory: Private Over-Parameterized Models and Robust LLMs"}],"date_created":"2026-02-18T11:58:00Z","article_processing_charge":"No","author":[{"full_name":"Bombari, Simone","id":"ca726dda-de17-11ea-bc14-f9da834f63aa","last_name":"Bombari","first_name":"Simone"},{"first_name":"Marco","orcid":"0000-0002-3242-7020","full_name":"Mondelli, Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","last_name":"Mondelli"}],"publication":"Proceedings of the 42nd International Conference on Machine Learning","corr_author":"1","conference":{"name":"ICML: International Conference on Machine Learning","end_date":"2025-07-19","location":"Vancouver, Canada","start_date":"2025-07-13"},"type":"conference","publication_identifier":{"eissn":["2640-3498"]},"oa":1,"title":"Spurious correlations in high dimensional regression: The roles of regularization, simplicity bias and over-parameterization","year":"2025","citation":{"chicago":"Bombari, Simone, and Marco Mondelli. “Spurious Correlations in High Dimensional Regression: The Roles of Regularization, Simplicity Bias and over-Parameterization.” In <i>Proceedings of the 42nd International Conference on Machine Learning</i>, 267:4839–73. ML Research Press, 2025.","mla":"Bombari, Simone, and Marco Mondelli. “Spurious Correlations in High Dimensional Regression: The Roles of Regularization, Simplicity Bias and over-Parameterization.” <i>Proceedings of the 42nd International Conference on Machine Learning</i>, vol. 267, ML Research Press, 2025, pp. 4839–73.","short":"S. Bombari, M. Mondelli, in:, Proceedings of the 42nd International Conference on Machine Learning, ML Research Press, 2025, pp. 4839–4873.","ama":"Bombari S, Mondelli M. Spurious correlations in high dimensional regression: The roles of regularization, simplicity bias and over-parameterization. In: <i>Proceedings of the 42nd International Conference on Machine Learning</i>. Vol 267. ML Research Press; 2025:4839-4873.","ista":"Bombari S, Mondelli M. 2025. Spurious correlations in high dimensional regression: The roles of regularization, simplicity bias and over-parameterization. Proceedings of the 42nd International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 267, 4839–4873.","apa":"Bombari, S., &#38; Mondelli, M. (2025). Spurious correlations in high dimensional regression: The roles of regularization, simplicity bias and over-parameterization. In <i>Proceedings of the 42nd International Conference on Machine Learning</i> (Vol. 267, pp. 4839–4873). Vancouver, Canada: ML Research Press.","ieee":"S. Bombari and M. Mondelli, “Spurious correlations in high dimensional regression: The roles of regularization, simplicity bias and over-parameterization,” in <i>Proceedings of the 42nd International Conference on Machine Learning</i>, Vancouver, Canada, 2025, vol. 267, pp. 4839–4873."},"acknowledgement":"Marco Mondelli is funded by the European Union (ERC, INF2, project number 101161364). Views and opinions expressed are however those of the author(s) only and do not necessarily reflect those of the European Union or the European Research Council Executive Agency. Neither the European Union nor the granting authority can be held responsible for them. Simone Bombari is supported by a Google PhD fellowship. The authors would like to thank GuanWen Qiu for helpful discussions.","publisher":"ML Research Press","arxiv":1,"day":"30","oa_version":"Published Version","_id":"21324","status":"public","file":[{"file_name":"2025_ICML_Bombari.pdf","file_size":887526,"date_created":"2026-02-19T08:04:38Z","success":1,"file_id":"21335","date_updated":"2026-02-19T08:04:38Z","content_type":"application/pdf","creator":"dernst","access_level":"open_access","checksum":"d4ba4f7717b362ca38878f45e57bd643","relation":"main_file"}],"intvolume":"       267","external_id":{"arxiv":["2502.01347"]},"file_date_updated":"2026-02-19T08:04:38Z","abstract":[{"lang":"eng","text":"Learning models have been shown to rely on spurious correlations between non-predictive features and the associated labels in the training data, with negative implications on robustness, bias and fairness. In this work, we provide a statistical characterization of this phenomenon for high-dimensional regression, when the data contains a predictive core feature x and a spurious feature y. Specifically, we quantify the amount of spurious correlations C learned via linear regression, in terms of the data covariance and the strength λ of the ridge regularization. As a consequence, we first capture the simplicity of y through the spectrum of its covariance, and its correlation with x through the Schur complement of the full data covariance. Next, we prove a trade-off between C and the in-distribution test loss L, by showing that the value of λ that minimizes L lies in an interval where C is increasing. Finally, we investigate the effects of over-parameterization via the random features model, by showing its equivalence to regularized linear regression. Our theoretical results are supported by numerical experiments on Gaussian, Color-MNIST, and CIFAR-10 datasets."}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","alternative_title":["PMLR"],"date_updated":"2026-09-21T13:07:01Z","OA_place":"publisher","tmp":{"image":"/images/cc_by.png","short":"CC BY (4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)"},"ddc":["000"],"OA_type":"gold","has_accepted_license":"1","date_published":"2025-07-30T00:00:00Z"}]