[{"date_updated":"2026-09-10T08:11:46Z","date_created":"2026-09-06T22:01:59Z","doi":"10.52202/085713-1450","oa_version":"Published Version","ddc":["000"],"corr_author":"1","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","arxiv":1,"conference":{"location":"San Diego, CA, United States","end_date":"2025-12-07","start_date":"2025-12-02","name":"NeurIPS: Neural Information Processing Systems"},"month":"12","project":[{"_id":"911e6d1f-16d5-11f0-9cad-c5c68c6a1cdf","grant_number":"101161364","name":"Inference in High Dimensions: Light-speed Algorithms and Information Limits"}],"page":"48646-48677","type":"conference","supplementarymaterial":"no","department":[{"_id":"MaMo"},{"_id":"GradSch"},{"_id":"ChLa"}],"OA_place":"publisher","main_file_link":[{"url":"https://doi.org/10.52202/085713-1450","open_access":"1"}],"scopus_import":"1","oa":1,"fulldoi":"https://doi.org/10.52202/085713-1450","publication_identifier":{"isbn":["9798331338275"],"issn":["1049-5258"]},"citation":{"short":"P. Súkeník, C. Lampert, M. Mondelli, in:, 39th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2025, pp. 48646–48677.","ista":"Súkeník P, Lampert C, Mondelli M. 2025. Neural collapse is globally optimal in deep regularized ResNets and transformers. 39th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 38, 48646–48677.","chicago":"Súkeník, Peter, Christoph Lampert, and Marco Mondelli. “Neural Collapse Is Globally Optimal in Deep Regularized ResNets and Transformers.” In <i>39th Conference on Neural Information Processing Systems</i>, 38:48646–77. Neural Information Processing Systems Foundation, 2025. <a href=\"https://doi.org/10.52202/085713-1450\">https://doi.org/10.52202/085713-1450</a>.","ama":"Súkeník P, Lampert C, Mondelli M. Neural collapse is globally optimal in deep regularized ResNets and transformers. In: <i>39th Conference on Neural Information Processing Systems</i>. Vol 38. Neural Information Processing Systems Foundation; 2025:48646-48677. doi:<a href=\"https://doi.org/10.52202/085713-1450\">10.52202/085713-1450</a>","apa":"Súkeník, P., Lampert, C., &#38; Mondelli, M. (2025). Neural collapse is globally optimal in deep regularized ResNets and transformers. In <i>39th Conference on Neural Information Processing Systems</i> (Vol. 38, pp. 48646–48677). San Diego, CA, United States: Neural Information Processing Systems Foundation. <a href=\"https://doi.org/10.52202/085713-1450\">https://doi.org/10.52202/085713-1450</a>","ieee":"P. Súkeník, C. Lampert, and M. Mondelli, “Neural collapse is globally optimal in deep regularized ResNets and transformers,” in <i>39th Conference on Neural Information Processing Systems</i>, San Diego, CA, United States, 2025, vol. 38, pp. 48646–48677.","mla":"Súkeník, Peter, et al. “Neural Collapse Is Globally Optimal in Deep Regularized ResNets and Transformers.” <i>39th Conference on Neural Information Processing Systems</i>, vol. 38, Neural Information Processing Systems Foundation, 2025, pp. 48646–77, doi:<a href=\"https://doi.org/10.52202/085713-1450\">10.52202/085713-1450</a>."},"acknowledged_ssus":[{"_id":"ScienComp"}],"day":"02","OA_type":"free access","year":"2025","publication_status":"published","date_published":"2025-12-02T00:00:00Z","researchdata_availability":"no","author":[{"first_name":"Peter","last_name":"Súkeník","full_name":"Súkeník, Peter","id":"d64d6a8d-eb8e-11eb-b029-96fd216dec3c"},{"id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","full_name":"Lampert, Christoph","first_name":"Christoph","orcid":"0000-0001-8622-7887","last_name":"Lampert"},{"full_name":"Mondelli, Marco","id":"27EB676C-8706-11E9-9510-7717E6697425","last_name":"Mondelli","orcid":"0000-0002-3242-7020","first_name":"Marco"}],"title":"Neural collapse is globally optimal in deep regularized ResNets and transformers","article_processing_charge":"No","publisher":"Neural Information Processing Systems Foundation","_id":"22825","alternative_title":["Advances in Neural Information Processing Systems"],"quality_controlled":"1","das_tickbox":"0","external_id":{"arxiv":["2505.15239"]},"abstract":[{"text":"The empirical emergence of neural collapse—a surprising symmetry in the feature representations of the training data in the penultimate layer of deep neural\r\nnetworks—has spurred a line of theoretical research aimed at its understanding.\r\nHowever, existing work focuses on data-agnostic models or, when data structure is\r\ntaken into account, it remains limited to multi-layer perceptrons. Our paper fills\r\nboth these gaps by analyzing modern architectures in a data-aware regime: we\r\nprove that global optima of deep regularized transformers and residual networks\r\n(ResNets) with LayerNorm trained with cross entropy or mean squared error loss\r\nare approximately collapsed, and the approximation gets tighter as the depth grows.\r\nMore generally, we formally reduce any end-to-end large-depth ResNet or transformer training into an equivalent unconstrained features model, thus justifying its\r\nwide use in the literature even beyond data-agnostic settings. Our theoretical results\r\nare supported by experiments on computer vision and language datasets showing\r\nthat, as the depth grows, neural collapse indeed becomes more prominent.","lang":"eng"}],"language":[{"iso":"eng"}],"intvolume":"        38","acknowledgement":"M. M. and P. S. are funded by the European Union (ERC, INF2\r\n, project number 101161364). Views\r\nand opinions expressed are however those of the author(s) only and do not necessarily reflect those\r\nof the European Union or the European Research Council Executive Agency. Neither the European\r\nUnion nor the granting authority can be held responsible for them. This research was supported\r\nby the Scientific Service Units (SSU) of ISTA through resources provided by Scientific Computing (SciComp).","status":"public","volume":38,"publication":"39th Conference on Neural Information Processing Systems"}]