[{"author":[{"first_name":"Thomas","last_name":"Robert","full_name":"Robert, Thomas"},{"full_name":"Safaryan, Mher","last_name":"Safaryan","first_name":"Mher","id":"dd546b39-0804-11ed-9c55-ef075c39778d"},{"last_name":"Modoranu","first_name":"Ionut-Vlad","id":"449f7a18-f128-11eb-9611-9b430c0c6333","full_name":"Modoranu, Ionut-Vlad"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"}],"department":[{"_id":"DaAl"}],"tmp":{"short":"CC BY (4.0)","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode"},"quality_controlled":"1","abstract":[{"lang":"eng","text":"We introduce LDAdam, a memory-efficient optimizer for training large models, that performs adaptive optimization steps within lower dimensional subspaces, while consistently exploring the full parameter space during training. This strategy keeps the optimizer's memory footprint to a fraction of the model size. LDAdam relies on a new projection-aware update rule for the optimizer states that allows for transitioning between subspaces, i.e., estimation of the statistics of the projected gradients. To mitigate the errors due to low-rank projection, LDAdam integrates a new generalized error feedback mechanism, which explicitly accounts for both gradient and optimizer state compression. We prove the convergence of LDAdam under standard assumptions, and provide empirical evidence that LDAdam allows for efficient fine-tuning and pre-training of language models."}],"arxiv":1,"publication_identifier":{"isbn":["9798331320850"]},"corr_author":"1","type":"conference","title":"LDAdam: Adaptive optimization from low-dimensional gradient statistics","related_material":{"link":[{"url":"https://github.com/IST-DASLab/LDAdam","relation":"software"}]},"language":[{"iso":"eng"}],"scopus_import":"1","OA_place":"publisher","date_created":"2025-07-20T22:02:02Z","page":"101877-101913","_id":"20034","date_published":"2025-04-01T00:00:00Z","publisher":"ICLR","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","conference":{"end_date":"2025-04-28","location":"Singapore, Singapore","name":"ICLR: International Conference on Learning Representations","start_date":"2025-04-24"},"oa_version":"Published Version","OA_type":"diamond","oa":1,"file_date_updated":"2025-08-04T08:39:51Z","file":[{"checksum":"9327d82569358d7bf1c3ec1a9952e721","access_level":"open_access","relation":"main_file","date_created":"2025-08-04T08:39:51Z","success":1,"file_name":"2025_ICLR_Robert.pdf","file_id":"20113","content_type":"application/pdf","creator":"dernst","date_updated":"2025-08-04T08:39:51Z","file_size":1346111}],"day":"01","article_processing_charge":"No","date_updated":"2025-08-04T08:41:10Z","status":"public","publication_status":"published","ddc":["000"],"month":"04","publication":"13th International Conference on Learning Representations","external_id":{"arxiv":["2410.16103"]},"citation":{"ieee":"T. Robert, M. Safaryan, I.-V. Modoranu, and D.-A. Alistarh, “LDAdam: Adaptive optimization from low-dimensional gradient statistics,” in <i>13th International Conference on Learning Representations</i>, Singapore, Singapore, 2025, pp. 101877–101913.","short":"T. Robert, M. Safaryan, I.-V. Modoranu, D.-A. Alistarh, in:, 13th International Conference on Learning Representations, ICLR, 2025, pp. 101877–101913.","mla":"Robert, Thomas, et al. “LDAdam: Adaptive Optimization from Low-Dimensional Gradient Statistics.” <i>13th International Conference on Learning Representations</i>, ICLR, 2025, pp. 101877–913.","ista":"Robert T, Safaryan M, Modoranu I-V, Alistarh D-A. 2025. LDAdam: Adaptive optimization from low-dimensional gradient statistics. 13th International Conference on Learning Representations. ICLR: International Conference on Learning Representations, 101877–101913.","chicago":"Robert, Thomas, Mher Safaryan, Ionut-Vlad Modoranu, and Dan-Adrian Alistarh. “LDAdam: Adaptive Optimization from Low-Dimensional Gradient Statistics.” In <i>13th International Conference on Learning Representations</i>, 101877–913. ICLR, 2025.","ama":"Robert T, Safaryan M, Modoranu I-V, Alistarh D-A. LDAdam: Adaptive optimization from low-dimensional gradient statistics. In: <i>13th International Conference on Learning Representations</i>. ICLR; 2025:101877-101913.","apa":"Robert, T., Safaryan, M., Modoranu, I.-V., &#38; Alistarh, D.-A. (2025). LDAdam: Adaptive optimization from low-dimensional gradient statistics. In <i>13th International Conference on Learning Representations</i> (pp. 101877–101913). Singapore, Singapore: ICLR."},"year":"2025","has_accepted_license":"1"},{"ec_funded":1,"publication":"Proceedings of The 27th International Conference on Artificial Intelligence and Statistics","month":"05","citation":{"ama":"Islamov R, Safaryan M, Alistarh D-A. AsGrad: A sharp unified analysis of asynchronous-SGD algorithms. In: <i>Proceedings of The 27th International Conference on Artificial Intelligence and Statistics</i>. Vol 238. ML Research Press; 2024:649-657.","apa":"Islamov, R., Safaryan, M., &#38; Alistarh, D.-A. (2024). AsGrad: A sharp unified analysis of asynchronous-SGD algorithms. In <i>Proceedings of The 27th International Conference on Artificial Intelligence and Statistics</i> (Vol. 238, pp. 649–657). Valencia, Spain: ML Research Press.","chicago":"Islamov, Rustem, Mher Safaryan, and Dan-Adrian Alistarh. “AsGrad: A Sharp Unified Analysis of Asynchronous-SGD Algorithms.” In <i>Proceedings of The 27th International Conference on Artificial Intelligence and Statistics</i>, 238:649–57. ML Research Press, 2024.","ista":"Islamov R, Safaryan M, Alistarh D-A. 2024. AsGrad: A sharp unified analysis of asynchronous-SGD algorithms. Proceedings of The 27th International Conference on Artificial Intelligence and Statistics. AISTATS: Conference on Artificial Intelligence and Statistics, PMLR, vol. 238, 649–657.","mla":"Islamov, Rustem, et al. “AsGrad: A Sharp Unified Analysis of Asynchronous-SGD Algorithms.” <i>Proceedings of The 27th International Conference on Artificial Intelligence and Statistics</i>, vol. 238, ML Research Press, 2024, pp. 649–57.","short":"R. Islamov, M. Safaryan, D.-A. Alistarh, in:, Proceedings of The 27th International Conference on Artificial Intelligence and Statistics, ML Research Press, 2024, pp. 649–657.","ieee":"R. Islamov, M. Safaryan, and D.-A. Alistarh, “AsGrad: A sharp unified analysis of asynchronous-SGD algorithms,” in <i>Proceedings of The 27th International Conference on Artificial Intelligence and Statistics</i>, Valencia, Spain, 2024, vol. 238, pp. 649–657."},"year":"2024","external_id":{"arxiv":["2310.20452"]},"volume":238,"day":"15","article_processing_charge":"No","project":[{"name":"IST-BRIDGE: International postdoctoral program","call_identifier":"H2020","_id":"fc2ed2f7-9c52-11eb-aca3-c01059dda49c","grant_number":"101034413"}],"oa":1,"OA_type":"green","oa_version":"Preprint","publication_status":"published","status":"public","date_updated":"2025-04-14T07:54:52Z","date_published":"2024-05-15T00:00:00Z","intvolume":"       238","_id":"18976","alternative_title":["PMLR"],"page":"649-657","OA_place":"repository","date_created":"2025-01-30T08:15:49Z","scopus_import":"1","language":[{"iso":"eng"}],"acknowledgement":"The authors thank all anonymous reviewers for their valuable comments and suggestions on how to improve the manuscript. This work was done when Rustem Islamov was a Master’s student at Institut Polytechnique de Paris (IP Paris) and an intern at Institute of Science and Technology Austria (ISTA). The research of Rustem Islamov was supported by ISTA internship\r\nprogram. Mher Safaryan has received funding from the European Union’s Horizon 2020 research and innovation program under the Marie Skłodowska-Curie grant agreement No 101034413.","conference":{"end_date":"2024-05-04","start_date":"2024-05-02","name":"AISTATS: Conference on Artificial Intelligence and Statistics","location":"Valencia, Spain"},"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","publisher":"ML Research Press","abstract":[{"lang":"eng","text":"We analyze asynchronous-type algorithms for distributed SGD in the heterogeneous setting, where each worker has its own computation and communication speeds, as well as data distribution. In these algorithms, workers compute possibly stale and stochastic gradients associated with their local data at some iteration back in history and then return those gradients to the server without synchronizing with other workers. We present a unified convergence theory for non-convex smooth functions in the heterogeneous regime. The proposed analysis provides convergence for pure asynchronous SGD and its various modifications. Moreover, our theory explains what affects the convergence rate and what can be done to improve the performance of asynchronous algorithms. In particular, we introduce a novel asynchronous method based on worker shuffling. As a by-product of our analysis, we also demonstrate convergence guarantees for gradient-type algorithms such as SGD with random reshuffling and shuffle-once mini-batch SGD. The derived rates match the best-known results for those algorithms, highlighting the tightness of our approach. Finally, our numerical evaluations support theoretical findings and show the good practical performance of our method."}],"quality_controlled":"1","department":[{"_id":"DaAl"}],"author":[{"last_name":"Islamov","first_name":"Rustem","full_name":"Islamov, Rustem"},{"full_name":"Safaryan, Mher","last_name":"Safaryan","id":"dd546b39-0804-11ed-9c55-ef075c39778d","first_name":"Mher"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"title":"AsGrad: A sharp unified analysis of asynchronous-SGD algorithms","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2310.20452"}],"corr_author":"1","type":"conference","arxiv":1,"publication_identifier":{"eissn":["2640-3498"]}},{"ec_funded":1,"acknowledged_ssus":[{"_id":"CampIT"}],"month":"12","publication":"38th Conference on Neural Information Processing Systems","year":"2024","citation":{"ama":"Modoranu I-V, Safaryan M, Malinovsky G, et al. MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence. In: <i>38th Conference on Neural Information Processing Systems</i>. Vol 37. Neural Information Processing Systems Foundation; 2024.","apa":"Modoranu, I.-V., Safaryan, M., Malinovsky, G., Kurtic, E., Robert, T., Richtárik, P., &#38; Alistarh, D.-A. (2024). MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence. In <i>38th Conference on Neural Information Processing Systems</i> (Vol. 37). Neural Information Processing Systems Foundation.","chicago":"Modoranu, Ionut-Vlad, Mher Safaryan, Grigory Malinovsky, Eldar Kurtic, Thomas Robert, Peter Richtárik, and Dan-Adrian Alistarh. “MICROADAM: Accurate Adaptive Optimization with Low Space Overhead and Provable Convergence.” In <i>38th Conference on Neural Information Processing Systems</i>, Vol. 37. Neural Information Processing Systems Foundation, 2024.","mla":"Modoranu, Ionut-Vlad, et al. “MICROADAM: Accurate Adaptive Optimization with Low Space Overhead and Provable Convergence.” <i>38th Conference on Neural Information Processing Systems</i>, vol. 37, Neural Information Processing Systems Foundation, 2024.","short":"I.-V. Modoranu, M. Safaryan, G. Malinovsky, E. Kurtic, T. Robert, P. Richtárik, D.-A. Alistarh, in:, 38th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2024.","ista":"Modoranu I-V, Safaryan M, Malinovsky G, Kurtic E, Robert T, Richtárik P, Alistarh D-A. 2024. MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence. 38th Conference on Neural Information Processing Systems. , Advances in Neural Information Processing Systems, vol. 37.","ieee":"I.-V. Modoranu <i>et al.</i>, “MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence,” in <i>38th Conference on Neural Information Processing Systems</i>, 2024, vol. 37."},"external_id":{"arxiv":["2405.15593"]},"volume":37,"article_processing_charge":"No","day":"20","project":[{"grant_number":"101034413","name":"IST-BRIDGE: International postdoctoral program","_id":"fc2ed2f7-9c52-11eb-aca3-c01059dda49c","call_identifier":"H2020"}],"oa":1,"OA_type":"green","oa_version":"Preprint","status":"public","publication_status":"published","date_updated":"2025-05-14T11:32:52Z","date_published":"2024-12-20T00:00:00Z","alternative_title":["Advances in Neural Information Processing Systems"],"_id":"19510","intvolume":"        37","OA_place":"repository","date_created":"2025-04-06T22:01:32Z","scopus_import":"1","language":[{"iso":"eng"}],"acknowledgement":"The authors thank Razvan Pascanu, Mahdi Nikdan and Soroush Tabesh for their valuable feedback, the IT department from Institute of Science and Technology Austria for the hardware support and Weights and Biases for the infrastructure to track all our experiments. Mher Safaryan has received funding from the European Union’s Horizon 2020 research and innovation program under the Marie Sklodowska-Curie grant agreement No 101034413.","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","publisher":"Neural Information Processing Systems Foundation","quality_controlled":"1","abstract":[{"text":"We propose a new variant of the Adam optimizer [Kingma and Ba, 2014] called\r\nMICROADAM that specifically minimizes memory overheads, while maintaining\r\ntheoretical convergence guarantees. We achieve this by compressing the gradient\r\ninformation before it is fed into the optimizer state, thereby reducing its memory\r\nfootprint significantly. We control the resulting compression error via a novel\r\ninstance of the classical error feedback mechanism from distributed optimization [Seide et al., 2014, Alistarh et al., 2018, Karimireddy et al., 2019] in which\r\nthe error correction information is itself compressed to allow for practical memory\r\ngains. We prove that the resulting approach maintains theoretical convergence\r\nguarantees competitive to those of AMSGrad, while providing good practical performance. Specifically, we show that MICROADAM can be implemented efficiently\r\non GPUs: on both million-scale (BERT) and billion-scale (LLaMA) models, MICROADAM provides practical convergence competitive to that of the uncompressed\r\nAdam baseline, with lower memory usage and similar running time. Our code is\r\navailable at https://github.com/IST-DASLab/MicroAdam.","lang":"eng"}],"department":[{"_id":"DaAl"}],"author":[{"full_name":"Modoranu, Ionut-Vlad","first_name":"Ionut-Vlad","id":"449f7a18-f128-11eb-9611-9b430c0c6333","last_name":"Modoranu"},{"first_name":"Mher","id":"dd546b39-0804-11ed-9c55-ef075c39778d","last_name":"Safaryan","full_name":"Safaryan, Mher"},{"full_name":"Malinovsky, Grigory","first_name":"Grigory","last_name":"Malinovsky"},{"full_name":"Kurtic, Eldar","last_name":"Kurtic","first_name":"Eldar","id":"47beb3a5-07b5-11eb-9b87-b108ec578218"},{"full_name":"Robert, Thomas","last_name":"Robert","id":"de632733-1457-11f0-ae22-b5914b8c1c41","first_name":"Thomas"},{"full_name":"Richtárik, Peter","last_name":"Richtárik","first_name":"Peter"},{"first_name":"Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"related_material":{"link":[{"url":"https://github.com/IST-DASLab/MicroAdam","relation":"software"}]},"title":"MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2405.15593"}],"corr_author":"1","type":"conference","arxiv":1,"publication_identifier":{"issn":["1049-5258"]}},{"department":[{"_id":"DaAl"},{"_id":"MaMo"}],"author":[{"full_name":"Wu, Diyuan","id":"1a5914c2-896a-11ed-bdf8-fb80621a0635","first_name":"Diyuan","last_name":"Wu"},{"last_name":"Modoranu","first_name":"Ionut-Vlad","id":"449f7a18-f128-11eb-9611-9b430c0c6333","full_name":"Modoranu, Ionut-Vlad"},{"full_name":"Safaryan, Mher","id":"dd546b39-0804-11ed-9c55-ef075c39778d","first_name":"Mher","last_name":"Safaryan"},{"full_name":"Kuznedelev, Denis","first_name":"Denis","last_name":"Kuznedelev"},{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh"}],"abstract":[{"text":"The rising footprint of machine learning has led to a focus on imposing model\r\nsparsity as a means of reducing computational and memory costs. For deep neural\r\nnetworks (DNNs), the state-of-the-art accuracy-vs-sparsity is achieved by heuristics\r\ninspired by the classical Optimal Brain Surgeon (OBS) framework [LeCun et al.,\r\n1989, Hassibi and Stork, 1992, Hassibi et al., 1993], which leverages loss curvature\r\ninformation to make better pruning decisions. Yet, these results still lack a solid\r\ntheoretical understanding, and it is unclear whether they can be improved by\r\nleveraging connections to the wealth of work on sparse recovery algorithms. In this\r\npaper, we draw new connections between these two areas and present new sparse\r\nrecovery algorithms inspired by the OBS framework that comes with theoretical\r\nguarantees under reasonable assumptions and have strong practical performance.\r\nSpecifically, our work starts from the observation that we can leverage curvature\r\ninformation in OBS-like fashion upon the projection step of classic iterative sparse\r\nrecovery algorithms such as IHT. We show for the first time that this leads both\r\nto improved convergence bounds under standard assumptions. Furthermore, we\r\npresent extensions of this approach to the practical task of obtaining accurate sparse\r\nDNNs, and validate it experimentally at scale for Transformer-based models on\r\nvision and language tasks.","lang":"eng"}],"quality_controlled":"1","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2408.17163"}],"publication_identifier":{"issn":["1049-5258"]},"arxiv":1,"corr_author":"1","type":"conference","title":"The iterative optimal brain surgeon: Faster sparse recovery by leveraging second-order information","scopus_import":"1","date_created":"2025-04-06T22:01:32Z","OA_place":"repository","acknowledgement":"The authors thank the anonymous NeurIPS reviewers for their useful comments and feedback, the IT department from the Institute of Science and Technology Austria for the hardware support, and Weights and Biases for the infrastructure to track all our experiments. Mher Safaryan has received funding from the European Union’s Horizon 2020 research and innovation program under the Maria Skłodowska-Curie grant agreement No 101034413.","language":[{"iso":"eng"}],"date_published":"2024-12-20T00:00:00Z","intvolume":"        37","_id":"19518","alternative_title":["Advances in Neural Information Processing Systems"],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","publisher":"Neural Information Processing Systems Foundation","conference":{"end_date":"2024-12-15","start_date":"2024-12-09","name":"NeurIPS: Neural Information Processing Systems","location":"Vancouver, Canada"},"oa":1,"oa_version":"Preprint","OA_type":"green","article_processing_charge":"No","day":"20","volume":37,"project":[{"grant_number":"101034413","name":"IST-BRIDGE: International postdoctoral program","call_identifier":"H2020","_id":"fc2ed2f7-9c52-11eb-aca3-c01059dda49c"}],"date_updated":"2025-05-14T11:37:10Z","status":"public","publication_status":"published","acknowledged_ssus":[{"_id":"CampIT"}],"month":"12","publication":"38th Conference on Neural Information Processing Systems","ec_funded":1,"external_id":{"arxiv":["2408.17163"]},"year":"2024","citation":{"ieee":"D. Wu, I.-V. Modoranu, M. Safaryan, D. Kuznedelev, and D.-A. Alistarh, “The iterative optimal brain surgeon: Faster sparse recovery by leveraging second-order information,” in <i>38th Conference on Neural Information Processing Systems</i>, Vancouver, Canada, 2024, vol. 37.","short":"D. Wu, I.-V. Modoranu, M. Safaryan, D. Kuznedelev, D.-A. Alistarh, in:, 38th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2024.","mla":"Wu, Diyuan, et al. “The Iterative Optimal Brain Surgeon: Faster Sparse Recovery by Leveraging Second-Order Information.” <i>38th Conference on Neural Information Processing Systems</i>, vol. 37, Neural Information Processing Systems Foundation, 2024.","ista":"Wu D, Modoranu I-V, Safaryan M, Kuznedelev D, Alistarh D-A. 2024. The iterative optimal brain surgeon: Faster sparse recovery by leveraging second-order information. 38th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 37.","chicago":"Wu, Diyuan, Ionut-Vlad Modoranu, Mher Safaryan, Denis Kuznedelev, and Dan-Adrian Alistarh. “The Iterative Optimal Brain Surgeon: Faster Sparse Recovery by Leveraging Second-Order Information.” In <i>38th Conference on Neural Information Processing Systems</i>, Vol. 37. Neural Information Processing Systems Foundation, 2024.","ama":"Wu D, Modoranu I-V, Safaryan M, Kuznedelev D, Alistarh D-A. The iterative optimal brain surgeon: Faster sparse recovery by leveraging second-order information. In: <i>38th Conference on Neural Information Processing Systems</i>. Vol 37. Neural Information Processing Systems Foundation; 2024.","apa":"Wu, D., Modoranu, I.-V., Safaryan, M., Kuznedelev, D., &#38; Alistarh, D.-A. (2024). The iterative optimal brain surgeon: Faster sparse recovery by leveraging second-order information. In <i>38th Conference on Neural Information Processing Systems</i> (Vol. 37). Vancouver, Canada: Neural Information Processing Systems Foundation."}},{"month":"10","publication":"Journal of Machine Learning Research","has_accepted_license":"1","year":"2023","citation":{"ama":"Beznosikov A, Horvath S, Richtarik P, Safaryan M. On biased compression for distributed learning. <i>Journal of Machine Learning Research</i>. 2023;24:1-50.","apa":"Beznosikov, A., Horvath, S., Richtarik, P., &#38; Safaryan, M. (2023). On biased compression for distributed learning. <i>Journal of Machine Learning Research</i>. Journal of Machine Learning Research.","short":"A. Beznosikov, S. Horvath, P. Richtarik, M. Safaryan, Journal of Machine Learning Research 24 (2023) 1–50.","mla":"Beznosikov, Aleksandr, et al. “On Biased Compression for Distributed Learning.” <i>Journal of Machine Learning Research</i>, vol. 24, Journal of Machine Learning Research, 2023, pp. 1–50.","ista":"Beznosikov A, Horvath S, Richtarik P, Safaryan M. 2023. On biased compression for distributed learning. Journal of Machine Learning Research. 24, 1–50.","chicago":"Beznosikov, Aleksandr, Samuel Horvath, Peter Richtarik, and Mher Safaryan. “On Biased Compression for Distributed Learning.” <i>Journal of Machine Learning Research</i>. Journal of Machine Learning Research, 2023.","ieee":"A. Beznosikov, S. Horvath, P. Richtarik, and M. Safaryan, “On biased compression for distributed learning,” <i>Journal of Machine Learning Research</i>, vol. 24. Journal of Machine Learning Research, pp. 1–50, 2023."},"external_id":{"arxiv":["2002.12410"],"isi":["001111578500001"]},"volume":24,"day":"01","article_processing_charge":"Yes (in subscription journal)","file_date_updated":"2024-01-16T12:13:27Z","file":[{"relation":"main_file","access_level":"open_access","checksum":"c50f2b9db53938b755e30a085f464059","file_name":"2023_JMLR_Beznosikov.pdf","file_id":"14816","success":1,"date_created":"2024-01-16T12:13:27Z","content_type":"application/pdf","file_size":1510993,"date_updated":"2024-01-16T12:13:27Z","creator":"dernst"}],"oa":1,"oa_version":"Published Version","ddc":["000"],"publication_status":"published","status":"public","date_updated":"2024-10-09T21:07:52Z","date_published":"2023-10-01T00:00:00Z","_id":"14815","intvolume":"        24","page":"1-50","date_created":"2024-01-16T12:13:36Z","language":[{"iso":"eng"}],"acknowledgement":"The work in Sections 1-5 was conducted while A. Beznosikov was a research intern in the Optimizationand Machine Learning Lab of Peter Richtárik at KAUST; this visit was funded by the KAUST Baseline Research Funding Scheme. The work of A. Beznosikov in Section 6 was conducted in Skoltech and was supported by Ministry of Science and Higher Education grant No. 075-10-2021-068. ","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","isi":1,"publisher":"Journal of Machine Learning Research","article_type":"original","quality_controlled":"1","abstract":[{"text":"In the last few years, various communication compression techniques have emerged as an indispensable tool helping to alleviate the communication bottleneck in distributed learning. However, despite the fact biased compressors often show superior performance in practice when compared to the much more studied and understood unbiased compressors, very little is known about them. In this work we study three classes of biased compression operators, two of which are new, and their performance when applied to (stochastic) gradient descent and distributed (stochastic) gradient descent. We show for the first time that biased compressors can lead to linear convergence rates both in the single node and distributed settings. We prove that distributed compressed SGD method, employed with error feedback mechanism, enjoys the ergodic rate O(δLexp[−μKδL]+(C+δD)Kμ), where δ≥1 is a compression parameter which grows when more compression is applied, L and μ are the smoothness and strong convexity constants, C captures stochastic gradient noise (C=0 if full gradients are computed on each node) and D captures the variance of the gradients at the optimum (D=0 for over-parameterized models). Further, via a theoretical study of several synthetic and empirical distributions of communicated gradients, we shed light on why and by how much biased compressors outperform their unbiased variants. Finally, we propose several new biased compressors with promising theoretical guarantees and practical performance.","lang":"eng"}],"tmp":{"short":"CC BY (4.0)","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode"},"department":[{"_id":"DaAl"}],"author":[{"first_name":"Aleksandr","last_name":"Beznosikov","full_name":"Beznosikov, Aleksandr"},{"full_name":"Horvath, Samuel","first_name":"Samuel","last_name":"Horvath"},{"last_name":"Richtarik","first_name":"Peter","full_name":"Richtarik, Peter"},{"full_name":"Safaryan, Mher","first_name":"Mher","id":"dd546b39-0804-11ed-9c55-ef075c39778d","last_name":"Safaryan"}],"title":"On biased compression for distributed learning","corr_author":"1","type":"journal_article","publication_identifier":{"eissn":["1533-7928"]},"arxiv":1},{"language":[{"iso":"eng"}],"acknowledgement":"MS has received funding from the European Union’s Horizon 2020 research and innovation programme\r\nunder the Marie Skłodowska-Curie grant agreement No 101034413.","date_created":"2024-05-05T22:01:04Z","scopus_import":"1","alternative_title":["NeurIPS"],"_id":"15363","intvolume":"        36","date_published":"2023-12-15T00:00:00Z","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","conference":{"start_date":"2023-12-10","name":"NeurIPS: Neural Information Processing Systems","location":"New Orleans, LA, United States","end_date":"2023-12-16"},"author":[{"last_name":"Safaryan","first_name":"Mher","id":"dd546b39-0804-11ed-9c55-ef075c39778d","full_name":"Safaryan, Mher"},{"last_name":"Peste","first_name":"Elena-Alexandra","id":"32D78294-F248-11E8-B48F-1D18A9856A87","full_name":"Peste, Elena-Alexandra"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","first_name":"Dan-Adrian"}],"department":[{"_id":"DaAl"}],"tmp":{"short":"CC BY (4.0)","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode"},"quality_controlled":"1","abstract":[{"lang":"eng","text":"Knowledge distillation is a popular approach for enhancing the performance of \"student\" models, with lower representational capacity, by taking advantage of more powerful \"teacher\" models. Despite its apparent simplicity, the underlying mechanics behind knowledge distillation (KD) are not yet fully understood. In this work, we shed new light on the inner workings of this method, by examining it from an optimization perspective. Specifically, we show that, in the context of linear and deep linear models, KD can be interpreted as a novel type of stochastic variance reduction mechanism. We provide a detailed convergence analysis of the resulting dynamics, which hold under standard assumptions for both strongly-convex and non-convex losses, showing that KD acts as a form of \\emph{partial variance reduction}, which can reduce the stochastic gradient noise, but may not eliminate it completely, depending on the properties of the teacher'' model. Our analysis puts further emphasis on the need for careful parametrization of KD, in particular w.r.t. the weighting of the distillation loss, and is validated empirically on both linear models and deep neural networks."}],"type":"conference","corr_author":"1","publication_identifier":{"issn":["1049-5258"]},"arxiv":1,"title":"Knowledge distillation performs partial variance reduction","month":"12","publication":"36th Conference on Neural Information Processing Systems","ec_funded":1,"citation":{"ieee":"M. Safaryan, A. Krumes, and D.-A. Alistarh, “Knowledge distillation performs partial variance reduction,” in <i>36th Conference on Neural Information Processing Systems</i>, New Orleans, LA, United States, 2023, vol. 36.","ama":"Safaryan M, Krumes A, Alistarh D-A. Knowledge distillation performs partial variance reduction. In: <i>36th Conference on Neural Information Processing Systems</i>. Vol 36. ; 2023.","apa":"Safaryan, M., Krumes, A., &#38; Alistarh, D.-A. (2023). Knowledge distillation performs partial variance reduction. In <i>36th Conference on Neural Information Processing Systems</i> (Vol. 36). New Orleans, LA, United States.","chicago":"Safaryan, Mher, Alexandra Krumes, and Dan-Adrian Alistarh. “Knowledge Distillation Performs Partial Variance Reduction.” In <i>36th Conference on Neural Information Processing Systems</i>, Vol. 36, 2023.","short":"M. Safaryan, A. Krumes, D.-A. Alistarh, in:, 36th Conference on Neural Information Processing Systems, 2023.","ista":"Safaryan M, Krumes A, Alistarh D-A. 2023. Knowledge distillation performs partial variance reduction. 36th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, NeurIPS, vol. 36.","mla":"Safaryan, Mher, et al. “Knowledge Distillation Performs Partial Variance Reduction.” <i>36th Conference on Neural Information Processing Systems</i>, vol. 36, 2023."},"year":"2023","external_id":{"arxiv":["2305.17581"]},"has_accepted_license":"1","oa_version":"Published Version","oa":1,"file_date_updated":"2024-05-22T08:08:08Z","project":[{"name":"IST-BRIDGE: International postdoctoral program","_id":"fc2ed2f7-9c52-11eb-aca3-c01059dda49c","call_identifier":"H2020","grant_number":"101034413"}],"file":[{"checksum":"288c5148a85abf24ad5e22a6b1183655","access_level":"open_access","relation":"main_file","date_created":"2024-05-22T08:08:08Z","file_name":"2023_Neurips_Safaryan.pdf","file_id":"15417","success":1,"content_type":"application/pdf","creator":"dernst","file_size":672571,"date_updated":"2024-05-22T08:08:08Z"}],"volume":36,"day":"15","article_processing_charge":"Yes","date_updated":"2025-04-14T07:54:55Z","status":"public","publication_status":"published","ddc":["000"]}]