[{"quality_controlled":"1","citation":{"chicago":"Nicolicioiu, Armand, Eugenia B Iofinova, Andrej Jovanovic, Eldar Kurtic, Mahdi Nikdan, Andrei Panferov, Ilia Markov, Nir Shavit, and Dan-Adrian Alistarh. <i>Panza: Investigating the Feasibility of Fully-Local Personalized Text Generation</i>. <i>Third Conference on Parsimony and Learning (Proceedings Track)</i>. OpenReview, 2026.","short":"A. Nicolicioiu, E.B. Iofinova, A. Jovanovic, E. Kurtic, M. Nikdan, A. Panferov, I. Markov, N. Shavit, D.-A. Alistarh, Panza: Investigating the Feasibility of Fully-Local Personalized Text Generation, OpenReview, 2026.","mla":"Nicolicioiu, Armand, et al. “Panza: Investigating the Feasibility of Fully-Local Personalized Text Generation.” <i>Third Conference on Parsimony and Learning (Proceedings Track)</i>, 81, OpenReview, 2026.","ama":"Nicolicioiu A, Iofinova EB, Jovanovic A, et al. <i>Panza: Investigating the Feasibility of Fully-Local Personalized Text Generation</i>. OpenReview; 2026.","apa":"Nicolicioiu, A., Iofinova, E. B., Jovanovic, A., Kurtic, E., Nikdan, M., Panferov, A., … Alistarh, D.-A. (2026). <i>Panza: Investigating the feasibility of fully-local personalized text generation</i>. <i>Third Conference on Parsimony and Learning (Proceedings Track)</i>. Tübíngen, Germany: OpenReview.","ista":"Nicolicioiu A, Iofinova EB, Jovanovic A, Kurtic E, Nikdan M, Panferov A, Markov I, Shavit N, Alistarh D-A. 2026. Panza: Investigating the feasibility of fully-local personalized text generation, OpenReview,p.","ieee":"A. Nicolicioiu <i>et al.</i>, <i>Panza: Investigating the feasibility of fully-local personalized text generation</i>. OpenReview, 2026."},"license":"https://creativecommons.org/licenses/by/4.0/","publication":"Third Conference on Parsimony and Learning (Proceedings Track)","title":"Panza: Investigating the feasibility of fully-local personalized text generation","main_file_link":[{"open_access":"1","url":"https://openreview.net/pdf?id=soFWnTqd23"}],"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)"},"type":"conference_poster","date_created":"2026-05-11T08:50:28Z","author":[{"first_name":"Armand","last_name":"Nicolicioiu","full_name":"Nicolicioiu, Armand"},{"id":"f9a17499-f6e0-11ea-865d-fdf9a3f77117","last_name":"Iofinova","first_name":"Eugenia B","orcid":"0000-0002-7778-3221","full_name":"Iofinova, Eugenia B"},{"last_name":"Jovanovic","first_name":"Andrej","full_name":"Jovanovic, Andrej"},{"first_name":"Eldar","last_name":"Kurtic","id":"47beb3a5-07b5-11eb-9b87-b108ec578218","full_name":"Kurtic, Eldar"},{"first_name":"Mahdi","last_name":"Nikdan","id":"66374281-f394-11eb-9cf6-869147deecc0","full_name":"Nikdan, Mahdi"},{"full_name":"Panferov, Andrei","last_name":"Panferov","id":"2c18daae-4dbe-11ef-8491-98ce2d960f09","first_name":"Andrei"},{"first_name":"Ilia","last_name":"Markov","id":"D0CF4148-C985-11E9-8066-0BDEE5697425","full_name":"Markov, Ilia"},{"first_name":"Nir","last_name":"Shavit","full_name":"Shavit, Nir"},{"last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian"}],"month":"03","article_number":"81","date_published":"2026-03-06T00:00:00Z","publication_status":"published","OA_type":"green","day":"06","OA_place":"publisher","oa":1,"user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","publisher":"OpenReview","department":[{"_id":"GradSch"},{"_id":"DaAl"}],"article_processing_charge":"No","status":"public","related_material":{"record":[{"status":"public","id":"21854","relation":"dissertation_contains"}]},"date_updated":"2026-07-27T12:50:03Z","_id":"21857","corr_author":"1","abstract":[{"text":"The availability of powerful open-source large language models (LLMs) opens exciting use cases, such as using personal data to fine-tune these models to imitate a user’s unique writing style. Two key requirements for this functionality are personalization–in the sense that the output should recognizably reflect the user’s own writing style—and privacy–users may justifiably be wary of uploading extremely personal data, such as their email archive, to a third-party service. In this paper, we demonstrate the feasibility of training and running such an assistant, which we call Panza, on commodity hardware, for the specific use case of email generation. Panza’s personalization features are based on a combination of parameter-efficient fine-tuning using a variant of the Reverse Instructions technique [1] and Retrieval-Augmented Generation (RAG) [2]. We demonstrate that this combination allows us to fine-tune an LLM to reflect a user’s writing style using limited data, while executing on extremely limited resources, e.g. on a free Google Colab instance. Our key methodological contribution is the first detailed study of evaluation metrics for this task, and\r\nof how different choices of system components–the use of RAG and of different fine-tuning approaches–impact the system’s performance. Additionally, we demonstrate that very little data - under 100 email samples - are sufficient to create models that convincingly imitate humans, showcasing a previously unknown attack vector in language models. We are releasing the full Panza code as well as three new email datasets licensed for research use.","lang":"eng"}],"year":"2026","conference":{"end_date":"2026-03-26","start_date":"2026-03-23","location":"Tübíngen, Germany","name":"CPAL: Conference on Parsimony and Learning"},"keyword":["LLMs","PEFT","LoRA","personalization","efficient ML"],"language":[{"iso":"eng"}],"oa_version":"Accepted Version"},{"alternative_title":["PMLR"],"publisher":"ML Research Press","has_accepted_license":"1","oa":1,"ddc":["000"],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","external_id":{"arxiv":["2410.14649"]},"conference":{"end_date":"2025-07-19","start_date":"2025-07-13","name":"ICML: International Conference on Machine Learning","location":"Vancouver, Canada"},"year":"2025","volume":267,"oa_version":"Published Version","language":[{"iso":"eng"}],"abstract":[{"text":"The high computational costs of large language models (LLMs) have led to a flurry of research on LLM compression, via methods such as quantization, sparsification, or structured pruning. A new frontier in this area is given by dynamic, non-uniform compression methods, which adjust the compression levels (e.g., sparsity) per-block or even per-layer in order to minimize accuracy loss, while guaranteeing a global compression threshold. Yet, current methods rely on estimating the \"importance\" of a given layer, implicitly assuming that layers contribute independently to the overall compression error. We begin from the motivating observation that this independence assumption does not generally hold for LLM compression: pruning a model further may even significantly recover performance. To address this, we propose EvoPress, a novel evolutionary framework for dynamic LLM compression. By formulating dynamic compression as a general optimization problem, EvoPress identifies optimal compression profiles in a highly efficient manner, and generalizes across diverse models and compression techniques. Via EvoPress, we achieve state-of-the-art performance for dynamic compression of Llama, Mistral, and Phi models, setting new benchmarks for structural pruning (block/layer dropping), unstructured sparsity, and quantization with dynamic bitwidths.","lang":"eng"}],"corr_author":"1","file":[{"relation":"main_file","access_level":"open_access","checksum":"1d744fbaeb199b08e8b6f48bc0dd047e","file_id":"20828","content_type":"application/pdf","file_name":"2025_ICML_Sieberling.pdf","file_size":908379,"success":1,"date_created":"2025-12-16T12:32:40Z","date_updated":"2025-12-16T12:32:40Z","creator":"dernst"}],"date_updated":"2025-12-16T12:34:32Z","status":"public","article_processing_charge":"No","_id":"20820","page":"55556-55590","intvolume":"       267","department":[{"_id":"DaAl"}],"publication":"42nd International Conference on Machine Learning","title":"EvoPress: Accurate dynamic model compression via evolutionary search","arxiv":1,"publication_identifier":{"eissn":["2640-3498"]},"citation":{"mla":"Sieberling, Oliver, et al. “EvoPress: Accurate Dynamic Model Compression via Evolutionary Search.” <i>42nd International Conference on Machine Learning</i>, vol. 267, ML Research Press, 2025, pp. 55556–90.","apa":"Sieberling, O., Kuznedelev, D., Kurtic, E., &#38; Alistarh, D.-A. (2025). EvoPress: Accurate dynamic model compression via evolutionary search. In <i>42nd International Conference on Machine Learning</i> (Vol. 267, pp. 55556–55590). Vancouver, Canada: ML Research Press.","ista":"Sieberling O, Kuznedelev D, Kurtic E, Alistarh D-A. 2025. EvoPress: Accurate dynamic model compression via evolutionary search. 42nd International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 267, 55556–55590.","ama":"Sieberling O, Kuznedelev D, Kurtic E, Alistarh D-A. EvoPress: Accurate dynamic model compression via evolutionary search. In: <i>42nd International Conference on Machine Learning</i>. Vol 267. ML Research Press; 2025:55556-55590.","ieee":"O. Sieberling, D. Kuznedelev, E. Kurtic, and D.-A. Alistarh, “EvoPress: Accurate dynamic model compression via evolutionary search,” in <i>42nd International Conference on Machine Learning</i>, Vancouver, Canada, 2025, vol. 267, pp. 55556–55590.","chicago":"Sieberling, Oliver, Denis Kuznedelev, Eldar Kurtic, and Dan-Adrian Alistarh. “EvoPress: Accurate Dynamic Model Compression via Evolutionary Search.” In <i>42nd International Conference on Machine Learning</i>, 267:55556–90. ML Research Press, 2025.","short":"O. Sieberling, D. Kuznedelev, E. Kurtic, D.-A. Alistarh, in:, 42nd International Conference on Machine Learning, ML Research Press, 2025, pp. 55556–55590."},"quality_controlled":"1","OA_place":"publisher","scopus_import":"1","publication_status":"published","date_published":"2025-05-01T00:00:00Z","file_date_updated":"2025-12-16T12:32:40Z","day":"01","OA_type":"gold","date_created":"2025-12-14T23:02:05Z","month":"05","author":[{"first_name":"Oliver","last_name":"Sieberling","full_name":"Sieberling, Oliver"},{"full_name":"Kuznedelev, Denis","last_name":"Kuznedelev","first_name":"Denis"},{"first_name":"Eldar","id":"47beb3a5-07b5-11eb-9b87-b108ec578218","last_name":"Kurtic","full_name":"Kurtic, Eldar"},{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh"}],"type":"conference","tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)"}},{"main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2310.06927"}],"title":"Sparse Fine-Tuning for Inference Acceleration of Large Language Models","publication":"Enhancing LLM Performance. Efficacy, Fine-Tuning, and Inference Techniques","arxiv":1,"publication_identifier":{"isbn":["9783031857461"],"eissn":["2522-803X"],"eisbn":["9783031857478"],"issn":["2522-8021"]},"citation":{"mla":"Kurtic, Eldar, et al. “Sparse Fine-Tuning for Inference Acceleration of Large Language Models.” <i>Enhancing LLM Performance. Efficacy, Fine-Tuning, and Inference Techniques</i>, edited by Peyman Passban et al., Springer Nature, 2025, pp. 83–97, doi:<a href=\"https://doi.org/10.1007/978-3-031-85747-8_6\">10.1007/978-3-031-85747-8_6</a>.","ama":"Kurtic E, Kuznedelev D, Frantar E, et al. Sparse Fine-Tuning for Inference Acceleration of Large Language Models. In: Passban P, Way A, Rezagholizadeh M, eds. <i>Enhancing LLM Performance. Efficacy, Fine-Tuning, and Inference Techniques</i>. Springer Nature; 2025:83-97. doi:<a href=\"https://doi.org/10.1007/978-3-031-85747-8_6\">10.1007/978-3-031-85747-8_6</a>","ista":"Kurtic E, Kuznedelev D, Frantar E, Goinv M, Pandit S, Agarwalla A, Nguyen T, Marques A, Kurtz M, Alistarh D-A. 2025.Sparse Fine-Tuning for Inference Acceleration of Large Language Models. In: Enhancing LLM Performance. Efficacy, Fine-Tuning, and Inference Techniques. Machine Translation: Technologies and Applications, , 83–97.","apa":"Kurtic, E., Kuznedelev, D., Frantar, E., Goinv, M., Pandit, S., Agarwalla, A., … Alistarh, D.-A. (2025). Sparse Fine-Tuning for Inference Acceleration of Large Language Models. In P. Passban, A. Way, &#38; M. Rezagholizadeh (Eds.), <i>Enhancing LLM Performance. Efficacy, Fine-Tuning, and Inference Techniques</i> (pp. 83–97). Springer Nature. <a href=\"https://doi.org/10.1007/978-3-031-85747-8_6\">https://doi.org/10.1007/978-3-031-85747-8_6</a>","ieee":"E. Kurtic <i>et al.</i>, “Sparse Fine-Tuning for Inference Acceleration of Large Language Models,” in <i>Enhancing LLM Performance. Efficacy, Fine-Tuning, and Inference Techniques</i>, P. Passban, A. Way, and M. Rezagholizadeh, Eds. Springer Nature, 2025, pp. 83–97.","chicago":"Kurtic, Eldar, Denis Kuznedelev, Elias Frantar, Michael Goinv, Shubhra Pandit, Abhinav Agarwalla, Tuan Nguyen, Alexandre Marques, Mark Kurtz, and Dan-Adrian Alistarh. “Sparse Fine-Tuning for Inference Acceleration of Large Language Models.” In <i>Enhancing LLM Performance. Efficacy, Fine-Tuning, and Inference Techniques</i>, edited by Peyman Passban, Andy Way, and Mehdi Rezagholizadeh, 83–97. Springer Nature, 2025. <a href=\"https://doi.org/10.1007/978-3-031-85747-8_6\">https://doi.org/10.1007/978-3-031-85747-8_6</a>.","short":"E. Kurtic, D. Kuznedelev, E. Frantar, M. Goinv, S. Pandit, A. Agarwalla, T. Nguyen, A. Marques, M. Kurtz, D.-A. Alistarh, in:, P. Passban, A. Way, M. Rezagholizadeh (Eds.), Enhancing LLM Performance. Efficacy, Fine-Tuning, and Inference Techniques, Springer Nature, 2025, pp. 83–97."},"doi":"10.1007/978-3-031-85747-8_6","quality_controlled":"1","OA_place":"repository","fulldoi":"https://doi.org/10.1007/978-3-031-85747-8_6","day":"05","OA_type":"green","date_published":"2025-07-05T00:00:00Z","publication_status":"published","month":"07","author":[{"id":"47beb3a5-07b5-11eb-9b87-b108ec578218","last_name":"Kurtic","first_name":"Eldar","full_name":"Kurtic, Eldar"},{"first_name":"Denis","last_name":"Kuznedelev","full_name":"Kuznedelev, Denis"},{"full_name":"Frantar, Elias","id":"09a8f98d-ec99-11ea-ae11-c063a7b7fe5f","last_name":"Frantar","first_name":"Elias"},{"first_name":"Michael","last_name":"Goinv","full_name":"Goinv, Michael"},{"full_name":"Pandit, Shubhra","last_name":"Pandit","first_name":"Shubhra"},{"first_name":"Abhinav","last_name":"Agarwalla","full_name":"Agarwalla, Abhinav"},{"full_name":"Nguyen, Tuan","first_name":"Tuan","last_name":"Nguyen"},{"full_name":"Marques, Alexandre","last_name":"Marques","first_name":"Alexandre"},{"first_name":"Mark","last_name":"Kurtz","full_name":"Kurtz, Mark"},{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh"}],"date_created":"2026-02-16T15:57:53Z","type":"book_chapter","publisher":"Springer Nature","alternative_title":["Machine Translation: Technologies and Applications"],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","acknowledgement":"We would like to thank Eugenia Iofinova for useful comments on an earlier version of this draft, and Artur Niederfahrenhorst for useful suggestions regarding fine-tuning on the GSM8k dataset.","oa":1,"editor":[{"last_name":"Passban","first_name":"Peyman","full_name":"Passban, Peyman"},{"first_name":"Andy","last_name":"Way","full_name":"Way, Andy"},{"last_name":"Rezagholizadeh","first_name":"Mehdi","full_name":"Rezagholizadeh, Mehdi"}],"oa_version":"Preprint","language":[{"iso":"eng"}],"external_id":{"arxiv":["2310.06927"]},"year":"2025","abstract":[{"text":"We investigate the problem of accurate sparse fine-tuning of large language models (LLMs), that is, fine-tuning pre-trained LLMs on specialized tasks, while inducing sparsity in their weights. Our work is motivated by experiments showing that standard loss-based fine-tuning methods are not able to achieve high accuracy in this setting, especially at high sparsity targets. To address this issue, we perform a detailed study of knowledge distillation losses for fine-tuning of sparse models. We determine an L2-based distillation approach that we term ‘SquareHead’, which enables accurate recovery even at higher sparsities. Investigating the question of efficient inference, we show that sparse LLMs can be executed faster by taking advantage of sparsity. Specifically, we exhibit end-to-end results showing speedups enabled by sparsity, while recovering accuracy, on the following models and tasks, respectively: T5 for language translation, Whisper for speech translation, and open GPT-type models such as the Mosaic Pre-Trained Transformer (MPT) and Llama-2 models for text generation. In particular, for popular generative tasks, we show for the first time that sparse fine-tuning can reach 75% sparsity without drops in accuracy, and provide notable end-to-end speedups for inference on CPUs. Moreover, we also highlight that sparsity is compatible with other compression approaches, such as quantization.","lang":"eng"}],"corr_author":"1","_id":"21257","page":"83-97","status":"public","date_updated":"2026-02-19T09:26:54Z","article_processing_charge":"No","department":[{"_id":"DaAl"},{"_id":"GradSch"}]},{"publication":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics","title":"“Give me BF16 or give me death”? Accuracy-performance trade-offs in LLM quantization","arxiv":1,"publication_identifier":{"isbn":["9798891762510"],"issn":["0736-587X"]},"citation":{"short":"E. Kurtic, A. Marques, S. Pandit, M. Kurtz, D.-A. Alistarh, in:, Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics, Association for Computational Linguistics, 2025, pp. 26872–26886.","chicago":"Kurtic, Eldar, Alexandre Marques, Shubhra Pandit, Mark Kurtz, and Dan-Adrian Alistarh. “‘Give Me BF16 or Give Me Death’? Accuracy-Performance Trade-Offs in LLM Quantization.” In <i>Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics</i>, 26872–86. Association for Computational Linguistics, 2025.","ieee":"E. Kurtic, A. Marques, S. Pandit, M. Kurtz, and D.-A. Alistarh, “‘Give me BF16 or give me death’? Accuracy-performance trade-offs in LLM quantization,” in <i>Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics</i>, Vienna, Austria, 2025, pp. 26872–26886.","ama":"Kurtic E, Marques A, Pandit S, Kurtz M, Alistarh D-A. “Give me BF16 or give me death”? Accuracy-performance trade-offs in LLM quantization. In: <i>Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics</i>. Association for Computational Linguistics; 2025:26872-26886.","apa":"Kurtic, E., Marques, A., Pandit, S., Kurtz, M., &#38; Alistarh, D.-A. (2025). “Give me BF16 or give me death”? Accuracy-performance trade-offs in LLM quantization. In <i>Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics</i> (pp. 26872–26886). Vienna, Austria: Association for Computational Linguistics.","ista":"Kurtic E, Marques A, Pandit S, Kurtz M, Alistarh D-A. 2025. “Give me BF16 or give me death”? Accuracy-performance trade-offs in LLM quantization. Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics. ACL: Meeting of the Association for Computational Linguistics, 26872–26886.","mla":"Kurtic, Eldar, et al. “‘Give Me BF16 or Give Me Death’? Accuracy-Performance Trade-Offs in LLM Quantization.” <i>Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics</i>, Association for Computational Linguistics, 2025, pp. 26872–86."},"quality_controlled":"1","OA_place":"publisher","scopus_import":"1","date_published":"2025-08-01T00:00:00Z","publication_status":"published","file_date_updated":"2025-11-26T11:06:57Z","day":"01","OA_type":"gold","date_created":"2025-11-24T14:20:46Z","month":"08","author":[{"id":"47beb3a5-07b5-11eb-9b87-b108ec578218","last_name":"Kurtic","first_name":"Eldar","full_name":"Kurtic, Eldar"},{"full_name":"Marques, Alexandre","last_name":"Marques","first_name":"Alexandre"},{"last_name":"Pandit","first_name":"Shubhra","full_name":"Pandit, Shubhra"},{"full_name":"Kurtz, Mark","first_name":"Mark","last_name":"Kurtz"},{"first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"}],"type":"conference","tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)"},"has_accepted_license":"1","publisher":"Association for Computational Linguistics","ddc":["000"],"oa":1,"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","conference":{"location":"Vienna, Austria","name":"ACL: Meeting of the Association for Computational Linguistics","start_date":"2025-07-27","end_date":"2025-08-01"},"external_id":{"arxiv":["2411.02355"]},"year":"2025","oa_version":"Published Version","language":[{"iso":"eng"}],"corr_author":"1","abstract":[{"lang":"eng","text":"Quantization is a powerful tool for accelerating large language model (LLM) inference, but the accuracy-performance trade-offs across different formats remain unclear. In this paper, we conduct the most comprehensive empirical study to date, evaluating FP8, INT8, and INT4\r\nquantization across academic benchmarks and real-world tasks on the entire Llama-3.1 model\r\nfamily. Through over 500,000 evaluations, our investigation yields several key findings: (1) FP8 (W8A8-FP) is effectively lossless across all model scales, (2) well-tuned INT8 (W8A8-INT) achieves surprisingly low (1-3%) accuracy degradation, and (3) INT4 weightonly (W4A16-INT) is more competitive than expected, rivaling 8-bit quantization. Further, we investigate the optimal quantization format for different deployments by analyzing inference performance through the popular vLLM framework. Our analysis provides clear deployment recommendations: W4A16 is the most cost-efficient for synchronous setups, while W8A8 dominates in asynchronous\r\ncontinuous batching. For mixed workloads, the optimal choice depends on the specific use\r\ncase. Our findings offer practical, data-driven guidelines for deploying quantized LLMs at scale—ensuring the best balance between speed, efficiency, and accuracy. "}],"file":[{"access_level":"open_access","relation":"main_file","content_type":"application/pdf","file_id":"20698","checksum":"4c066ee20f9ab17619c95652c0eb75f1","file_name":"2025_ACL_Kurtic.pdf","success":1,"file_size":417450,"creator":"dernst","date_created":"2025-11-26T11:06:57Z","date_updated":"2025-11-26T11:06:57Z"}],"date_updated":"2025-11-26T11:15:11Z","status":"public","article_processing_charge":"No","_id":"20684","page":"26872-26886","department":[{"_id":"DaAl"}]},{"quality_controlled":"1","arxiv":1,"citation":{"chicago":"Kurtic, Eldar, Torsten Hoefler, and Dan-Adrian Alistarh. “How to Prune Your Language Model: Recovering Accuracy on the ‘Sparsity May Cry’ Benchmark.” In <i>Proceedings of Machine Learning Research</i>, 234:542–53. ML Research Press, 2024.","short":"E. Kurtic, T. Hoefler, D.-A. Alistarh, in:, Proceedings of Machine Learning Research, ML Research Press, 2024, pp. 542–553.","mla":"Kurtic, Eldar, et al. “How to Prune Your Language Model: Recovering Accuracy on the ‘Sparsity May Cry’ Benchmark.” <i>Proceedings of Machine Learning Research</i>, vol. 234, ML Research Press, 2024, pp. 542–53.","ista":"Kurtic E, Hoefler T, Alistarh D-A. 2024. How to prune your language model: Recovering accuracy on the ‘Sparsity May Cry’ benchmark. Proceedings of Machine Learning Research. CPAL: Conference on Parsimony and Learning, PMLR, vol. 234, 542–553.","apa":"Kurtic, E., Hoefler, T., &#38; Alistarh, D.-A. (2024). How to prune your language model: Recovering accuracy on the “Sparsity May Cry” benchmark. In <i>Proceedings of Machine Learning Research</i> (Vol. 234, pp. 542–553). Hongkong, China: ML Research Press.","ama":"Kurtic E, Hoefler T, Alistarh D-A. How to prune your language model: Recovering accuracy on the “Sparsity May Cry” benchmark. In: <i>Proceedings of Machine Learning Research</i>. Vol 234. ML Research Press; 2024:542-553.","ieee":"E. Kurtic, T. Hoefler, and D.-A. Alistarh, “How to prune your language model: Recovering accuracy on the ‘Sparsity May Cry’ benchmark,” in <i>Proceedings of Machine Learning Research</i>, Hongkong, China, 2024, vol. 234, pp. 542–553."},"publication_identifier":{"eissn":["2640-3498"]},"publication":"Proceedings of Machine Learning Research","main_file_link":[{"open_access":"1","url":"https://proceedings.mlr.press/v234/kurtic24a"}],"title":"How to prune your language model: Recovering accuracy on the \"Sparsity May Cry\" benchmark","type":"conference","date_created":"2024-02-18T23:01:03Z","month":"01","author":[{"last_name":"Kurtic","id":"47beb3a5-07b5-11eb-9b87-b108ec578218","first_name":"Eldar","full_name":"Kurtic, Eldar"},{"full_name":"Hoefler, Torsten","last_name":"Hoefler","first_name":"Torsten"},{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"}],"scopus_import":"1","date_published":"2024-01-08T00:00:00Z","publication_status":"published","day":"08","das_tickbox":"0","oa":1,"user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","alternative_title":["PMLR"],"publisher":"ML Research Press","supplementarymaterial":"no","intvolume":"       234","department":[{"_id":"DaAl"}],"date_updated":"2026-10-01T09:31:02Z","status":"public","article_processing_charge":"No","_id":"15011","page":"542-553","corr_author":"1","abstract":[{"text":"Pruning large language models (LLMs) from the BERT family has emerged as a standard compression benchmark, and several pruning methods have been proposed for this task. The recent “Sparsity May Cry” (SMC) benchmark put into question the validity of all existing methods, exhibiting a more complex setup where many known pruning methods appear to fail. We revisit the question of accurate BERT-pruning during fine-tuning on downstream datasets, and propose a set of general guidelines for successful pruning, even on the challenging SMC benchmark. First, we perform a cost-vs-benefits analysis of pruning model components, such as the embeddings and the classification head; second, we provide a simple-yet-general way of scaling training, sparsification and learning rate schedules relative to the desired target sparsity; finally, we investigate the importance of proper parametrization for Knowledge Distillation in the context of LLMs. Our simple insights lead to state-of-the-art results, both on classic BERT-pruning benchmarks, as well as on the SMC benchmark, showing that even classic gradual magnitude pruning (GMP) can yield competitive results, with the right approach.","lang":"eng"}],"conference":{"start_date":"2024-01-03","end_date":"2024-01-06","name":"CPAL: Conference on Parsimony and Learning","location":"Hongkong, China"},"external_id":{"arxiv":["2312.13547"]},"year":"2024","volume":234,"researchdata_availability":"no","oa_version":"Preprint","language":[{"iso":"eng"}]},{"citation":{"ieee":"I.-V. Modoranu, A. Kalinov, E. Kurtic, E. Frantar, and D.-A. Alistarh, “Error feedback can accurately compress preconditioners,” in <i>41st International Conference on Machine Learning</i>, Vienna, Austria, 2024, vol. 235, pp. 35910–35933.","apa":"Modoranu, I.-V., Kalinov, A., Kurtic, E., Frantar, E., &#38; Alistarh, D.-A. (2024). Error feedback can accurately compress preconditioners. In <i>41st International Conference on Machine Learning</i> (Vol. 235, pp. 35910–35933). Vienna, Austria: ML Research Press.","ama":"Modoranu I-V, Kalinov A, Kurtic E, Frantar E, Alistarh D-A. Error feedback can accurately compress preconditioners. In: <i>41st International Conference on Machine Learning</i>. Vol 235. ML Research Press; 2024:35910-35933.","ista":"Modoranu I-V, Kalinov A, Kurtic E, Frantar E, Alistarh D-A. 2024. Error feedback can accurately compress preconditioners. 41st International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 235, 35910–35933.","mla":"Modoranu, Ionut-Vlad, et al. “Error Feedback Can Accurately Compress Preconditioners.” <i>41st International Conference on Machine Learning</i>, vol. 235, ML Research Press, 2024, pp. 35910–33.","short":"I.-V. Modoranu, A. Kalinov, E. Kurtic, E. Frantar, D.-A. Alistarh, in:, 41st International Conference on Machine Learning, ML Research Press, 2024, pp. 35910–35933.","chicago":"Modoranu, Ionut-Vlad, Aleksei Kalinov, Eldar Kurtic, Elias Frantar, and Dan-Adrian Alistarh. “Error Feedback Can Accurately Compress Preconditioners.” In <i>41st International Conference on Machine Learning</i>, 235:35910–33. ML Research Press, 2024."},"publication_identifier":{"eissn":["2640-3498"]},"arxiv":1,"publication":"41st International Conference on Machine Learning","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2306.06098"}],"title":"Error feedback can accurately compress preconditioners","quality_controlled":"1","publication_status":"published","date_published":"2024-07-30T00:00:00Z","scopus_import":"1","day":"30","OA_type":"green","OA_place":"repository","das_tickbox":"0","type":"conference","acknowledged_ssus":[{"_id":"CampIT"}],"date_created":"2025-01-30T07:53:22Z","author":[{"full_name":"Modoranu, Ionut-Vlad","id":"449f7a18-f128-11eb-9611-9b430c0c6333","last_name":"Modoranu","first_name":"Ionut-Vlad"},{"full_name":"Kalinov, Aleksei","last_name":"Kalinov","id":"44b7120e-eb97-11eb-a6c2-e1557aa81d02","first_name":"Aleksei","orcid":"0000-0003-2189-3904"},{"full_name":"Kurtic, Eldar","first_name":"Eldar","last_name":"Kurtic","id":"47beb3a5-07b5-11eb-9b87-b108ec578218"},{"last_name":"Frantar","id":"09a8f98d-ec99-11ea-ae11-c063a7b7fe5f","first_name":"Elias","full_name":"Frantar, Elias"},{"first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"}],"month":"07","alternative_title":["PMLR"],"supplementarymaterial":"no","publisher":"ML Research Press","oa":1,"acknowledgement":"The authors thank Adrian Vladu, Razvan Pascanu, Alexandra Peste, Mher Safaryan for their valuable feedback, the IT department from Institute of Science and Technology Austria for the hardware support and Weights and Biases for the infrastructure to track all our experiments.","user_id":"317138e5-6ab7-11ef-aa6d-ffef3953e345","abstract":[{"text":"Leveraging second-order information about the loss at the scale of deep networks is one of the main lines of approach for improving the performance of current optimizers for deep learning. Yet, existing approaches for accurate full-matrix preconditioning, such as Full-Matrix Adagrad (GGT) or Matrix-Free Approximate Curvature (M-FAC) suffer from massive storage costs when applied even to small-scale models, as they must store a sliding window of gradients, whose memory requirements are multiplicative in the model dimension. In this paper, we address this issue via a novel and efficient error-feedback technique that can be applied to compress preconditioners by up to two orders of magnitude in practice, without loss of convergence. Specifically, our approach compresses the gradient information via sparsification or low-rank compression before it is fed into the preconditioner, feeding the compression error back into future iterations. Extensive experiments on deep neural networks show that this approach can compress full-matrix preconditioners to up to 99% sparsity without accuracy loss, effectively removing the memory overhead of fullmatrix preconditioners such as GGT and M-FAC.","lang":"eng"}],"corr_author":"1","researchdata_availability":"yes","volume":235,"year":"2024","conference":{"end_date":"2024-07-27","start_date":"2024-07-21","location":"Vienna, Austria","name":"ICML: International Conference on Machine Learning"},"external_id":{"arxiv":["2306.06098"]},"language":[{"iso":"eng"}],"oa_version":"Preprint","intvolume":"       235","department":[{"_id":"DaAl"}],"article_processing_charge":"No","status":"public","date_updated":"2026-10-01T09:33:07Z","page":"35910-35933","_id":"18975"},{"quality_controlled":"1","doi":"10.52202/079017-0001","citation":{"chicago":"Modoranu, Ionut-Vlad, Mher Safaryan, Grigory Malinovsky, Eldar Kurtic, Thomas Robert, Peter Richtárik, and Dan-Adrian Alistarh. “MICROADAM: Accurate Adaptive Optimization with Low Space Overhead and Provable Convergence.” In <i>38th Conference on Neural Information Processing Systems</i>, Vol. 37. Neural Information Processing Systems Foundation, 2024. <a href=\"https://doi.org/10.52202/079017-0001\">https://doi.org/10.52202/079017-0001</a>.","short":"I.-V. Modoranu, M. Safaryan, G. Malinovsky, E. Kurtic, T. Robert, P. Richtárik, D.-A. Alistarh, in:, 38th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2024.","mla":"Modoranu, Ionut-Vlad, et al. “MICROADAM: Accurate Adaptive Optimization with Low Space Overhead and Provable Convergence.” <i>38th Conference on Neural Information Processing Systems</i>, vol. 37, Neural Information Processing Systems Foundation, 2024, doi:<a href=\"https://doi.org/10.52202/079017-0001\">10.52202/079017-0001</a>.","apa":"Modoranu, I.-V., Safaryan, M., Malinovsky, G., Kurtic, E., Robert, T., Richtárik, P., &#38; Alistarh, D.-A. (2024). MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence. In <i>38th Conference on Neural Information Processing Systems</i> (Vol. 37). Neural Information Processing Systems Foundation. <a href=\"https://doi.org/10.52202/079017-0001\">https://doi.org/10.52202/079017-0001</a>","ama":"Modoranu I-V, Safaryan M, Malinovsky G, et al. MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence. In: <i>38th Conference on Neural Information Processing Systems</i>. Vol 37. Neural Information Processing Systems Foundation; 2024. doi:<a href=\"https://doi.org/10.52202/079017-0001\">10.52202/079017-0001</a>","ista":"Modoranu I-V, Safaryan M, Malinovsky G, Kurtic E, Robert T, Richtárik P, Alistarh D-A. 2024. MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence. 38th Conference on Neural Information Processing Systems. , Advances in Neural Information Processing Systems, vol. 37.","ieee":"I.-V. Modoranu <i>et al.</i>, “MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence,” in <i>38th Conference on Neural Information Processing Systems</i>, 2024, vol. 37."},"publication_identifier":{"issn":["1049-5258"]},"arxiv":1,"publication":"38th Conference on Neural Information Processing Systems","title":"MICROADAM: Accurate adaptive optimization with low space overhead and provable convergence","main_file_link":[{"url":"https://doi.org/10.48550/arXiv.2405.15593","open_access":"1"}],"ec_funded":1,"type":"conference","acknowledged_ssus":[{"_id":"CampIT"}],"date_created":"2025-04-06T22:01:32Z","author":[{"full_name":"Modoranu, Ionut-Vlad","last_name":"Modoranu","id":"449f7a18-f128-11eb-9611-9b430c0c6333","first_name":"Ionut-Vlad"},{"full_name":"Safaryan, Mher","first_name":"Mher","last_name":"Safaryan","id":"dd546b39-0804-11ed-9c55-ef075c39778d"},{"first_name":"Grigory","last_name":"Malinovsky","full_name":"Malinovsky, Grigory"},{"full_name":"Kurtic, Eldar","first_name":"Eldar","id":"47beb3a5-07b5-11eb-9b87-b108ec578218","last_name":"Kurtic"},{"first_name":"Thomas","last_name":"Robert","id":"de632733-1457-11f0-ae22-b5914b8c1c41","full_name":"Robert, Thomas"},{"full_name":"Richtárik, Peter","first_name":"Peter","last_name":"Richtárik"},{"full_name":"Alistarh, Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X"}],"month":"12","date_published":"2024-12-20T00:00:00Z","publication_status":"published","scopus_import":"1","OA_type":"green","day":"20","fulldoi":"https://doi.org/10.52202/079017-0001","OA_place":"repository","oa":1,"acknowledgement":"The authors thank Razvan Pascanu, Mahdi Nikdan and Soroush Tabesh for their valuable feedback, the IT department from Institute of Science and Technology Austria for the hardware support and Weights and Biases for the infrastructure to track all our experiments. Mher Safaryan has received funding from the European Union’s Horizon 2020 research and innovation program under the Marie Sklodowska-Curie grant agreement No 101034413.","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","alternative_title":["Advances in Neural Information Processing Systems"],"publisher":"Neural Information Processing Systems Foundation","intvolume":"        37","department":[{"_id":"DaAl"}],"article_processing_charge":"No","related_material":{"link":[{"relation":"software","url":"https://github.com/IST-DASLab/MicroAdam"}]},"date_updated":"2026-10-15T09:49:27Z","status":"public","_id":"19510","abstract":[{"text":"We propose a new variant of the Adam optimizer [Kingma and Ba, 2014] called\r\nMICROADAM that specifically minimizes memory overheads, while maintaining\r\ntheoretical convergence guarantees. We achieve this by compressing the gradient\r\ninformation before it is fed into the optimizer state, thereby reducing its memory\r\nfootprint significantly. We control the resulting compression error via a novel\r\ninstance of the classical error feedback mechanism from distributed optimization [Seide et al., 2014, Alistarh et al., 2018, Karimireddy et al., 2019] in which\r\nthe error correction information is itself compressed to allow for practical memory\r\ngains. We prove that the resulting approach maintains theoretical convergence\r\nguarantees competitive to those of AMSGrad, while providing good practical performance. Specifically, we show that MICROADAM can be implemented efficiently\r\non GPUs: on both million-scale (BERT) and billion-scale (LLaMA) models, MICROADAM provides practical convergence competitive to that of the uncompressed\r\nAdam baseline, with lower memory usage and similar running time. Our code is\r\navailable at https://github.com/IST-DASLab/MicroAdam.","lang":"eng"}],"corr_author":"1","project":[{"call_identifier":"H2020","_id":"fc2ed2f7-9c52-11eb-aca3-c01059dda49c","grant_number":"101034413","name":"IST-BRIDGE: International postdoctoral program"}],"year":"2024","volume":37,"external_id":{"arxiv":["2405.15593"]},"language":[{"iso":"eng"}],"oa_version":"Preprint"},{"date_created":"2023-05-23T11:36:18Z","author":[{"first_name":"Elena-Alexandra","id":"32D78294-F248-11E8-B48F-1D18A9856A87","last_name":"Peste","full_name":"Peste, Elena-Alexandra"},{"full_name":"Vladu, Adrian","last_name":"Vladu","first_name":"Adrian"},{"first_name":"Eldar","id":"47beb3a5-07b5-11eb-9b87-b108ec578218","last_name":"Kurtic","full_name":"Kurtic, Eldar"},{"orcid":"0000-0001-8622-7887","first_name":"Christoph","last_name":"Lampert","id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","full_name":"Lampert, Christoph"},{"orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"month":"05","ec_funded":1,"type":"conference","acknowledged_ssus":[{"_id":"ScienComp"}],"file_date_updated":"2024-07-22T09:09:45Z","date_published":"2023-05-01T00:00:00Z","publication_status":"published","day":"01","quality_controlled":"1","publication":"11th International Conference on Learning Representations ","main_file_link":[{"open_access":"1","url":"https://openreview.net/pdf?id=_eTZBs-yedr"}],"title":"CrAM: A Compression-Aware Minimizer","citation":{"chicago":"Krumes, Alexandra, Adrian Vladu, Eldar Kurtic, Christoph Lampert, and Dan-Adrian Alistarh. “CrAM: A Compression-Aware Minimizer.” In <i>11th International Conference on Learning Representations </i>. OpenReview, 2023.","short":"A. Krumes, A. Vladu, E. Kurtic, C. Lampert, D.-A. Alistarh, in:, 11th International Conference on Learning Representations , OpenReview, 2023.","mla":"Krumes, Alexandra, et al. “CrAM: A Compression-Aware Minimizer.” <i>11th International Conference on Learning Representations </i>, OpenReview, 2023.","apa":"Krumes, A., Vladu, A., Kurtic, E., Lampert, C., &#38; Alistarh, D.-A. (2023). CrAM: A Compression-Aware Minimizer. In <i>11th International Conference on Learning Representations </i>. Kigali, Rwanda : OpenReview.","ista":"Krumes A, Vladu A, Kurtic E, Lampert C, Alistarh D-A. 2023. CrAM: A Compression-Aware Minimizer. 11th International Conference on Learning Representations . ICLR: International Conference on Learning Representations.","ama":"Krumes A, Vladu A, Kurtic E, Lampert C, Alistarh D-A. CrAM: A Compression-Aware Minimizer. In: <i>11th International Conference on Learning Representations </i>. OpenReview; 2023.","ieee":"A. Krumes, A. Vladu, E. Kurtic, C. Lampert, and D.-A. Alistarh, “CrAM: A Compression-Aware Minimizer,” in <i>11th International Conference on Learning Representations </i>, Kigali, Rwanda , 2023."},"arxiv":1,"article_processing_charge":"No","status":"public","date_updated":"2026-04-07T13:30:19Z","related_material":{"link":[{"url":"https://github.com/IST-DASLab/CrAM","relation":"software"}],"record":[{"relation":"dissertation_contains","id":"13074","status":"public"}]},"_id":"13053","department":[{"_id":"GradSch"},{"_id":"DaAl"},{"_id":"ChLa"}],"year":"2023","external_id":{"arxiv":["2207.14200"]},"conference":{"location":"Kigali, Rwanda ","name":"ICLR: International Conference on Learning Representations","start_date":"2023-05-01","end_date":"2023-05-05"},"language":[{"iso":"eng"}],"oa_version":"Published Version","file":[{"checksum":"a6eec897e13a91cdc3eeaf309801752c","file_id":"17294","content_type":"application/pdf","relation":"main_file","access_level":"open_access","date_updated":"2024-07-22T09:09:45Z","date_created":"2024-07-22T09:09:45Z","creator":"dernst","file_name":"2023_ICLR_Peste.pdf","file_size":458201,"success":1}],"corr_author":"1","abstract":[{"text":"Deep neural networks (DNNs) often have to be compressed, via pruning and/or quantization, before they can be deployed in practical settings. In this work we propose a new compression-aware minimizer dubbed CrAM that modifies the optimization step in a principled way, in order to produce models whose local loss behavior is stable under compression operations such as pruning. Thus, dense models trained via CrAM should be compressible post-training, in a single step, without significant accuracy loss. Experimental results on standard benchmarks, such as residual networks for ImageNet classification and BERT models for language modelling, show that CrAM produces dense models that can be more accurate than the standard SGD/Adam-based baselines, but which are stable under weight pruning: specifically, we can prune models in one-shot to 70-80% sparsity with almost no accuracy loss, and to 90% with reasonable (∼1%) accuracy loss, which is competitive with gradual compression methods. Additionally, CrAM can produce sparse models which perform well for transfer learning, and it also works for semi-structured 2:4 pruning patterns supported by GPU hardware. The code for reproducing the results is available at this https URL .","lang":"eng"}],"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"oa":1,"ddc":["000"],"acknowledgement":"AP, EK, DA received funding from the European Research Council (ERC) under the European\r\nUnion’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). AV acknowledges the support of the French Agence Nationale de la Recherche (ANR), under grant ANR-21-CE48-0016 (project COMCOPT). We further acknowledge the support from the Scientific Service Units (SSU) of ISTA through resources provided by Scientific Computing (SciComp).","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","has_accepted_license":"1","publisher":"OpenReview"},{"oa":1,"acknowledgement":"We would like to thank Elias Frantar for his valuable assistance and support at the outset of this project, and the anonymous ICML and SNN reviewers for very constructive feedback. EI was supported in part by the FWF DK VGSCO, grant agreement number W1260-N35. DA acknowledges generous ERC support, via Starting Grant 805223 ScaleML. ","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","alternative_title":["PMLR"],"publisher":"ML Research Press","intvolume":"       202","department":[{"_id":"DaAl"}],"article_processing_charge":"No","status":"public","date_updated":"2025-04-14T07:49:12Z","page":"26215-26227","_id":"14460","corr_author":"1","abstract":[{"text":"We provide an efficient implementation of the backpropagation algorithm, specialized to the case where the weights of the neural network being trained are sparse. Our algorithm is general, as it applies to arbitrary (unstructured) sparsity and common layer types (e.g., convolutional or linear). We provide a fast vectorized implementation on commodity CPUs, and show that it can yield speedups in end-to-end runtime experiments, both in transfer learning using already-sparsified networks, and in training sparse networks from scratch. Thus, our results provide the first support for sparse training on commodity hardware.","lang":"eng"}],"project":[{"call_identifier":"H2020","_id":"268A44D6-B435-11E9-9278-68D0E5697425","name":"Elastic Coordination for Scalable Machine Learning","grant_number":"805223"}],"year":"2023","volume":202,"conference":{"end_date":"2023-07-29","start_date":"2023-07-23","location":"Honolulu, Hawaii, HI, United States","name":"ICML: International Conference on Machine Learning"},"external_id":{"arxiv":["2302.04852"]},"language":[{"iso":"eng"}],"oa_version":"Preprint","quality_controlled":"1","citation":{"ieee":"M. Nikdan, T. Pegolotti, E. B. Iofinova, E. Kurtic, and D.-A. Alistarh, “SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge,” in <i>Proceedings of the 40th International Conference on Machine Learning</i>, Honolulu, Hawaii, HI, United States, 2023, vol. 202, pp. 26215–26227.","mla":"Nikdan, Mahdi, et al. “SparseProp: Efficient Sparse Backpropagation for Faster Training of Neural Networks at the Edge.” <i>Proceedings of the 40th International Conference on Machine Learning</i>, vol. 202, ML Research Press, 2023, pp. 26215–27.","ista":"Nikdan M, Pegolotti T, Iofinova EB, Kurtic E, Alistarh D-A. 2023. SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge. Proceedings of the 40th International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 202, 26215–26227.","apa":"Nikdan, M., Pegolotti, T., Iofinova, E. B., Kurtic, E., &#38; Alistarh, D.-A. (2023). SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge. In <i>Proceedings of the 40th International Conference on Machine Learning</i> (Vol. 202, pp. 26215–26227). Honolulu, Hawaii, HI, United States: ML Research Press.","ama":"Nikdan M, Pegolotti T, Iofinova EB, Kurtic E, Alistarh D-A. SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge. In: <i>Proceedings of the 40th International Conference on Machine Learning</i>. Vol 202. ML Research Press; 2023:26215-26227.","short":"M. Nikdan, T. Pegolotti, E.B. Iofinova, E. Kurtic, D.-A. Alistarh, in:, Proceedings of the 40th International Conference on Machine Learning, ML Research Press, 2023, pp. 26215–26227.","chicago":"Nikdan, Mahdi, Tommaso Pegolotti, Eugenia B Iofinova, Eldar Kurtic, and Dan-Adrian Alistarh. “SparseProp: Efficient Sparse Backpropagation for Faster Training of Neural Networks at the Edge.” In <i>Proceedings of the 40th International Conference on Machine Learning</i>, 202:26215–27. ML Research Press, 2023."},"publication_identifier":{"eissn":["2640-3498"]},"arxiv":1,"publication":"Proceedings of the 40th International Conference on Machine Learning","main_file_link":[{"url":"https://doi.org/10.48550/arXiv.2302.04852","open_access":"1"}],"title":"SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge","ec_funded":1,"type":"conference","date_created":"2023-10-29T23:01:17Z","author":[{"full_name":"Nikdan, Mahdi","first_name":"Mahdi","id":"66374281-f394-11eb-9cf6-869147deecc0","last_name":"Nikdan"},{"first_name":"Tommaso","last_name":"Pegolotti","full_name":"Pegolotti, Tommaso"},{"full_name":"Iofinova, Eugenia B","id":"f9a17499-f6e0-11ea-865d-fdf9a3f77117","last_name":"Iofinova","orcid":"0000-0002-7778-3221","first_name":"Eugenia B"},{"full_name":"Kurtic, Eldar","last_name":"Kurtic","id":"47beb3a5-07b5-11eb-9b87-b108ec578218","first_name":"Eldar"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian"}],"month":"07","date_published":"2023-07-30T00:00:00Z","publication_status":"published","scopus_import":"1","day":"30"},{"publication":"Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing","title":"The optimal BERT surgeon: Scalable and accurate second-order pruning for large language models","doi":"10.18653/v1/2022.emnlp-main.279","arxiv":1,"citation":{"apa":"Kurtic, E., Campos, D., Nguyen, T., Frantar, E., Kurtz, M., Fineran, B., … Alistarh, D.-A. (2022). The optimal BERT surgeon: Scalable and accurate second-order pruning for large language models. In <i>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing</i> (pp. 4163–4181). Abu Dhabi, United Arab Emirates: Association for Computational Linguistics. <a href=\"https://doi.org/10.18653/v1/2022.emnlp-main.279\">https://doi.org/10.18653/v1/2022.emnlp-main.279</a>","ista":"Kurtic E, Campos D, Nguyen T, Frantar E, Kurtz M, Fineran B, Goin M, Alistarh D-A. 2022. The optimal BERT surgeon: Scalable and accurate second-order pruning for large language models. Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing. EMNLP: Conference on Empirical Methods in Natural Language Processing, 4163–4181.","ama":"Kurtic E, Campos D, Nguyen T, et al. The optimal BERT surgeon: Scalable and accurate second-order pruning for large language models. In: <i>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing</i>. Association for Computational Linguistics; 2022:4163-4181. doi:<a href=\"https://doi.org/10.18653/v1/2022.emnlp-main.279\">10.18653/v1/2022.emnlp-main.279</a>","mla":"Kurtic, Eldar, et al. “The Optimal BERT Surgeon: Scalable and Accurate Second-Order Pruning for Large Language Models.” <i>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing</i>, Association for Computational Linguistics, 2022, pp. 4163–81, doi:<a href=\"https://doi.org/10.18653/v1/2022.emnlp-main.279\">10.18653/v1/2022.emnlp-main.279</a>.","ieee":"E. Kurtic <i>et al.</i>, “The optimal BERT surgeon: Scalable and accurate second-order pruning for large language models,” in <i>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing</i>, Abu Dhabi, United Arab Emirates, 2022, pp. 4163–4181.","chicago":"Kurtic, Eldar, Daniel Campos, Tuan Nguyen, Elias Frantar, Mark Kurtz, Benjamin Fineran, Michael Goin, and Dan-Adrian Alistarh. “The Optimal BERT Surgeon: Scalable and Accurate Second-Order Pruning for Large Language Models.” In <i>Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing</i>, 4163–81. Association for Computational Linguistics, 2022. <a href=\"https://doi.org/10.18653/v1/2022.emnlp-main.279\">https://doi.org/10.18653/v1/2022.emnlp-main.279</a>.","short":"E. Kurtic, D. Campos, T. Nguyen, E. Frantar, M. Kurtz, B. Fineran, M. Goin, D.-A. Alistarh, in:, Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, Association for Computational Linguistics, 2022, pp. 4163–4181."},"quality_controlled":"1","scopus_import":"1","publication_status":"published","date_published":"2022-12-01T00:00:00Z","file_date_updated":"2024-07-31T11:03:34Z","fulldoi":"https://doi.org/10.18653/v1/2022.emnlp-main.279","day":"01","date_created":"2024-05-29T06:40:55Z","month":"12","author":[{"first_name":"Eldar","last_name":"Kurtic","id":"47beb3a5-07b5-11eb-9b87-b108ec578218","full_name":"Kurtic, Eldar"},{"full_name":"Campos, Daniel","last_name":"Campos","first_name":"Daniel"},{"full_name":"Nguyen, Tuan","first_name":"Tuan","last_name":"Nguyen"},{"full_name":"Frantar, Elias","last_name":"Frantar","id":"09a8f98d-ec99-11ea-ae11-c063a7b7fe5f","first_name":"Elias"},{"last_name":"Kurtz","first_name":"Mark","full_name":"Kurtz, Mark"},{"last_name":"Fineran","first_name":"Benjamin","full_name":"Fineran, Benjamin"},{"full_name":"Goin, Michael","last_name":"Goin","first_name":"Michael"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X"}],"type":"conference","tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)"},"has_accepted_license":"1","publisher":"Association for Computational Linguistics","ddc":["000"],"oa":1,"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","external_id":{"arxiv":["2203.07259"]},"conference":{"start_date":"2022-12-07","end_date":"2022-12-11","name":"EMNLP: Conference on Empirical Methods in Natural Language Processing","location":"Abu Dhabi, United Arab Emirates"},"year":"2022","oa_version":"Published Version","language":[{"iso":"eng"}],"abstract":[{"text":"In this paper, we consider the problem of sparsifying BERT models, which are a key building block for natural language processing, in order to reduce their storage and computational cost. We introduce the Optimal BERT Surgeon (oBERT), an efficient and accurate pruning method based on approximate second-order information, which we show to yield state-of-the-art results in both stages of language tasks: pre-training and fine-tuning. Specifically, oBERT extends existing work on second-order pruning by allowing for pruning weight blocks, and is the first such method that is applicable at BERT scale. Second, we investigate compounding compression approaches to obtain highly compressed but accurate models for deployment on edge devices. These models significantly push boundaries of the current state-of-the-art sparse BERT models with respect to all metrics: model size, inference speed and task accuracy. For example, relative to the dense BERT-base, we obtain 10x model size compression with < 1% accuracy drop, 10x CPU-inference speedup with < 2% accuracy drop, and 29x CPU-inference speedup with < 7.5% accuracy drop. Our code, fully integrated with Transformers and SparseML, is available at https://github.com/neuralmagic/sparseml/tree/main/research/optimal_BERT_surgeon_oBERT.","lang":"eng"}],"corr_author":"1","file":[{"relation":"main_file","access_level":"open_access","checksum":"c47b9edd8a9f743ac77a593de6d2e84a","file_id":"17354","content_type":"application/pdf","success":1,"file_name":"2022_EMNLP_Kurtic.pdf","file_size":522563,"date_updated":"2024-07-31T11:03:34Z","date_created":"2024-07-31T11:03:34Z","creator":"dernst"}],"status":"public","date_updated":"2024-07-31T11:05:32Z","related_material":{"link":[{"relation":"software","url":"https://github.com/neuralmagic/sparseml/tree/main/research/optimal_BERT_surgeon_oBERT"}]},"article_processing_charge":"Yes","_id":"17088","page":"4163-4181","department":[{"_id":"DaAl"}]},{"publisher":"Neural Information Processing Systems Foundation","alternative_title":["Advances in Neural Information Processing Systems"],"acknowledgement":"We gratefully acknowledge funding the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML), as well as computational support from Amazon Web Services (AWS) EC2.","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","oa":1,"ddc":["000"],"project":[{"call_identifier":"H2020","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","name":"Elastic Coordination for Scalable Machine Learning"}],"abstract":[{"text":"Efficiently approximating local curvature information of the loss function is a key tool for optimization and compression of deep neural networks. Yet, most existing methods to approximate second-order information have high computational\r\nor storage costs, which limits their practicality. In this work, we investigate matrix-free, linear-time approaches for estimating Inverse-Hessian Vector Products (IHVPs) for the case when the Hessian can be approximated as a sum of rank-one matrices, as in the classic approximation of the Hessian by the empirical Fisher matrix. We propose two new algorithms: the first is tailored towards network compression and can compute the IHVP for dimension d, if the Hessian is given as a sum of m rank-one matrices, using O(dm2) precomputation, O(dm) cost for computing the IHVP, and query cost O(m) for any single element of the inverse Hessian. The second algorithm targets an optimization setting, where we wish to compute the product between the inverse Hessian, estimated over a sliding window of optimization steps, and a given gradient direction, as required for preconditioned SGD. We give an algorithm with cost O(dm + m2) for computing the IHVP and O(dm + m3) for adding or removing any gradient from the sliding window. These\r\ntwo algorithms yield state-of-the-art results for network pruning and optimization with lower computational overhead relative to existing second-order methods. Implementations are available at [9] and [17].","lang":"eng"}],"corr_author":"1","language":[{"iso":"eng"}],"oa_version":"Published Version","year":"2021","volume":34,"external_id":{"arxiv":["2010.08222"]},"conference":{"start_date":"2021-12-06","end_date":"2021-12-14","location":"Virtual, Online","name":"NeurIPS: Neural Information Processing Systems"},"department":[{"_id":"DaAl"}],"intvolume":"        34","page":"14873-14886","_id":"11463","article_processing_charge":"No","status":"public","date_updated":"2026-06-18T17:18:44Z","citation":{"apa":"Frantar, E., Kurtic, E., &#38; Alistarh, D.-A. (2021). M-FAC: Efficient matrix-free approximations of second-order information. In <i>35th Conference on Neural Information Processing Systems</i> (Vol. 34, pp. 14873–14886). Virtual, Online: Neural Information Processing Systems Foundation.","ista":"Frantar E, Kurtic E, Alistarh D-A. 2021. M-FAC: Efficient matrix-free approximations of second-order information. 35th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 34, 14873–14886.","ama":"Frantar E, Kurtic E, Alistarh D-A. M-FAC: Efficient matrix-free approximations of second-order information. In: <i>35th Conference on Neural Information Processing Systems</i>. Vol 34. Neural Information Processing Systems Foundation; 2021:14873-14886.","mla":"Frantar, Elias, et al. “M-FAC: Efficient Matrix-Free Approximations of Second-Order Information.” <i>35th Conference on Neural Information Processing Systems</i>, vol. 34, Neural Information Processing Systems Foundation, 2021, pp. 14873–86.","ieee":"E. Frantar, E. Kurtic, and D.-A. Alistarh, “M-FAC: Efficient matrix-free approximations of second-order information,” in <i>35th Conference on Neural Information Processing Systems</i>, Virtual, Online, 2021, vol. 34, pp. 14873–14886.","chicago":"Frantar, Elias, Eldar Kurtic, and Dan-Adrian Alistarh. “M-FAC: Efficient Matrix-Free Approximations of Second-Order Information.” In <i>35th Conference on Neural Information Processing Systems</i>, 34:14873–86. Neural Information Processing Systems Foundation, 2021.","short":"E. Frantar, E. Kurtic, D.-A. Alistarh, in:, 35th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2021, pp. 14873–14886."},"publication_identifier":{"issn":["1049-5258"],"isbn":["9781713845393"]},"arxiv":1,"main_file_link":[{"open_access":"1","url":"https://proceedings.neurips.cc/paper/2021/file/7cfd5df443b4eb0d69886a583b33de4c-Paper.pdf"}],"title":"M-FAC: Efficient matrix-free approximations of second-order information","publication":"35th Conference on Neural Information Processing Systems","quality_controlled":"1","day":"06","date_published":"2021-12-06T00:00:00Z","publication_status":"published","scopus_import":"1","type":"conference","ec_funded":1,"author":[{"id":"09a8f98d-ec99-11ea-ae11-c063a7b7fe5f","last_name":"Frantar","first_name":"Elias","full_name":"Frantar, Elias"},{"full_name":"Kurtic, Eldar","first_name":"Eldar","last_name":"Kurtic","id":"47beb3a5-07b5-11eb-9b87-b108ec578218"},{"orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"month":"12","date_created":"2022-06-26T22:01:35Z"}]