[{"date_published":"2026-04-06T00:00:00Z","title":"LLMQ: Efficient lower-precision LLM training for consumer GPUs","corr_author":"1","ddc":["000"],"date_updated":"2026-06-03T05:53:30Z","oa_version":"Published Version","date_created":"2026-05-31T22:02:13Z","language":[{"iso":"eng"}],"publisher":"ML Research Press","tmp":{"legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)"},"conference":{"start_date":"2025-03-24","location":"Stanford, CA, United States","name":"CPAL: Conference on Parsimony and Learning","end_date":"2025-03-27"},"related_material":{"link":[{"relation":"software","url":"https://github.com/IST-DASLab/llmq"}]},"year":"2026","OA_place":"publisher","OA_type":"diamond","quality_controlled":"1","status":"public","citation":{"mla":"Schultheis, Erik, and Dan-Adrian Alistarh. “LLMQ: Efficient Lower-Precision LLM Training for Consumer GPUs.” <i>2nd Conference on Parsimony and Learning</i>, vol. 328, ML Research Press, 2026, pp. 265–84.","ieee":"E. Schultheis and D.-A. Alistarh, “LLMQ: Efficient lower-precision LLM training for consumer GPUs,” in <i>2nd Conference on Parsimony and Learning</i>, Stanford, CA, United States, 2026, vol. 328, pp. 265–284.","chicago":"Schultheis, Erik, and Dan-Adrian Alistarh. “LLMQ: Efficient Lower-Precision LLM Training for Consumer GPUs.” In <i>2nd Conference on Parsimony and Learning</i>, 328:265–84. ML Research Press, 2026.","short":"E. Schultheis, D.-A. Alistarh, in:, 2nd Conference on Parsimony and Learning, ML Research Press, 2026, pp. 265–284.","apa":"Schultheis, E., &#38; Alistarh, D.-A. (2026). LLMQ: Efficient lower-precision LLM training for consumer GPUs. In <i>2nd Conference on Parsimony and Learning</i> (Vol. 328, pp. 265–284). Stanford, CA, United States: ML Research Press.","ama":"Schultheis E, Alistarh D-A. LLMQ: Efficient lower-precision LLM training for consumer GPUs. In: <i>2nd Conference on Parsimony and Learning</i>. Vol 328. ML Research Press; 2026:265-284.","ista":"Schultheis E, Alistarh D-A. 2026. LLMQ: Efficient lower-precision LLM training for consumer GPUs. 2nd Conference on Parsimony and Learning. CPAL: Conference on Parsimony and Learning, PMLR, vol. 328, 265–284."},"alternative_title":["PMLR"],"intvolume":"       328","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","volume":328,"page":"265-284","publication_identifier":{"eissn":["2640-3498"]},"abstract":[{"text":"We present LLMQ, an end-to-end CUDA/C++ implementation for medium-sized language-model training, e.g. 3B to 32B parameters, on affordable, commodity GPUs. These devices are characterized by low memory availability and slow communication compared to datacentre-grade GPUs. Consequently, we showcase a range of optimizations that target these bottlenecks, including activation checkpointing, offloading, and copy-engine based collectives. LLMQ is able to train or fine-tune a 7B model on a single 16GB mid-range gaming card, or a 32B model on a workstation equipped with 4 RTX 4090s. This is achieved while executing a standard 8-bit training pipeline, without additional algorithmic approximations, and maintaining FLOP utilization of around 50%. The efficiency of LLMQ rivals that of production-scale systems on much more expensive cloud-grade GPUs.","lang":"eng"}],"type":"conference","_id":"21932","author":[{"last_name":"Schultheis","first_name":"Erik","id":"2786b299-e6b0-11f0-91da-9243fe3ef96b","full_name":"Schultheis, Erik"},{"first_name":"Dan-Adrian","last_name":"Alistarh","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"}],"acknowledgement":"We would like to thank contacts at NVIDIA (Vartika Singh, Nina Carrejo, Kyla Wilkes, and Tijmen\r\nBlankevoort), HP (Curtis Burkhalter), and Datacrunch/Verda (Paul Chang and Antonio\r\nDominguez) for hardware support that was essential to this project. ES was supported in part\r\nby ERC Proof-of-Concept grant FastML.","has_accepted_license":"1","month":"04","day":"06","department":[{"_id":"DaAl"}],"publication_status":"published","oa":1,"file_date_updated":"2026-06-03T05:51:19Z","file":[{"file_name":"2026_CPAL_Schultheis.pdf","relation":"main_file","success":1,"file_id":"21942","checksum":"72f9a87c70f1e2105ef64050ee5017e5","creator":"dernst","date_created":"2026-06-03T05:51:19Z","content_type":"application/pdf","access_level":"open_access","file_size":2099944,"date_updated":"2026-06-03T05:51:19Z"}],"publication":"2nd Conference on Parsimony and Learning","article_processing_charge":"No","scopus_import":"1"},{"year":"2025","OA_place":"publisher","OA_type":"gold","status":"public","quality_controlled":"1","citation":{"mla":"Rodionov, Gleb, et al. “Hogwild! Inference: Parallel LLM Generation via Concurrent Attention.” <i>39th Conference on Neural Information Processing Systems</i>, vol. 38, Neural Information Processing Systems Foundation, 2025, pp. 52014–55, doi:<a href=\"https://doi.org/10.52202/085713-1551\">10.52202/085713-1551</a>.","ieee":"G. Rodionov <i>et al.</i>, “Hogwild! Inference: Parallel LLM generation via concurrent attention,” in <i>39th Conference on Neural Information Processing Systems</i>, San Diego, CA, United States, 2025, vol. 38, pp. 52014–52055.","chicago":"Rodionov, Gleb, Roman Garipov, Alina Shutova, George Yakushev, Erik Schultheis, Vage Egiazarian, Anton Sinitsin, Denis Kuznedelev, and Dan-Adrian Alistarh. “Hogwild! Inference: Parallel LLM Generation via Concurrent Attention.” In <i>39th Conference on Neural Information Processing Systems</i>, 38:52014–55. Neural Information Processing Systems Foundation, 2025. <a href=\"https://doi.org/10.52202/085713-1551\">https://doi.org/10.52202/085713-1551</a>.","short":"G. Rodionov, R. Garipov, A. Shutova, G. Yakushev, E. Schultheis, V. Egiazarian, A. Sinitsin, D. Kuznedelev, D.-A. Alistarh, in:, 39th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2025, pp. 52014–52055.","apa":"Rodionov, G., Garipov, R., Shutova, A., Yakushev, G., Schultheis, E., Egiazarian, V., … Alistarh, D.-A. (2025). Hogwild! Inference: Parallel LLM generation via concurrent attention. In <i>39th Conference on Neural Information Processing Systems</i> (Vol. 38, pp. 52014–52055). San Diego, CA, United States: Neural Information Processing Systems Foundation. <a href=\"https://doi.org/10.52202/085713-1551\">https://doi.org/10.52202/085713-1551</a>","ama":"Rodionov G, Garipov R, Shutova A, et al. Hogwild! Inference: Parallel LLM generation via concurrent attention. In: <i>39th Conference on Neural Information Processing Systems</i>. Vol 38. Neural Information Processing Systems Foundation; 2025:52014-52055. doi:<a href=\"https://doi.org/10.52202/085713-1551\">10.52202/085713-1551</a>","ista":"Rodionov G, Garipov R, Shutova A, Yakushev G, Schultheis E, Egiazarian V, Sinitsin A, Kuznedelev D, Alistarh D-A. 2025. Hogwild! Inference: Parallel LLM generation via concurrent attention. 39th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 38, 52014–52055."},"alternative_title":["Advances in Neural Information Processing Systems"],"intvolume":"        38","date_published":"2025-12-02T00:00:00Z","doi":"10.52202/085713-1551","title":"Hogwild! Inference: Parallel LLM generation via concurrent attention","ddc":["000"],"date_updated":"2026-09-10T07:06:31Z","date_created":"2026-09-06T22:01:59Z","oa_version":"Published Version","das_tickbox":"0","language":[{"iso":"eng"}],"publisher":"Neural Information Processing Systems Foundation","conference":{"start_date":"2025-12-02","location":"San Diego, CA, United States","name":"NeurIPS: Neural Information Processing Systems","end_date":"2025-12-07"},"has_accepted_license":"1","researchdata_availability":"no","day":"02","month":"12","publication_status":"published","department":[{"_id":"DaAl"}],"oa":1,"publication":"39th Conference on Neural Information Processing Systems","article_processing_charge":"No","scopus_import":"1","fulldoi":"https://doi.org/10.52202/085713-1551","main_file_link":[{"open_access":"1","url":"https://doi.org/10.52202/085713-1551"}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","volume":38,"page":"52014-52055","publication_identifier":{"isbn":["9798331338275"],"issn":["1049-5258"]},"abstract":[{"lang":"eng","text":"Large Language Models (LLMs) have demonstrated the ability to tackle increasingly complex tasks through advanced reasoning, long-form content generation,\r\nand tool use. Solving these tasks often involves long inference-time computations.\r\nIn human problem solving, a common strategy to expedite work is collaboration: by\r\ndividing the problem into sub-tasks, exploring different strategies concurrently, etc.\r\nRecent research has shown that LLMs can also operate in parallel by implementing\r\nexplicit cooperation frameworks, such as voting mechanisms or the explicit creation of independent sub-tasks that can be executed in parallel. However, each of\r\nthese frameworks may not be suitable for all types of tasks, which can hinder their\r\napplicability. In this work, we propose a different design approach: we run LLM\r\n“workers” in parallel , allowing them to synchronize via a concurrently-updated\r\nattention cache and prompt these workers to decide how best to collaborate. Our\r\napproach allows the LLM instances to come up with their own collaboration strategy for the problem at hand, all the while “seeing” each other’s memory in the\r\nconcurrent KV cache. We implement this approach via Hogwild! Inference: a\r\nparallel LLM inference engine where multiple instances of the same LLM run in\r\nparallel with the same attention cache, with “instant” access to each other’s memory.1 Hogwild! Inference takes advantage of Rotary Position Embeddings (RoPE)\r\nto avoid recomputation while improving parallel hardware utilization. We find that\r\nmodern reasoning-capable LLMs can perform inference with shared Key-Value\r\ncache out of the box, without additional fine-tuning."}],"type":"conference","supplementarymaterial":"yes","_id":"22826","author":[{"full_name":"Rodionov, Gleb","first_name":"Gleb","last_name":"Rodionov"},{"full_name":"Garipov, Roman","last_name":"Garipov","first_name":"Roman"},{"first_name":"Alina","last_name":"Shutova","full_name":"Shutova, Alina"},{"first_name":"George","last_name":"Yakushev","full_name":"Yakushev, George"},{"full_name":"Schultheis, Erik","id":"2786b299-e6b0-11f0-91da-9243fe3ef96b","last_name":"Schultheis","first_name":"Erik"},{"first_name":"Vage","last_name":"Egiazarian","id":"77451e76-92b2-11ef-a4d1-8dbaa06e16ad","full_name":"Egiazarian, Vage"},{"last_name":"Sinitsin","first_name":"Anton","full_name":"Sinitsin, Anton"},{"full_name":"Kuznedelev, Denis","first_name":"Denis","last_name":"Kuznedelev"},{"first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X"}],"acknowledgement":"We thank Vladimir Malinovskii for his help with brainstorming, helpful\r\nfeedback and suggesting future work directions. We also thank Philip Zmushko for proofreading."}]