[{"date_updated":"2026-06-03T05:53:30Z","file_date_updated":"2026-06-03T05:51:19Z","citation":{"ista":"Schultheis E, Alistarh D-A. 2026. LLMQ: Efficient lower-precision LLM training for consumer GPUs. 2nd Conference on Parsimony and Learning. CPAL: Conference on Parsimony and Learning, PMLR, vol. 328, 265–284.","short":"E. Schultheis, D.-A. Alistarh, in:, 2nd Conference on Parsimony and Learning, ML Research Press, 2026, pp. 265–284.","ieee":"E. Schultheis and D.-A. Alistarh, “LLMQ: Efficient lower-precision LLM training for consumer GPUs,” in <i>2nd Conference on Parsimony and Learning</i>, Stanford, CA, United States, 2026, vol. 328, pp. 265–284.","apa":"Schultheis, E., &#38; Alistarh, D.-A. (2026). LLMQ: Efficient lower-precision LLM training for consumer GPUs. In <i>2nd Conference on Parsimony and Learning</i> (Vol. 328, pp. 265–284). Stanford, CA, United States: ML Research Press.","ama":"Schultheis E, Alistarh D-A. LLMQ: Efficient lower-precision LLM training for consumer GPUs. In: <i>2nd Conference on Parsimony and Learning</i>. Vol 328. ML Research Press; 2026:265-284.","mla":"Schultheis, Erik, and Dan-Adrian Alistarh. “LLMQ: Efficient Lower-Precision LLM Training for Consumer GPUs.” <i>2nd Conference on Parsimony and Learning</i>, vol. 328, ML Research Press, 2026, pp. 265–84.","chicago":"Schultheis, Erik, and Dan-Adrian Alistarh. “LLMQ: Efficient Lower-Precision LLM Training for Consumer GPUs.” In <i>2nd Conference on Parsimony and Learning</i>, 328:265–84. ML Research Press, 2026."},"page":"265-284","license":"https://creativecommons.org/licenses/by/4.0/","conference":{"name":"CPAL: Conference on Parsimony and Learning","start_date":"2025-03-24","end_date":"2025-03-27","location":"Stanford, CA, United States"},"quality_controlled":"1","department":[{"_id":"DaAl"}],"related_material":{"link":[{"relation":"software","url":"https://github.com/IST-DASLab/llmq"}]},"status":"public","title":"LLMQ: Efficient lower-precision LLM training for consumer GPUs","month":"04","type":"conference","alternative_title":["PMLR"],"publisher":"ML Research Press","_id":"21932","tmp":{"image":"/images/cc_by.png","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","short":"CC BY (4.0)"},"scopus_import":"1","author":[{"id":"2786b299-e6b0-11f0-91da-9243fe3ef96b","last_name":"Schultheis","full_name":"Schultheis, Erik","first_name":"Erik"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian"}],"has_accepted_license":"1","OA_type":"diamond","publication_status":"published","file":[{"file_name":"2026_CPAL_Schultheis.pdf","content_type":"application/pdf","access_level":"open_access","date_updated":"2026-06-03T05:51:19Z","creator":"dernst","checksum":"72f9a87c70f1e2105ef64050ee5017e5","date_created":"2026-06-03T05:51:19Z","relation":"main_file","file_size":2099944,"file_id":"21942","success":1}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","publication_identifier":{"eissn":["2640-3498"]},"OA_place":"publisher","oa":1,"day":"06","language":[{"iso":"eng"}],"acknowledgement":"We would like to thank contacts at NVIDIA (Vartika Singh, Nina Carrejo, Kyla Wilkes, and Tijmen\r\nBlankevoort), HP (Curtis Burkhalter), and Datacrunch/Verda (Paul Chang and Antonio\r\nDominguez) for hardware support that was essential to this project. ES was supported in part\r\nby ERC Proof-of-Concept grant FastML.","year":"2026","intvolume":"       328","abstract":[{"text":"We present LLMQ, an end-to-end CUDA/C++ implementation for medium-sized language-model training, e.g. 3B to 32B parameters, on affordable, commodity GPUs. These devices are characterized by low memory availability and slow communication compared to datacentre-grade GPUs. Consequently, we showcase a range of optimizations that target these bottlenecks, including activation checkpointing, offloading, and copy-engine based collectives. LLMQ is able to train or fine-tune a 7B model on a single 16GB mid-range gaming card, or a 32B model on a workstation equipped with 4 RTX 4090s. This is achieved while executing a standard 8-bit training pipeline, without additional algorithmic approximations, and maintaining FLOP utilization of around 50%. The efficiency of LLMQ rivals that of production-scale systems on much more expensive cloud-grade GPUs.","lang":"eng"}],"article_processing_charge":"No","date_published":"2026-04-06T00:00:00Z","publication":"2nd Conference on Parsimony and Learning","date_created":"2026-05-31T22:02:13Z","corr_author":"1","ddc":["000"],"oa_version":"Published Version","volume":328}]