[{"status":"public","language":[{"iso":"eng"}],"author":[{"last_name":"Rodionov","full_name":"Rodionov, Gleb","first_name":"Gleb"},{"last_name":"Garipov","full_name":"Garipov, Roman","first_name":"Roman"},{"full_name":"Shutova, Alina","last_name":"Shutova","first_name":"Alina"},{"last_name":"Yakushev","full_name":"Yakushev, George","first_name":"George"},{"full_name":"Schultheis, Erik","id":"2786b299-e6b0-11f0-91da-9243fe3ef96b","last_name":"Schultheis","first_name":"Erik"},{"id":"77451e76-92b2-11ef-a4d1-8dbaa06e16ad","full_name":"Egiazarian, Vage","last_name":"Egiazarian","first_name":"Vage"},{"last_name":"Sinitsin","full_name":"Sinitsin, Anton","first_name":"Anton"},{"first_name":"Denis","full_name":"Kuznedelev, Denis","last_name":"Kuznedelev"},{"full_name":"Alistarh, Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian"}],"day":"02","abstract":[{"text":"Large Language Models (LLMs) have demonstrated the ability to tackle increasingly complex tasks through advanced reasoning, long-form content generation,\r\nand tool use. Solving these tasks often involves long inference-time computations.\r\nIn human problem solving, a common strategy to expedite work is collaboration: by\r\ndividing the problem into sub-tasks, exploring different strategies concurrently, etc.\r\nRecent research has shown that LLMs can also operate in parallel by implementing\r\nexplicit cooperation frameworks, such as voting mechanisms or the explicit creation of independent sub-tasks that can be executed in parallel. However, each of\r\nthese frameworks may not be suitable for all types of tasks, which can hinder their\r\napplicability. In this work, we propose a different design approach: we run LLM\r\n“workers” in parallel , allowing them to synchronize via a concurrently-updated\r\nattention cache and prompt these workers to decide how best to collaborate. Our\r\napproach allows the LLM instances to come up with their own collaboration strategy for the problem at hand, all the while “seeing” each other’s memory in the\r\nconcurrent KV cache. We implement this approach via Hogwild! Inference: a\r\nparallel LLM inference engine where multiple instances of the same LLM run in\r\nparallel with the same attention cache, with “instant” access to each other’s memory.1 Hogwild! Inference takes advantage of Rotary Position Embeddings (RoPE)\r\nto avoid recomputation while improving parallel hardware utilization. We find that\r\nmodern reasoning-capable LLMs can perform inference with shared Key-Value\r\ncache out of the box, without additional fine-tuning.","lang":"eng"}],"OA_type":"gold","alternative_title":["Advances in Neural Information Processing Systems"],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","publisher":"Neural Information Processing Systems Foundation","researchdata_availability":"no","_id":"22826","supplementarymaterial":"yes","department":[{"_id":"DaAl"}],"scopus_import":"1","citation":{"apa":"Rodionov, G., Garipov, R., Shutova, A., Yakushev, G., Schultheis, E., Egiazarian, V., … Alistarh, D.-A. (2025). Hogwild! Inference: Parallel LLM generation via concurrent attention. In <i>39th Conference on Neural Information Processing Systems</i> (Vol. 38, pp. 52014–52055). San Diego, CA, United States: Neural Information Processing Systems Foundation. <a href=\"https://doi.org/10.52202/085713-1551\">https://doi.org/10.52202/085713-1551</a>","short":"G. Rodionov, R. Garipov, A. Shutova, G. Yakushev, E. Schultheis, V. Egiazarian, A. Sinitsin, D. Kuznedelev, D.-A. Alistarh, in:, 39th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2025, pp. 52014–52055.","ama":"Rodionov G, Garipov R, Shutova A, et al. Hogwild! Inference: Parallel LLM generation via concurrent attention. In: <i>39th Conference on Neural Information Processing Systems</i>. Vol 38. Neural Information Processing Systems Foundation; 2025:52014-52055. doi:<a href=\"https://doi.org/10.52202/085713-1551\">10.52202/085713-1551</a>","mla":"Rodionov, Gleb, et al. “Hogwild! Inference: Parallel LLM Generation via Concurrent Attention.” <i>39th Conference on Neural Information Processing Systems</i>, vol. 38, Neural Information Processing Systems Foundation, 2025, pp. 52014–55, doi:<a href=\"https://doi.org/10.52202/085713-1551\">10.52202/085713-1551</a>.","ieee":"G. Rodionov <i>et al.</i>, “Hogwild! Inference: Parallel LLM generation via concurrent attention,” in <i>39th Conference on Neural Information Processing Systems</i>, San Diego, CA, United States, 2025, vol. 38, pp. 52014–52055.","chicago":"Rodionov, Gleb, Roman Garipov, Alina Shutova, George Yakushev, Erik Schultheis, Vage Egiazarian, Anton Sinitsin, Denis Kuznedelev, and Dan-Adrian Alistarh. “Hogwild! Inference: Parallel LLM Generation via Concurrent Attention.” In <i>39th Conference on Neural Information Processing Systems</i>, 38:52014–55. Neural Information Processing Systems Foundation, 2025. <a href=\"https://doi.org/10.52202/085713-1551\">https://doi.org/10.52202/085713-1551</a>.","ista":"Rodionov G, Garipov R, Shutova A, Yakushev G, Schultheis E, Egiazarian V, Sinitsin A, Kuznedelev D, Alistarh D-A. 2025. Hogwild! Inference: Parallel LLM generation via concurrent attention. 39th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 38, 52014–52055."},"das_tickbox":"0","acknowledgement":"We thank Vladimir Malinovskii for his help with brainstorming, helpful\r\nfeedback and suggesting future work directions. We also thank Philip Zmushko for proofreading.","date_published":"2025-12-02T00:00:00Z","OA_place":"publisher","doi":"10.52202/085713-1551","oa_version":"Published Version","main_file_link":[{"open_access":"1","url":"https://doi.org/10.52202/085713-1551"}],"ddc":["000"],"quality_controlled":"1","has_accepted_license":"1","page":"52014-52055","month":"12","oa":1,"title":"Hogwild! Inference: Parallel LLM generation via concurrent attention","article_processing_charge":"No","intvolume":"        38","volume":38,"conference":{"location":"San Diego, CA, United States","end_date":"2025-12-07","name":"NeurIPS: Neural Information Processing Systems","start_date":"2025-12-02"},"type":"conference","year":"2025","publication":"39th Conference on Neural Information Processing Systems","publication_status":"published","publication_identifier":{"issn":["1049-5258"],"isbn":["9798331338275"]},"date_created":"2026-09-06T22:01:59Z","fulldoi":"https://doi.org/10.52202/085713-1551","date_updated":"2026-09-10T07:06:31Z"}]