[{"quality_controlled":"1","article_processing_charge":"No","date_created":"2026-09-06T22:01:59Z","supplementarymaterial":"yes","month":"12","page":"52014-52055","publication_status":"published","language":[{"iso":"eng"}],"volume":38,"department":[{"_id":"DaAl"}],"oa":1,"title":"Hogwild! Inference: Parallel LLM generation via concurrent attention","publication_identifier":{"isbn":["9798331338275"],"issn":["1049-5258"]},"type":"conference","main_file_link":[{"url":"https://doi.org/10.52202/085713-1551","open_access":"1"}],"doi":"10.52202/085713-1551","publication":"39th Conference on Neural Information Processing Systems","author":[{"last_name":"Rodionov","full_name":"Rodionov, Gleb","first_name":"Gleb"},{"first_name":"Roman","full_name":"Garipov, Roman","last_name":"Garipov"},{"last_name":"Shutova","full_name":"Shutova, Alina","first_name":"Alina"},{"last_name":"Yakushev","full_name":"Yakushev, George","first_name":"George"},{"first_name":"Erik","id":"2786b299-e6b0-11f0-91da-9243fe3ef96b","last_name":"Schultheis","full_name":"Schultheis, Erik"},{"full_name":"Egiazarian, Vage","id":"77451e76-92b2-11ef-a4d1-8dbaa06e16ad","last_name":"Egiazarian","first_name":"Vage"},{"first_name":"Anton","full_name":"Sinitsin, Anton","last_name":"Sinitsin"},{"last_name":"Kuznedelev","full_name":"Kuznedelev, Denis","first_name":"Denis"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian"}],"conference":{"start_date":"2025-12-02","location":"San Diego, CA, United States","end_date":"2025-12-07","name":"NeurIPS: Neural Information Processing Systems"},"scopus_import":"1","intvolume":"        38","das_tickbox":"0","day":"02","publisher":"Neural Information Processing Systems Foundation","acknowledgement":"We thank Vladimir Malinovskii for his help with brainstorming, helpful\r\nfeedback and suggesting future work directions. We also thank Philip Zmushko for proofreading.","year":"2025","citation":{"ieee":"G. Rodionov <i>et al.</i>, “Hogwild! Inference: Parallel LLM generation via concurrent attention,” in <i>39th Conference on Neural Information Processing Systems</i>, San Diego, CA, United States, 2025, vol. 38, pp. 52014–52055.","apa":"Rodionov, G., Garipov, R., Shutova, A., Yakushev, G., Schultheis, E., Egiazarian, V., … Alistarh, D.-A. (2025). Hogwild! Inference: Parallel LLM generation via concurrent attention. In <i>39th Conference on Neural Information Processing Systems</i> (Vol. 38, pp. 52014–52055). San Diego, CA, United States: Neural Information Processing Systems Foundation. <a href=\"https://doi.org/10.52202/085713-1551\">https://doi.org/10.52202/085713-1551</a>","ista":"Rodionov G, Garipov R, Shutova A, Yakushev G, Schultheis E, Egiazarian V, Sinitsin A, Kuznedelev D, Alistarh D-A. 2025. Hogwild! Inference: Parallel LLM generation via concurrent attention. 39th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 38, 52014–52055.","ama":"Rodionov G, Garipov R, Shutova A, et al. Hogwild! Inference: Parallel LLM generation via concurrent attention. In: <i>39th Conference on Neural Information Processing Systems</i>. Vol 38. Neural Information Processing Systems Foundation; 2025:52014-52055. doi:<a href=\"https://doi.org/10.52202/085713-1551\">10.52202/085713-1551</a>","short":"G. Rodionov, R. Garipov, A. Shutova, G. Yakushev, E. Schultheis, V. Egiazarian, A. Sinitsin, D. Kuznedelev, D.-A. Alistarh, in:, 39th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2025, pp. 52014–52055.","mla":"Rodionov, Gleb, et al. “Hogwild! Inference: Parallel LLM Generation via Concurrent Attention.” <i>39th Conference on Neural Information Processing Systems</i>, vol. 38, Neural Information Processing Systems Foundation, 2025, pp. 52014–55, doi:<a href=\"https://doi.org/10.52202/085713-1551\">10.52202/085713-1551</a>.","chicago":"Rodionov, Gleb, Roman Garipov, Alina Shutova, George Yakushev, Erik Schultheis, Vage Egiazarian, Anton Sinitsin, Denis Kuznedelev, and Dan-Adrian Alistarh. “Hogwild! Inference: Parallel LLM Generation via Concurrent Attention.” In <i>39th Conference on Neural Information Processing Systems</i>, 38:52014–55. Neural Information Processing Systems Foundation, 2025. <a href=\"https://doi.org/10.52202/085713-1551\">https://doi.org/10.52202/085713-1551</a>."},"_id":"22826","status":"public","researchdata_availability":"no","oa_version":"Published Version","OA_place":"publisher","OA_type":"gold","has_accepted_license":"1","date_published":"2025-12-02T00:00:00Z","ddc":["000"],"fulldoi":"https://doi.org/10.52202/085713-1551","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","abstract":[{"text":"Large Language Models (LLMs) have demonstrated the ability to tackle increasingly complex tasks through advanced reasoning, long-form content generation,\r\nand tool use. Solving these tasks often involves long inference-time computations.\r\nIn human problem solving, a common strategy to expedite work is collaboration: by\r\ndividing the problem into sub-tasks, exploring different strategies concurrently, etc.\r\nRecent research has shown that LLMs can also operate in parallel by implementing\r\nexplicit cooperation frameworks, such as voting mechanisms or the explicit creation of independent sub-tasks that can be executed in parallel. However, each of\r\nthese frameworks may not be suitable for all types of tasks, which can hinder their\r\napplicability. In this work, we propose a different design approach: we run LLM\r\n“workers” in parallel , allowing them to synchronize via a concurrently-updated\r\nattention cache and prompt these workers to decide how best to collaborate. Our\r\napproach allows the LLM instances to come up with their own collaboration strategy for the problem at hand, all the while “seeing” each other’s memory in the\r\nconcurrent KV cache. We implement this approach via Hogwild! Inference: a\r\nparallel LLM inference engine where multiple instances of the same LLM run in\r\nparallel with the same attention cache, with “instant” access to each other’s memory.1 Hogwild! Inference takes advantage of Rotary Position Embeddings (RoPE)\r\nto avoid recomputation while improving parallel hardware utilization. We find that\r\nmodern reasoning-capable LLMs can perform inference with shared Key-Value\r\ncache out of the box, without additional fine-tuning.","lang":"eng"}],"date_updated":"2026-09-10T07:06:31Z","alternative_title":["Advances in Neural Information Processing Systems"]},{"intvolume":"        38","external_id":{"arxiv":["2504.20039"]},"oa_version":"Published Version","status":"public","_id":"22830","researchdata_availability":"no","year":"2025","citation":{"mla":"Garipov, Roman, et al. “AutoJudge: Judge Decoding without Manual Annotation.” <i>39th Annual Conference on Neural Information Processing Systems</i>, vol. 38, Neural Information Processing Systems Foundation, 2025, pp. 104904–41, doi:<a href=\"https://doi.org/10.52202/085713-3163\">10.52202/085713-3163</a>.","short":"R. Garipov, F. Velikonivtsev, I. Ermakov, R. Svirschevski, V. Egiazarian, M. Ryabinin, in:, 39th Annual Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2025, pp. 104904–104941.","chicago":"Garipov, Roman, Fedor Velikonivtsev, Ivan Ermakov, Ruslan Svirschevski, Vage Egiazarian, and Max Ryabinin. “AutoJudge: Judge Decoding without Manual Annotation.” In <i>39th Annual Conference on Neural Information Processing Systems</i>, 38:104904–41. Neural Information Processing Systems Foundation, 2025. <a href=\"https://doi.org/10.52202/085713-3163\">https://doi.org/10.52202/085713-3163</a>.","apa":"Garipov, R., Velikonivtsev, F., Ermakov, I., Svirschevski, R., Egiazarian, V., &#38; Ryabinin, M. (2025). AutoJudge: Judge decoding without manual annotation. In <i>39th Annual Conference on Neural Information Processing Systems</i> (Vol. 38, pp. 104904–104941). San Diego, CA, United States: Neural Information Processing Systems Foundation. <a href=\"https://doi.org/10.52202/085713-3163\">https://doi.org/10.52202/085713-3163</a>","ieee":"R. Garipov, F. Velikonivtsev, I. Ermakov, R. Svirschevski, V. Egiazarian, and M. Ryabinin, “AutoJudge: Judge decoding without manual annotation,” in <i>39th Annual Conference on Neural Information Processing Systems</i>, San Diego, CA, United States, 2025, vol. 38, pp. 104904–104941.","ista":"Garipov R, Velikonivtsev F, Ermakov I, Svirschevski R, Egiazarian V, Ryabinin M. 2025. AutoJudge: Judge decoding without manual annotation. 39th Annual Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems, Advances in Neural Information Processing Systems, vol. 38, 104904–104941.","ama":"Garipov R, Velikonivtsev F, Ermakov I, Svirschevski R, Egiazarian V, Ryabinin M. AutoJudge: Judge decoding without manual annotation. In: <i>39th Annual Conference on Neural Information Processing Systems</i>. Vol 38. Neural Information Processing Systems Foundation; 2025:104904-104941. doi:<a href=\"https://doi.org/10.52202/085713-3163\">10.52202/085713-3163</a>"},"acknowledgement":"We would like to express our sincere gratitude to Denis Mazur for his valuable contributions to the\r\nimplementation of API calls used in Algorithm 1 and for supporting inference with the Llama 405B\r\nmodel. We are also thankful for his positive influence on the overall atmosphere and team morale\r\nthroughout the course of this project.","publisher":"Neural Information Processing Systems Foundation","arxiv":1,"day":"02","das_tickbox":"0","fulldoi":"https://doi.org/10.52202/085713-3163","date_published":"2025-12-02T00:00:00Z","OA_type":"free access","OA_place":"publisher","date_updated":"2026-09-10T11:08:33Z","alternative_title":["Advances in Neural Information Processing Systems"],"abstract":[{"text":"We introduce AutoJudge, a method that accelerates large language model (LLM) inference with task-specific lossy speculative decoding. Instead of matching the original model output distribution token-by-token, we identify the generated tokens that affect the downstream quality of the response, relaxing the distribution match guarantee so that the \"unimportant\" tokens can be generated faster. Our approach relies on a semi‑greedy search algorithm to test which of the mismatches between target and draft models should be corrected to preserve quality and which ones may be skipped. We then train a lightweight classifier based on existing LLM embeddings to predict, at inference time, which mismatching tokens can be safely accepted without compromising the final answer quality. We evaluate AutoJudge with multiple draft/target model pairs on mathematical reasoning and programming benchmarks, achieving significant speedups at the cost of a minor accuracy reduction. Notably, on GSM8K with the Llama 3.1 70B target model, our approach achieves up to \r\n≈\r\n2\r\n×\r\n speedup \\textit{over speculative decoding} at the cost of a \r\n≤\r\n1\r\n%\r\n drop in accuracy. When applied to the LiveCodeBench benchmark, AutoJudge automatically detects programming-specific important tokens, accepting \r\n≥\r\n25\r\n tokens per speculation cycle at a\r\n \r\n2\r\n%\r\n drop in Pass@1. Our approach requires no human annotation and is easy to integrate with modern LLM inference frameworks.","lang":"eng"}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","supplementarymaterial":"yes","date_created":"2026-09-06T22:02:00Z","article_processing_charge":"No","quality_controlled":"1","language":[{"iso":"eng"}],"department":[{"_id":"DaAl"}],"volume":38,"publication_status":"published","page":"104904-104941","month":"12","main_file_link":[{"open_access":"1","url":"https://doi.org/10.52202/085713-3163"}],"doi":"10.52202/085713-3163","type":"conference","publication_identifier":{"isbn":["9798331338275"],"issn":["1049-5258"]},"title":"AutoJudge: Judge decoding without manual annotation","oa":1,"scopus_import":"1","conference":{"start_date":"2025-12-02","location":"San Diego, CA, United States","end_date":"2025-12-07","name":"NeurIPS: Neural Information Processing Systems"},"author":[{"first_name":"Roman","full_name":"Garipov, Roman","last_name":"Garipov"},{"last_name":"Velikonivtsev","full_name":"Velikonivtsev, Fedor","first_name":"Fedor"},{"full_name":"Ermakov, Ivan","last_name":"Ermakov","first_name":"Ivan"},{"last_name":"Svirschevski","full_name":"Svirschevski, Ruslan","first_name":"Ruslan"},{"first_name":"Vage","full_name":"Egiazarian, Vage","id":"77451e76-92b2-11ef-a4d1-8dbaa06e16ad","last_name":"Egiazarian"},{"first_name":"Max","full_name":"Ryabinin, Max","last_name":"Ryabinin"}],"publication":"39th Annual Conference on Neural Information Processing Systems"}]