@inproceedings{22826,
  abstract     = {Large Language Models (LLMs) have demonstrated the ability to tackle increasingly complex tasks through advanced reasoning, long-form content generation,
and tool use. Solving these tasks often involves long inference-time computations.
In human problem solving, a common strategy to expedite work is collaboration: by
dividing the problem into sub-tasks, exploring different strategies concurrently, etc.
Recent research has shown that LLMs can also operate in parallel by implementing
explicit cooperation frameworks, such as voting mechanisms or the explicit creation of independent sub-tasks that can be executed in parallel. However, each of
these frameworks may not be suitable for all types of tasks, which can hinder their
applicability. In this work, we propose a different design approach: we run LLM
“workers” in parallel , allowing them to synchronize via a concurrently-updated
attention cache and prompt these workers to decide how best to collaborate. Our
approach allows the LLM instances to come up with their own collaboration strategy for the problem at hand, all the while “seeing” each other’s memory in the
concurrent KV cache. We implement this approach via Hogwild! Inference: a
parallel LLM inference engine where multiple instances of the same LLM run in
parallel with the same attention cache, with “instant” access to each other’s memory.1 Hogwild! Inference takes advantage of Rotary Position Embeddings (RoPE)
to avoid recomputation while improving parallel hardware utilization. We find that
modern reasoning-capable LLMs can perform inference with shared Key-Value
cache out of the box, without additional fine-tuning.},
  author       = {Rodionov, Gleb and Garipov, Roman and Shutova, Alina and Yakushev, George and Schultheis, Erik and Egiazarian, Vage and Sinitsin, Anton and Kuznedelev, Denis and Alistarh, Dan-Adrian},
  booktitle    = {39th Conference on Neural Information Processing Systems},
  isbn         = {9798331338275},
  issn         = {1049-5258},
  location     = {San Diego, CA, United States},
  pages        = {52014--52055},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Hogwild! Inference: Parallel LLM generation via concurrent attention}},
  doi          = {10.52202/085713-1551},
  volume       = {38},
  year         = {2025},
}