@inproceedings{20684,
  abstract     = {Quantization is a powerful tool for accelerating large language model (LLM) inference, but the accuracy-performance trade-offs across different formats remain unclear. In this paper, we conduct the most comprehensive empirical study to date, evaluating FP8, INT8, and INT4
quantization across academic benchmarks and real-world tasks on the entire Llama-3.1 model
family. Through over 500,000 evaluations, our investigation yields several key findings: (1) FP8 (W8A8-FP) is effectively lossless across all model scales, (2) well-tuned INT8 (W8A8-INT) achieves surprisingly low (1-3%) accuracy degradation, and (3) INT4 weightonly (W4A16-INT) is more competitive than expected, rivaling 8-bit quantization. Further, we investigate the optimal quantization format for different deployments by analyzing inference performance through the popular vLLM framework. Our analysis provides clear deployment recommendations: W4A16 is the most cost-efficient for synchronous setups, while W8A8 dominates in asynchronous
continuous batching. For mixed workloads, the optimal choice depends on the specific use
case. Our findings offer practical, data-driven guidelines for deploying quantized LLMs at scale—ensuring the best balance between speed, efficiency, and accuracy. },
  author       = {Kurtic, Eldar and Marques, Alexandre and Pandit, Shubhra and Kurtz, Mark and Alistarh, Dan-Adrian},
  booktitle    = {Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics},
  isbn         = {9798891762510},
  issn         = {0736-587X},
  location     = {Vienna, Austria},
  pages        = {26872--26886},
  publisher    = {Association for Computational Linguistics},
  title        = {{“Give me BF16 or give me death”? Accuracy-performance trade-offs in LLM quantization}},
  year         = {2025},
}