@article{19713,
  abstract     = {Distributed optimization is the standard way of speeding up machine learning training, and most of the research in the area focuses on distributed first-order, gradient-based methods. Yet, there are settings where some computationally-bounded nodes may not be able to implement first-order, gradient-based optimization, while they could still contribute to joint optimization tasks. In this paper, we initiate the study of hybrid decentralized optimization, studying settings where nodes with zeroth-order and first-order optimization capabilities co-exist in a distributed system, and attempt to jointly solve an optimization task over some data distribution. We essentially show that, under reasonable parameter settings, such a system can not only withstand noisier zeroth-order agents but can even benefit from integrating such agents into the optimization process, rather than ignoring their information. At the core of our approach is a new analysis of distributed optimization with noisy and possibly-biased gradient estimators, which may be of independent interest. Our results hold for both convex and non-convex objectives. Experimental results on standard optimization tasks confirm our analysis, showing that hybrid first-zeroth order optimization can be practical, even when training deep neural networks.},
  author       = {Talaei, Shayan and Ansaripour, Matin and Nadiradze, Giorgi and Alistarh, Dan-Adrian},
  issn         = {2374-3468},
  journal      = {Proceedings of the 39th AAAI Conference on Artificial Intelligence},
  number       = {19},
  pages        = {20778--20786},
  publisher    = {Association for the Advancement of Artificial Intelligence},
  title        = {{Hybrid decentralized optimization: Leveraging both first- and zeroth-order optimizers for faster convergence}},
  doi          = {10.1609/aaai.v39i19.34290},
  volume       = {39},
  year         = {2025},
}