@inproceedings{18070,
  abstract     = {Parallel SGD in a shared-memory setting is oft-represented by the popular Hogwild! algorithm, in which lock-free updates are asynchronously performed by multiple computing processes. Unfortunately, scaling Hogwild! to distributed workers is largely unexplored. Specifically, it is unknown if any adaptation of Hogwild! to the popular decentralized multi-GPU setting offers any competitive speedup, either empirically or theoretically. In this work, we investigate the potential of decentralizing Hogwild! by incorporating simultaneously (a) asynchronous local gradient updates on the shared memory of GPUs, and (b) non-blocking asynchronous decentralized federated averaging. A naive direct implementation shows degradation in performance, arising from scheduling overheads and concurrent write conflicts on GPUs. To mitigate these drawbacks, we investigate and propose a new method, based on careful block selection rules, which update only portions of the parameter vectors. Our experiments show that the resulting decentralized training method exhibits improved throughput and competitive accuracy for standard image classification benchmarks on the CIFAR-10, CIFAR-100, and Imagenet datasets. On the theoretical side, we prove that our method guarantees sublinear ergodic convergence rates for non-convex objectives.},
  author       = {Chatterjee, Bapi and Kungurtsev, Vyacheslav and Alistarh, Dan-Adrian},
  booktitle    = {Proceedings of the 44th International Conference on Distributed Computing Systems},
  isbn         = {9798350386059},
  issn         = {2575-8411},
  location     = {Jersey City, NJ, United States},
  pages        = {857--868},
  publisher    = {IEEE},
  title        = {{Federated SGD with local asynchrony}},
  doi          = {10.1109/ICDCS60910.2024.00084},
  year         = {2024},
}

@inproceedings{18071,
  abstract     = {Recent advancements on DAG-based consensus protocols allow for blockchains with improved metrics and properties, such as throughput and censorship-resistance. Variants of the Bullshark [18] consensus protocol are adopted for practical use by the Sui blockchain, for improved latency. However, the protocol is leader-based, and is strongly affected by crashed leaders that can lead to various performance issues, for example, decreased transaction throughput. In this paper, we propose HammerHead, a DAG-based consensus protocol, that is inspired by Carousel [8] and provides Leader-Utilization. Our proposal differs from Carousel, which is built for a chained consensus protocol; in HammerHead chain quality is inherited by the DAG. HammerHead needs to preserve safety and liveness, despite validators committing leader vertices asynchronously. The key idea is to update leader schedules dynamically, based on the validators' scores during the previous schedule. We implement HammerHead and show a minor improvement in performance for cases without faults. The major improvements in comparison to Bullshark appear in faulty settings. Specifically, we show a drastic, 2x-latency improvement and up to 40% increased throughput when crash faults occur (100 validators, 33 faults).},
  author       = {Tsimos, Giorgos and Kichidis, Anastasios and Sonnino, Alberto and Kokoris Kogias, Eleftherios},
  booktitle    = {Proceedings - International Conference on Distributed Computing Systems},
  isbn         = {9798350386059},
  issn         = {2575-8411},
  location     = {Jersey City, NJ, United States},
  pages        = {1377--1387},
  publisher    = {IEEE},
  title        = {{HammerHead: Leader reputation for dynamic scheduling}},
  doi          = {10.1109/ICDCS60910.2024.00129},
  year         = {2024},
}