[{"oa":1,"main_file_link":[{"url":"https://proceedings.mlr.press/v234/kurtic24a","open_access":"1"}],"external_id":{"arxiv":["2312.13547"]},"quality_controlled":"1","conference":{"name":"CPAL: Conference on Parsimony and Learning","start_date":"2024-01-03","location":"Hongkong, China","end_date":"2024-01-06"},"language":[{"iso":"eng"}],"month":"01","publication_identifier":{"eissn":["2640-3498"]},"year":"2024","publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"ML Research Press","author":[{"full_name":"Kurtic, Eldar","id":"47beb3a5-07b5-11eb-9b87-b108ec578218","last_name":"Kurtic","first_name":"Eldar"},{"full_name":"Hoefler, Torsten","last_name":"Hoefler","first_name":"Torsten"},{"first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","full_name":"Alistarh, Dan-Adrian"}],"date_updated":"2024-02-26T10:30:52Z","date_created":"2024-02-18T23:01:03Z","volume":234,"publication":"Proceedings of Machine Learning Research","citation":{"short":"E. Kurtic, T. Hoefler, D.-A. Alistarh, in:, Proceedings of Machine Learning Research, ML Research Press, 2024, pp. 542–553.","mla":"Kurtic, Eldar, et al. “How to Prune Your Language Model: Recovering Accuracy on the ‘Sparsity May Cry’ Benchmark.” Proceedings of Machine Learning Research, vol. 234, ML Research Press, 2024, pp. 542–53.","chicago":"Kurtic, Eldar, Torsten Hoefler, and Dan-Adrian Alistarh. “How to Prune Your Language Model: Recovering Accuracy on the ‘Sparsity May Cry’ Benchmark.” In Proceedings of Machine Learning Research, 234:542–53. ML Research Press, 2024.","ama":"Kurtic E, Hoefler T, Alistarh D-A. How to prune your language model: Recovering accuracy on the “Sparsity May Cry” benchmark. In: Proceedings of Machine Learning Research. Vol 234. ML Research Press; 2024:542-553.","ieee":"E. Kurtic, T. Hoefler, and D.-A. Alistarh, “How to prune your language model: Recovering accuracy on the ‘Sparsity May Cry’ benchmark,” in Proceedings of Machine Learning Research, Hongkong, China, 2024, vol. 234, pp. 542–553.","apa":"Kurtic, E., Hoefler, T., & Alistarh, D.-A. (2024). How to prune your language model: Recovering accuracy on the “Sparsity May Cry” benchmark. In Proceedings of Machine Learning Research (Vol. 234, pp. 542–553). Hongkong, China: ML Research Press.","ista":"Kurtic E, Hoefler T, Alistarh D-A. 2024. How to prune your language model: Recovering accuracy on the ‘Sparsity May Cry’ benchmark. Proceedings of Machine Learning Research. CPAL: Conference on Parsimony and Learning, PMLR, vol. 234, 542–553."},"page":"542-553","date_published":"2024-01-08T00:00:00Z","scopus_import":"1","day":"08","article_processing_charge":"No","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"15011","status":"public","title":"How to prune your language model: Recovering accuracy on the \"Sparsity May Cry\" benchmark","intvolume":" 234","oa_version":"Preprint","type":"conference","alternative_title":["PMLR"],"abstract":[{"lang":"eng","text":"Pruning large language models (LLMs) from the BERT family has emerged as a standard compression benchmark, and several pruning methods have been proposed for this task. The recent “Sparsity May Cry” (SMC) benchmark put into question the validity of all existing methods, exhibiting a more complex setup where many known pruning methods appear to fail. We revisit the question of accurate BERT-pruning during fine-tuning on downstream datasets, and propose a set of general guidelines for successful pruning, even on the challenging SMC benchmark. First, we perform a cost-vs-benefits analysis of pruning model components, such as the embeddings and the classification head; second, we provide a simple-yet-general way of scaling training, sparsification and learning rate schedules relative to the desired target sparsity; finally, we investigate the importance of proper parametrization for Knowledge Distillation in the context of LLMs. Our simple insights lead to state-of-the-art results, both on classic BERT-pruning benchmarks, as well as on the SMC benchmark, showing that even classic gradual magnitude pruning (GMP) can yield competitive results, with the right approach."}]},{"type":"conference","abstract":[{"text":"Asynchronous programming has gained significant popularity over the last decade: support for this programming pattern is available in many popular languages via libraries and native language implementations, typically in the form of coroutines or the async/await construct. Instead of programming via shared memory, this concept assumes implicit synchronization through message passing. The key data structure enabling such communication is the rendezvous channel. Roughly, a rendezvous channel is a blocking queue of size zero, so both send(e) and receive() operations wait for each other, performing a rendezvous when they meet. To optimize the message passing pattern, channels are usually equipped with a fixed-size buffer, so sends do not suspend and put elements into the buffer until its capacity is exceeded. This primitive is known as a buffered channel.\r\n\r\nThis paper presents a fast and scalable algorithm for both rendezvous and buffered channels. Similarly to modern queues, our solution is based on an infinite array with two positional counters for send(e) and receive() operations, leveraging the unconditional Fetch-And-Add instruction to update them. Yet, the algorithm requires non-trivial modifications of this classic pattern, in order to support the full channel semantics, such as buffering and cancellation of waiting requests. We compare the performance of our solution to that of the Kotlin implementation, as well as against other academic proposals, showing up to 9.8× speedup. To showcase its expressiveness and performance, we also integrated the proposed algorithm into the standard Kotlin Coroutines library, replacing the previous channel implementations.","lang":"eng"}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"12735","year":"2023","publication_status":"published","title":"Fast and scalable channels in Kotlin Coroutines","status":"public","publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"author":[{"full_name":"Koval, Nikita","id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87","first_name":"Nikita","last_name":"Koval"},{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X"},{"last_name":"Elizarov","first_name":"Roman","full_name":"Elizarov, Roman"}],"date_created":"2023-03-19T23:00:58Z","date_updated":"2023-03-20T07:29:28Z","oa_version":"Preprint","scopus_import":"1","month":"02","day":"25","article_processing_charge":"No","publication_identifier":{"isbn":["9798400700156"]},"publication":"Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming","oa":1,"citation":{"chicago":"Koval, Nikita, Dan-Adrian Alistarh, and Roman Elizarov. “Fast and Scalable Channels in Kotlin Coroutines.” In Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, 107–18. Association for Computing Machinery, 2023. https://doi.org/10.1145/3572848.3577481.","mla":"Koval, Nikita, et al. “Fast and Scalable Channels in Kotlin Coroutines.” Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2023, pp. 107–18, doi:10.1145/3572848.3577481.","short":"N. Koval, D.-A. Alistarh, R. Elizarov, in:, Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2023, pp. 107–118.","ista":"Koval N, Alistarh D-A, Elizarov R. 2023. Fast and scalable channels in Kotlin Coroutines. Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. PPoPP: Sympopsium on Principles and Practice of Parallel Programming, 107–118.","apa":"Koval, N., Alistarh, D.-A., & Elizarov, R. (2023). Fast and scalable channels in Kotlin Coroutines. In Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (pp. 107–118). Montreal, QC, Canada: Association for Computing Machinery. https://doi.org/10.1145/3572848.3577481","ieee":"N. Koval, D.-A. Alistarh, and R. Elizarov, “Fast and scalable channels in Kotlin Coroutines,” in Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Montreal, QC, Canada, 2023, pp. 107–118.","ama":"Koval N, Alistarh D-A, Elizarov R. Fast and scalable channels in Kotlin Coroutines. In: Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. Association for Computing Machinery; 2023:107-118. doi:10.1145/3572848.3577481"},"external_id":{"arxiv":["2211.04986"]},"main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2211.04986"}],"quality_controlled":"1","page":"107-118","conference":{"name":"PPoPP: Sympopsium on Principles and Practice of Parallel Programming","end_date":"2023-03-01","location":"Montreal, QC, Canada","start_date":"2023-02-25"},"date_published":"2023-02-25T00:00:00Z","doi":"10.1145/3572848.3577481","language":[{"iso":"eng"}]},{"oa_version":"Published Version","date_updated":"2023-03-20T07:57:27Z","date_created":"2023-03-19T23:00:58Z","author":[{"full_name":"Aksenov, Vitaly","last_name":"Aksenov","first_name":"Vitaly"},{"full_name":"Brown, Trevor A","first_name":"Trevor A","last_name":"Brown","id":"3569F0A0-F248-11E8-B48F-1D18A9856A87"},{"full_name":"Fedorov, Alexander","id":"2e711909-896a-11ed-bdf8-eb0f5a2984c6","last_name":"Fedorov","first_name":"Alexander"},{"full_name":"Kokorin, Ilya","first_name":"Ilya","last_name":"Kokorin"}],"department":[{"_id":"DaAl"},{"_id":"GradSch"}],"publisher":"Association for Computing Machinery","status":"public","publication_status":"published","title":"Unexpected scaling in path copying trees","_id":"12736","year":"2023","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","acknowledgement":"This work was supported by: the Natural Sciences and Engineering Research Council of Canada (NSERC) Discovery Program grant: RGPIN-2019-04227, and the Canada Foundation for Innovation John R. Evans Leaders Fund (CFI-JELF) with equal support from the Ontario Research Fund CFI Leaders Opportunity Fund: 38512.","abstract":[{"text":"Although a wide variety of handcrafted concurrent data structures have been proposed, there is considerable interest in universal approaches (Universal Constructions or UCs) for building concurrent data structures. UCs (semi-)automatically convert a sequential data structure into a concurrent one. The simplest approach uses locks [3, 6] that protect a sequential data structure and allow only one process to access it at a time. However, the resulting data structure is blocking. Most work on UCs instead focuses on obtaining non-blocking progress guarantees such as obstruction-freedom, lock-freedom or wait-freedom. Many non-blocking UCs have appeared. Key examples include the seminal wait-free UC [2] by Herlihy, a NUMA-aware UC [10] by Yi et al., and an efficient UC for large objects [1] by Fatourou et al.","lang":"eng"}],"type":"conference_poster","language":[{"iso":"eng"}],"doi":"10.1145/3572848.3577512","date_published":"2023-02-25T00:00:00Z","conference":{"start_date":"2023-02-25","location":"Montreal, QB, Canada","end_date":"2023-03-01","name":"PPoPP: Sympopsium on Principles and Practice of Parallel Programming"},"page":"438-440","quality_controlled":"1","oa":1,"main_file_link":[{"url":"https://doi.org/10.1145/3572848.3577512","open_access":"1"}],"citation":{"chicago":"Aksenov, Vitaly, Trevor A Brown, Alexander Fedorov, and Ilya Kokorin. Unexpected Scaling in Path Copying Trees. Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. Association for Computing Machinery, 2023. https://doi.org/10.1145/3572848.3577512.","mla":"Aksenov, Vitaly, et al. “Unexpected Scaling in Path Copying Trees.” Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2023, pp. 438–40, doi:10.1145/3572848.3577512.","short":"V. Aksenov, T.A. Brown, A. Fedorov, I. Kokorin, Unexpected Scaling in Path Copying Trees, Association for Computing Machinery, 2023.","ista":"Aksenov V, Brown TA, Fedorov A, Kokorin I. 2023. Unexpected scaling in path copying trees, Association for Computing Machinery,p.","apa":"Aksenov, V., Brown, T. A., Fedorov, A., & Kokorin, I. (2023). Unexpected scaling in path copying trees. Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (pp. 438–440). Montreal, QB, Canada: Association for Computing Machinery. https://doi.org/10.1145/3572848.3577512","ieee":"V. Aksenov, T. A. Brown, A. Fedorov, and I. Kokorin, Unexpected scaling in path copying trees. Association for Computing Machinery, 2023, pp. 438–440.","ama":"Aksenov V, Brown TA, Fedorov A, Kokorin I. Unexpected Scaling in Path Copying Trees. Association for Computing Machinery; 2023:438-440. doi:10.1145/3572848.3577512"},"publication":"Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming","publication_identifier":{"isbn":["9798400700156"]},"article_processing_charge":"No","day":"25","month":"02"},{"project":[{"_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning"}],"quality_controlled":"1","oa":1,"external_id":{"arxiv":["2207.14200"]},"main_file_link":[{"url":"https://openreview.net/pdf?id=_eTZBs-yedr","open_access":"1"}],"citation":{"short":"E.-A. Peste, A. Vladu, E. Kurtic, C. Lampert, D.-A. Alistarh, in:, 11th International Conference on Learning Representations , n.d.","mla":"Peste, Elena-Alexandra, et al. “CrAM: A Compression-Aware Minimizer.” 11th International Conference on Learning Representations .","chicago":"Peste, Elena-Alexandra, Adrian Vladu, Eldar Kurtic, Christoph Lampert, and Dan-Adrian Alistarh. “CrAM: A Compression-Aware Minimizer.” In 11th International Conference on Learning Representations , n.d.","ama":"Peste E-A, Vladu A, Kurtic E, Lampert C, Alistarh D-A. CrAM: A Compression-Aware Minimizer. In: 11th International Conference on Learning Representations .","ieee":"E.-A. Peste, A. Vladu, E. Kurtic, C. Lampert, and D.-A. Alistarh, “CrAM: A Compression-Aware Minimizer,” in 11th International Conference on Learning Representations , Kigali, Rwanda .","apa":"Peste, E.-A., Vladu, A., Kurtic, E., Lampert, C., & Alistarh, D.-A. (n.d.). CrAM: A Compression-Aware Minimizer. In 11th International Conference on Learning Representations . Kigali, Rwanda .","ista":"Peste E-A, Vladu A, Kurtic E, Lampert C, Alistarh D-A. CrAM: A Compression-Aware Minimizer. 11th International Conference on Learning Representations . ICLR: International Conference on Learning Representations."},"publication":"11th International Conference on Learning Representations ","language":[{"iso":"eng"}],"acknowledged_ssus":[{"_id":"ScienComp"}],"date_published":"2023-05-01T00:00:00Z","conference":{"location":"Kigali, Rwanda ","start_date":"2023-05-01","end_date":"2023-05-05","name":"ICLR: International Conference on Learning Representations"},"article_processing_charge":"No","month":"05","department":[{"_id":"GradSch"},{"_id":"DaAl"},{"_id":"ChLa"}],"title":"CrAM: A Compression-Aware Minimizer","status":"public","publication_status":"accepted","year":"2023","_id":"13053","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","acknowledgement":"AP, EK, DA received funding from the European Research Council (ERC) under the European\r\nUnion’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). AV acknowledges the support of the French Agence Nationale de la Recherche (ANR), under grant ANR-21-CE48-0016 (project COMCOPT). We further acknowledge the support from the Scientific Service Units (SSU) of ISTA through resources provided by Scientific Computing (SciComp)-","oa_version":"Preprint","date_updated":"2023-06-01T12:54:45Z","date_created":"2023-05-23T11:36:18Z","related_material":{"record":[{"status":"public","relation":"dissertation_contains","id":"13074"}]},"author":[{"last_name":"Peste","first_name":"Elena-Alexandra","id":"32D78294-F248-11E8-B48F-1D18A9856A87","full_name":"Peste, Elena-Alexandra"},{"first_name":"Adrian","last_name":"Vladu","full_name":"Vladu, Adrian"},{"last_name":"Kurtic","first_name":"Eldar","id":"47beb3a5-07b5-11eb-9b87-b108ec578218","full_name":"Kurtic, Eldar"},{"full_name":"Lampert, Christoph","id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0001-8622-7887","first_name":"Christoph","last_name":"Lampert"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"type":"conference","ec_funded":1,"abstract":[{"lang":"eng","text":"Deep neural networks (DNNs) often have to be compressed, via pruning and/or quantization, before they can be deployed in practical settings. In this work we propose a new compression-aware minimizer dubbed CrAM that modifies the optimization step in a principled way, in order to produce models whose local loss behavior is stable under compression operations such as pruning. Thus, dense models trained via CrAM should be compressible post-training, in a single step, without significant accuracy loss. Experimental results on standard benchmarks, such as residual networks for ImageNet classification and BERT models for language modelling, show that CrAM produces dense models that can be more accurate than the standard SGD/Adam-based baselines, but which are stable under weight pruning: specifically, we can prune models in one-shot to 70-80% sparsity with almost no accuracy loss, and to 90% with reasonable (∼1%) accuracy loss, which is competitive with gradual compression methods. Additionally, CrAM can produce sparse models which perform well for transfer learning, and it also works for semi-structured 2:4 pruning patterns supported by GPU hardware. The code for reproducing the results is available at this https URL ."}]},{"date_published":"2023-06-06T00:00:00Z","article_type":"original","publication":"Proceedings of the ACM on Programming Languages","citation":{"chicago":"Koval, Nikita, Dmitry Khalanskiy, and Dan-Adrian Alistarh. “CQS: A Formally-Verified Framework for Fair and Abortable Synchronization.” Proceedings of the ACM on Programming Languages. Association for Computing Machinery , 2023. https://doi.org/10.1145/3591230.","mla":"Koval, Nikita, et al. “CQS: A Formally-Verified Framework for Fair and Abortable Synchronization.” Proceedings of the ACM on Programming Languages, vol. 7, 116, Association for Computing Machinery , 2023, doi:10.1145/3591230.","short":"N. Koval, D. Khalanskiy, D.-A. Alistarh, Proceedings of the ACM on Programming Languages 7 (2023).","ista":"Koval N, Khalanskiy D, Alistarh D-A. 2023. CQS: A formally-verified framework for fair and abortable synchronization. Proceedings of the ACM on Programming Languages. 7, 116.","apa":"Koval, N., Khalanskiy, D., & Alistarh, D.-A. (2023). CQS: A formally-verified framework for fair and abortable synchronization. Proceedings of the ACM on Programming Languages. Association for Computing Machinery . https://doi.org/10.1145/3591230","ieee":"N. Koval, D. Khalanskiy, and D.-A. Alistarh, “CQS: A formally-verified framework for fair and abortable synchronization,” Proceedings of the ACM on Programming Languages, vol. 7. Association for Computing Machinery , 2023.","ama":"Koval N, Khalanskiy D, Alistarh D-A. CQS: A formally-verified framework for fair and abortable synchronization. Proceedings of the ACM on Programming Languages. 2023;7. doi:10.1145/3591230"},"day":"06","has_accepted_license":"1","article_processing_charge":"No","scopus_import":"1","file":[{"file_id":"13187","relation":"main_file","success":1,"checksum":"5dba6e73f0ed79adbdae14d165bc2f68","date_created":"2023-07-03T13:09:39Z","date_updated":"2023-07-03T13:09:39Z","access_level":"open_access","file_name":"2023_ACMProgram.Lang._Koval.pdf","creator":"alisjak","file_size":1266773,"content_type":"application/pdf"}],"oa_version":"Published Version","ddc":["000"],"title":"CQS: A formally-verified framework for fair and abortable synchronization","status":"public","intvolume":" 7","_id":"13179","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","abstract":[{"lang":"eng","text":"Writing concurrent code that is both correct and efficient is notoriously difficult. Thus, programmers often prefer to use synchronization abstractions, which render code simpler and easier to reason about. Despite a wealth of work on this topic, there is still a gap between the rich semantics provided by synchronization abstractions in modern programming languages—specifically, fair FIFO ordering of synchronization requests and support for abortable operations—and frameworks for implementing it correctly and efficiently. Supporting such semantics is critical given the rising popularity of constructs for asynchronous programming, such as coroutines, which abort frequently and are cheaper to suspend and resume compared to native threads.\r\n\r\nThis paper introduces a new framework called CancellableQueueSynchronizer (CQS), which enables simple yet efficient implementations of a wide range of fair and abortable synchronization primitives: mutexes, semaphores, barriers, count-down latches, and blocking pools. Our main contribution is algorithmic, as implementing both fairness and abortability efficiently at this level of generality is non-trivial. Importantly, all our algorithms, including the CQS framework and the primitives built on top of it, come with formal proofs in the Iris framework for Coq for many of their properties. These proofs are modular, so it is easy to show correctness for new primitives implemented on top of CQS. From a practical perspective, implementation of CQS for native threads on the JVM improves throughput by up to two orders of magnitude over Java’s AbstractQueuedSynchronizer, the only practical abstraction offering similar semantics. Further, we successfully integrated CQS as a core component of the popular Kotlin Coroutines library, validating the framework’s practical impact and expressiveness in a real-world environment. In sum, CancellableQueueSynchronizer is the first framework to combine expressiveness with formal guarantees and solid practical performance. Our approach should be extensible to other languages and families of synchronization primitives."}],"type":"journal_article","language":[{"iso":"eng"}],"doi":"10.1145/3591230","quality_controlled":"1","oa":1,"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"month":"06","publication_identifier":{"eissn":["2475-1421"]},"date_updated":"2023-07-17T08:43:19Z","date_created":"2023-07-02T22:00:43Z","volume":7,"author":[{"full_name":"Koval, Nikita","first_name":"Nikita","last_name":"Koval","id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87"},{"last_name":"Khalanskiy","first_name":"Dmitry","full_name":"Khalanskiy, Dmitry"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"}],"publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"Association for Computing Machinery ","year":"2023","license":"https://creativecommons.org/licenses/by/4.0/","file_date_updated":"2023-07-03T13:09:39Z","article_number":"116"},{"oa_version":"Published Version","file":[{"file_size":2087937,"content_type":"application/pdf","creator":"dernst","file_name":"2023_SPAA_Fedorov.pdf","access_level":"open_access","date_updated":"2023-07-31T10:53:08Z","date_created":"2023-07-31T10:53:08Z","checksum":"72e312aabf0c5248c99b5cd3a88e4c88","success":1,"relation":"main_file","file_id":"13334"}],"title":"Provably-efficient and internally-deterministic parallel Union-Find","ddc":["000"],"status":"public","_id":"13262","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","abstract":[{"lang":"eng","text":"Determining the degree of inherent parallelism in classical sequential algorithms and leveraging it for fast parallel execution is a key topic in parallel computing, and detailed analyses are known for a wide range of classical algorithms. In this paper, we perform the first such analysis for the fundamental Union-Find problem, in which we are given a graph as a sequence of edges, and must maintain its connectivity structure under edge additions. We prove that classic sequential algorithms for this problem are well-parallelizable under reasonable assumptions, addressing a conjecture by [Blelloch, 2017]. More precisely, we show via a new potential argument that, under uniform random edge ordering, parallel union-find operations are unlikely to interfere: T concurrent threads processing the graph in parallel will encounter memory contention O(T2 · log |V| · log |E|) times in expectation, where |E| and |V| are the number of edges and nodes in the graph, respectively. We leverage this result to design a new parallel Union-Find algorithm that is both internally deterministic, i.e., its results are guaranteed to match those of a sequential execution, but also work-efficient and scalable, as long as the number of threads T is O(|E|1 over 3 - ε), for an arbitrarily small constant ε > 0, which holds for most large real-world graphs. We present lower bounds which show that our analysis is close to optimal, and experimental results suggesting that the performance cost of internal determinism is limited."}],"type":"conference","date_published":"2023-06-17T00:00:00Z","page":"261-271","citation":{"ista":"Fedorov A, Hashemi D, Nadiradze G, Alistarh D-A. 2023. Provably-efficient and internally-deterministic parallel Union-Find. Proceedings of the 35th ACM Symposium on Parallelism in Algorithms and Architectures. SPAA: Symposium on Parallelism in Algorithms and Architectures, 261–271.","ieee":"A. Fedorov, D. Hashemi, G. Nadiradze, and D.-A. Alistarh, “Provably-efficient and internally-deterministic parallel Union-Find,” in Proceedings of the 35th ACM Symposium on Parallelism in Algorithms and Architectures, Orlando, FL, United States, 2023, pp. 261–271.","apa":"Fedorov, A., Hashemi, D., Nadiradze, G., & Alistarh, D.-A. (2023). Provably-efficient and internally-deterministic parallel Union-Find. In Proceedings of the 35th ACM Symposium on Parallelism in Algorithms and Architectures (pp. 261–271). Orlando, FL, United States: Association for Computing Machinery. https://doi.org/10.1145/3558481.3591082","ama":"Fedorov A, Hashemi D, Nadiradze G, Alistarh D-A. Provably-efficient and internally-deterministic parallel Union-Find. In: Proceedings of the 35th ACM Symposium on Parallelism in Algorithms and Architectures. Association for Computing Machinery; 2023:261-271. doi:10.1145/3558481.3591082","chicago":"Fedorov, Alexander, Diba Hashemi, Giorgi Nadiradze, and Dan-Adrian Alistarh. “Provably-Efficient and Internally-Deterministic Parallel Union-Find.” In Proceedings of the 35th ACM Symposium on Parallelism in Algorithms and Architectures, 261–71. Association for Computing Machinery, 2023. https://doi.org/10.1145/3558481.3591082.","mla":"Fedorov, Alexander, et al. “Provably-Efficient and Internally-Deterministic Parallel Union-Find.” Proceedings of the 35th ACM Symposium on Parallelism in Algorithms and Architectures, Association for Computing Machinery, 2023, pp. 261–71, doi:10.1145/3558481.3591082.","short":"A. Fedorov, D. Hashemi, G. Nadiradze, D.-A. Alistarh, in:, Proceedings of the 35th ACM Symposium on Parallelism in Algorithms and Architectures, Association for Computing Machinery, 2023, pp. 261–271."},"publication":"Proceedings of the 35th ACM Symposium on Parallelism in Algorithms and Architectures","article_processing_charge":"Yes (in subscription journal)","has_accepted_license":"1","day":"17","scopus_import":"1","date_updated":"2023-07-31T10:54:32Z","date_created":"2023-07-23T22:01:12Z","author":[{"full_name":"Fedorov, Alexander","first_name":"Alexander","last_name":"Fedorov","id":"2e711909-896a-11ed-bdf8-eb0f5a2984c6"},{"id":"ed9595ea-2f8f-11ee-ba95-d2b546540783","first_name":"Diba","last_name":"Hashemi","full_name":"Hashemi, Diba"},{"last_name":"Nadiradze","first_name":"Giorgi","id":"3279A00C-F248-11E8-B48F-1D18A9856A87","full_name":"Nadiradze, Giorgi"},{"first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","full_name":"Alistarh, Dan-Adrian"}],"publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"},{"_id":"GradSch"}],"publication_status":"published","year":"2023","file_date_updated":"2023-07-31T10:53:08Z","language":[{"iso":"eng"}],"doi":"10.1145/3558481.3591082","conference":{"name":"SPAA: Symposium on Parallelism in Algorithms and Architectures","location":"Orlando, FL, United States","start_date":"2023-06-17","end_date":"2023-06-19"},"quality_controlled":"1","tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"external_id":{"arxiv":["2304.09331"]},"publication_identifier":{"isbn":["9781450395458"]},"month":"06"},{"author":[{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"},{"full_name":"Ellen, Faith","last_name":"Ellen","first_name":"Faith"},{"orcid":"0000-0002-6432-6646","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","last_name":"Rybicki","first_name":"Joel","full_name":"Rybicki, Joel"}],"volume":948,"date_updated":"2023-08-01T13:17:20Z","date_created":"2023-02-19T23:00:55Z","year":"2023","acknowledgement":"This project has received funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No. 805223 ScaleML) and under the Marie Skłodowska-Curie grant agreement No. 840605 and from the Natural Sciences and Engineering Research Council of Canada grant RGPIN-2020-04178. Part of this work was done while Faith Ellen was visiting IST Austria.","department":[{"_id":"DaAl"}],"publisher":"Elsevier","publication_status":"published","ec_funded":1,"file_date_updated":"2023-02-20T07:30:20Z","article_number":"113733","doi":"10.1016/j.tcs.2023.113733","language":[{"iso":"eng"}],"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"external_id":{"isi":["000934262700001"]},"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"},{"call_identifier":"H2020","name":"Coordination in constrained and natural distributed systems","_id":"26A5D39A-B435-11E9-9278-68D0E5697425","grant_number":"840605"}],"isi":1,"quality_controlled":"1","publication_identifier":{"issn":["0304-3975"]},"month":"02","file":[{"relation":"main_file","file_id":"12570","date_created":"2023-02-20T07:30:20Z","date_updated":"2023-02-20T07:30:20Z","checksum":"b27c5290f2f1500c403494364ee39c9f","success":1,"file_name":"2023_TheoreticalCompScience_Alistarh.pdf","access_level":"open_access","content_type":"application/pdf","file_size":602333,"creator":"dernst"}],"oa_version":"Published Version","_id":"12566","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","intvolume":" 948","status":"public","title":"Wait-free approximate agreement on graphs","ddc":["000"],"issue":"2","abstract":[{"text":"Approximate agreement is one of the few variants of consensus that can be solved in a wait-free manner in asynchronous systems where processes communicate by reading and writing to shared memory. In this work, we consider a natural generalisation of approximate agreement on arbitrary undirected connected graphs. Each process is given a node of the graph as input and, if non-faulty, must output a node such that\r\n– all the outputs are within distance 1 of one another, and\r\n– each output value lies on a shortest path between two input values.\r\nFrom prior work, it is known that there is no wait-free algorithm among processes for this problem on any cycle of length , by reduction from 2-set agreement (Castañeda et al., 2018).\r\n\r\nIn this work, we investigate the solvability of this task on general graphs. We give a new, direct proof of the impossibility of approximate agreement on cycles of length , via a generalisation of Sperner's Lemma to convex polygons. We also extend the reduction from 2-set agreement to a larger class of graphs, showing that approximate agreement on these graphs is unsolvable. On the positive side, we present a wait-free algorithm for a different class of graphs, which properly contains the class of chordal graphs.","lang":"eng"}],"type":"journal_article","date_published":"2023-02-28T00:00:00Z","citation":{"ama":"Alistarh D-A, Ellen F, Rybicki J. Wait-free approximate agreement on graphs. Theoretical Computer Science. 2023;948(2). doi:10.1016/j.tcs.2023.113733","ista":"Alistarh D-A, Ellen F, Rybicki J. 2023. Wait-free approximate agreement on graphs. Theoretical Computer Science. 948(2), 113733.","apa":"Alistarh, D.-A., Ellen, F., & Rybicki, J. (2023). Wait-free approximate agreement on graphs. Theoretical Computer Science. Elsevier. https://doi.org/10.1016/j.tcs.2023.113733","ieee":"D.-A. Alistarh, F. Ellen, and J. Rybicki, “Wait-free approximate agreement on graphs,” Theoretical Computer Science, vol. 948, no. 2. Elsevier, 2023.","mla":"Alistarh, Dan-Adrian, et al. “Wait-Free Approximate Agreement on Graphs.” Theoretical Computer Science, vol. 948, no. 2, 113733, Elsevier, 2023, doi:10.1016/j.tcs.2023.113733.","short":"D.-A. Alistarh, F. Ellen, J. Rybicki, Theoretical Computer Science 948 (2023).","chicago":"Alistarh, Dan-Adrian, Faith Ellen, and Joel Rybicki. “Wait-Free Approximate Agreement on Graphs.” Theoretical Computer Science. Elsevier, 2023. https://doi.org/10.1016/j.tcs.2023.113733."},"publication":"Theoretical Computer Science","article_type":"original","has_accepted_license":"1","article_processing_charge":"Yes (via OA deal)","day":"28","scopus_import":"1"},{"oa_version":"Published Version","file":[{"creator":"epeste","content_type":"application/pdf","file_size":2152072,"access_level":"open_access","file_name":"PhD_Thesis_Alexandra_Peste_final.pdf","success":1,"checksum":"6b3354968403cb9d48cc5a83611fb571","date_created":"2023-05-24T16:11:16Z","date_updated":"2023-05-24T16:11:16Z","file_id":"13087","relation":"main_file"},{"relation":"source_file","file_id":"13088","checksum":"8d0df94bbcf4db72c991f22503b3fd60","date_updated":"2023-05-24T16:12:59Z","date_created":"2023-05-24T16:12:59Z","access_level":"closed","file_name":"PhD_Thesis_APeste.zip","content_type":"application/zip","file_size":1658293,"creator":"epeste"}],"_id":"13074","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","status":"public","ddc":["000"],"title":"Efficiency and generalization of sparse neural networks","abstract":[{"text":"Deep learning has become an integral part of a large number of important applications, and many of the recent breakthroughs have been enabled by the ability to train very large models, capable to capture complex patterns and relationships from the data. At the same time, the massive sizes of modern deep learning models have made their deployment to smaller devices more challenging; this is particularly important, as in many applications the users rely on accurate deep learning predictions, but they only have access to devices with limited memory and compute power. One solution to this problem is to prune neural networks, by setting as many of their parameters as possible to zero, to obtain accurate sparse models with lower memory footprint. Despite the great research progress in obtaining sparse models that preserve accuracy, while satisfying memory and computational constraints, there are still many challenges associated with efficiently training sparse models, as well as understanding their generalization properties.\r\n\r\nThe focus of this thesis is to investigate how the training process of sparse models can be made more efficient, and to understand the differences between sparse and dense models in terms of how well they can generalize to changes in the data distribution. We first study a method for co-training sparse and dense models, at a lower cost compared to regular training. With our method we can obtain very accurate sparse networks, and dense models that can recover the baseline accuracy. Furthermore, we are able to more easily analyze the differences, at prediction level, between the sparse-dense model pairs. Next, we investigate the generalization properties of sparse neural networks in more detail, by studying how well different sparse models trained on a larger task can adapt to smaller, more specialized tasks, in a transfer learning scenario. Our analysis across multiple pruning methods and sparsity levels reveals that sparse models provide features that can transfer similarly to or better than the dense baseline. However, the choice of the pruning method plays an important role, and can influence the results when the features are fixed (linear finetuning), or when they are allowed to adapt to the new task (full finetuning). Using sparse models with fixed masks for finetuning on new tasks has an important practical advantage, as it enables training neural networks on smaller devices. However, one drawback of current pruning methods is that the entire training cycle has to be repeated to obtain the initial sparse model, for every sparsity target; in consequence, the entire training process is costly and also multiple models need to be stored. In the last part of the thesis we propose a method that can train accurate dense models that are compressible in a single step, to multiple sparsity levels, without additional finetuning. Our method results in sparse models that can be competitive with existing pruning methods, and which can also successfully generalize to new tasks.","lang":"eng"}],"type":"dissertation","alternative_title":["ISTA Thesis"],"date_published":"2023-05-23T00:00:00Z","citation":{"ama":"Peste E-A. Efficiency and generalization of sparse neural networks. 2023. doi:10.15479/at:ista:13074","ista":"Peste E-A. 2023. Efficiency and generalization of sparse neural networks. Institute of Science and Technology Austria.","apa":"Peste, E.-A. (2023). Efficiency and generalization of sparse neural networks. Institute of Science and Technology Austria. https://doi.org/10.15479/at:ista:13074","ieee":"E.-A. Peste, “Efficiency and generalization of sparse neural networks,” Institute of Science and Technology Austria, 2023.","mla":"Peste, Elena-Alexandra. Efficiency and Generalization of Sparse Neural Networks. Institute of Science and Technology Austria, 2023, doi:10.15479/at:ista:13074.","short":"E.-A. Peste, Efficiency and Generalization of Sparse Neural Networks, Institute of Science and Technology Austria, 2023.","chicago":"Peste, Elena-Alexandra. “Efficiency and Generalization of Sparse Neural Networks.” Institute of Science and Technology Austria, 2023. https://doi.org/10.15479/at:ista:13074."},"page":"147","has_accepted_license":"1","article_processing_charge":"No","day":"23","related_material":{"record":[{"relation":"part_of_dissertation","status":"public","id":"11458"},{"id":"13053","relation":"part_of_dissertation","status":"public"},{"status":"public","relation":"part_of_dissertation","id":"12299"}]},"author":[{"first_name":"Elena-Alexandra","last_name":"Peste","id":"32D78294-F248-11E8-B48F-1D18A9856A87","full_name":"Peste, Elena-Alexandra"}],"date_created":"2023-05-23T17:07:53Z","date_updated":"2023-08-04T10:33:27Z","year":"2023","publisher":"Institute of Science and Technology Austria","department":[{"_id":"GradSch"},{"_id":"DaAl"},{"_id":"ChLa"}],"publication_status":"published","ec_funded":1,"file_date_updated":"2023-05-24T16:12:59Z","doi":"10.15479/at:ista:13074","language":[{"iso":"eng"}],"supervisor":[{"full_name":"Lampert, Christoph","id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0001-8622-7887","first_name":"Christoph","last_name":"Lampert"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"}],"degree_awarded":"PhD","acknowledged_ssus":[{"_id":"ScienComp"}],"oa":1,"project":[{"name":"International IST Doctoral Program","call_identifier":"H2020","grant_number":"665385","_id":"2564DBCA-B435-11E9-9278-68D0E5697425"},{"_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning"}],"publication_identifier":{"issn":["2663-337X"]},"month":"05"},{"page":"395-418","article_type":"original","citation":{"short":"V. Aksenov, D.-A. Alistarh, A. Drozdova, A. Mohtashami, Distributed Computing 36 (2023) 395–418.","mla":"Aksenov, Vitalii, et al. “The Splay-List: A Distribution-Adaptive Concurrent Skip-List.” Distributed Computing, vol. 36, Springer Nature, 2023, pp. 395–418, doi:10.1007/s00446-022-00441-x.","chicago":"Aksenov, Vitalii, Dan-Adrian Alistarh, Alexandra Drozdova, and Amirkeivan Mohtashami. “The Splay-List: A Distribution-Adaptive Concurrent Skip-List.” Distributed Computing. Springer Nature, 2023. https://doi.org/10.1007/s00446-022-00441-x.","ama":"Aksenov V, Alistarh D-A, Drozdova A, Mohtashami A. The splay-list: A distribution-adaptive concurrent skip-list. Distributed Computing. 2023;36:395-418. doi:10.1007/s00446-022-00441-x","ieee":"V. Aksenov, D.-A. Alistarh, A. Drozdova, and A. Mohtashami, “The splay-list: A distribution-adaptive concurrent skip-list,” Distributed Computing, vol. 36. Springer Nature, pp. 395–418, 2023.","apa":"Aksenov, V., Alistarh, D.-A., Drozdova, A., & Mohtashami, A. (2023). The splay-list: A distribution-adaptive concurrent skip-list. Distributed Computing. Springer Nature. https://doi.org/10.1007/s00446-022-00441-x","ista":"Aksenov V, Alistarh D-A, Drozdova A, Mohtashami A. 2023. The splay-list: A distribution-adaptive concurrent skip-list. Distributed Computing. 36, 395–418."},"publication":"Distributed Computing","date_published":"2023-09-01T00:00:00Z","scopus_import":"1","article_processing_charge":"No","day":"01","intvolume":" 36","title":"The splay-list: A distribution-adaptive concurrent skip-list","status":"public","_id":"12330","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","oa_version":"Preprint","type":"journal_article","abstract":[{"lang":"eng","text":"The design and implementation of efficient concurrent data structures has seen significant attention. However, most of this work has focused on concurrent data structures providing good worst-case guarantees, although, in real workloads, objects are often accessed at different rates. Efficient distribution-adaptive data structures, such as splay-trees, are known in the sequential case; however, they often are hard to translate efficiently to the concurrent case. We investigate distribution-adaptive concurrent data structures, and propose a new design called the splay-list. At a high level, the splay-list is similar to a standard skip-list, with the key distinction that the height of each element adapts dynamically to its access rate: popular elements “move up,” whereas rarely-accessed elements decrease in height. We show that the splay-list provides order-optimal amortized complexity bounds for a subset of operations, while being amenable to efficient concurrent implementation. Experiments show that the splay-list can leverage distribution-adaptivity for performance, and can outperform the only previously-known distribution-adaptive concurrent design in certain workloads."}],"isi":1,"quality_controlled":"1","external_id":{"isi":["000913424000001"],"arxiv":["2008.01009"]},"oa":1,"main_file_link":[{"url":"https://doi.org/10.48550/arXiv.2008.01009","open_access":"1"}],"language":[{"iso":"eng"}],"doi":"10.1007/s00446-022-00441-x","publication_identifier":{"issn":["0178-2770"],"eissn":["1432-0452"]},"month":"09","publisher":"Springer Nature","department":[{"_id":"DaAl"}],"publication_status":"published","year":"2023","volume":36,"date_created":"2023-01-22T23:00:55Z","date_updated":"2023-08-14T12:54:32Z","author":[{"full_name":"Aksenov, Vitalii","id":"2980135A-F248-11E8-B48F-1D18A9856A87","last_name":"Aksenov","first_name":"Vitalii"},{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian"},{"full_name":"Drozdova, Alexandra","last_name":"Drozdova","first_name":"Alexandra"},{"last_name":"Mohtashami","first_name":"Amirkeivan","full_name":"Mohtashami, Amirkeivan"}]},{"type":"conference","alternative_title":["PMLR"],"abstract":[{"lang":"eng","text":"Communication-reduction techniques are a popular way to improve scalability in data-parallel training of deep neural networks (DNNs). The recent emergence of large language models such as GPT has created the need for new approaches to exploit data-parallelism. Among these, fully-sharded data parallel (FSDP) training is highly popular, yet it still encounters scalability bottlenecks. One reason is that applying compression techniques to FSDP is challenging: as the vast majority of the communication involves the model’s weights, direct compression alters convergence and leads to accuracy loss. We present QSDP, a variant of FSDP which supports both gradient and weight quantization with theoretical guarantees, is simple to implement and has essentially no overheads. To derive QSDP we prove that a natural modification of SGD achieves convergence even when we only maintain quantized weights, and thus the domain over which we train consists of quantized points and is, therefore, highly non-convex. We validate this approach by training GPT-family models with up to 1.3 billion parameters on a multi-node cluster. Experiments show that QSDP preserves model accuracy, while completely removing the communication bottlenecks of FSDP, providing end-to-end speedups of up to 2.2x."}],"_id":"14461","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","intvolume":" 202","status":"public","title":"Quantized distributed training of large models with convergence guarantees","oa_version":"Preprint","scopus_import":"1","article_processing_charge":"No","day":"30","citation":{"ieee":"I. Markov, A. Vladu, Q. Guo, and D.-A. Alistarh, “Quantized distributed training of large models with convergence guarantees,” in Proceedings of the 40th International Conference on Machine Learning, Honolulu, Hawaii, HI, United States, 2023, vol. 202, pp. 24020–24044.","apa":"Markov, I., Vladu, A., Guo, Q., & Alistarh, D.-A. (2023). Quantized distributed training of large models with convergence guarantees. In Proceedings of the 40th International Conference on Machine Learning (Vol. 202, pp. 24020–24044). Honolulu, Hawaii, HI, United States: ML Research Press.","ista":"Markov I, Vladu A, Guo Q, Alistarh D-A. 2023. Quantized distributed training of large models with convergence guarantees. Proceedings of the 40th International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 202, 24020–24044.","ama":"Markov I, Vladu A, Guo Q, Alistarh D-A. Quantized distributed training of large models with convergence guarantees. In: Proceedings of the 40th International Conference on Machine Learning. Vol 202. ML Research Press; 2023:24020-24044.","chicago":"Markov, Ilia, Adrian Vladu, Qi Guo, and Dan-Adrian Alistarh. “Quantized Distributed Training of Large Models with Convergence Guarantees.” In Proceedings of the 40th International Conference on Machine Learning, 202:24020–44. ML Research Press, 2023.","short":"I. Markov, A. Vladu, Q. Guo, D.-A. Alistarh, in:, Proceedings of the 40th International Conference on Machine Learning, ML Research Press, 2023, pp. 24020–24044.","mla":"Markov, Ilia, et al. “Quantized Distributed Training of Large Models with Convergence Guarantees.” Proceedings of the 40th International Conference on Machine Learning, vol. 202, ML Research Press, 2023, pp. 24020–44."},"publication":"Proceedings of the 40th International Conference on Machine Learning","page":"24020-24044","date_published":"2023-07-30T00:00:00Z","ec_funded":1,"acknowledgement":"The authors gratefully acknowledge funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML), as well as experimental support from the IST Austria IT department, in particular Stefano Elefante, Andrei Hornoiu, and Alois Schloegl. AV acknowledges the support of the French Agence Nationale de la Recherche (ANR), under grant ANR-21-CE48-0016 (project COMCOPT), the support of Fondation Hadamard with a PRMO grant, and the support of CNRS with a CoopIntEER IEA grant (project ALFRED).","year":"2023","publisher":"ML Research Press","department":[{"_id":"DaAl"}],"publication_status":"published","author":[{"id":"D0CF4148-C985-11E9-8066-0BDEE5697425","last_name":"Markov","first_name":"Ilia","full_name":"Markov, Ilia"},{"full_name":"Vladu, Adrian","first_name":"Adrian","last_name":"Vladu"},{"first_name":"Qi","last_name":"Guo","full_name":"Guo, Qi"},{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"}],"volume":202,"date_updated":"2023-10-31T09:40:45Z","date_created":"2023-10-29T23:01:17Z","publication_identifier":{"eissn":["2640-3498"]},"month":"07","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2302.02390"}],"oa":1,"external_id":{"arxiv":["2302.02390"]},"project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"quality_controlled":"1","conference":{"end_date":"2023-07-29","start_date":"2023-07-23","location":"Honolulu, Hawaii, HI, United States","name":"ICML: International Conference on Machine Learning"},"language":[{"iso":"eng"}],"acknowledged_ssus":[{"_id":"ScienComp"}]},{"alternative_title":["PMLR"],"type":"conference","abstract":[{"lang":"eng","text":"Autoencoders are a popular model in many branches of machine learning and lossy data compression. However, their fundamental limits, the performance of gradient methods and the features learnt during optimization remain poorly understood, even in the two-layer setting. In fact, earlier work has considered either linear autoencoders or specific training regimes (leading to vanishing or diverging compression rates). Our paper addresses this gap by focusing on non-linear two-layer autoencoders trained in the challenging proportional regime in which the input dimension scales linearly with the size of the representation. Our results characterize the minimizers of the population risk, and show that such minimizers are achieved by gradient methods; their structure is also unveiled, thus leading to a concise description of the features obtained via training. For the special case of a sign activation function, our analysis establishes the fundamental limits for the lossy compression of Gaussian sources via (shallow) autoencoders. Finally, while the results are proved for Gaussian data, numerical simulations on standard datasets display the universality of the theoretical predictions."}],"intvolume":" 202","status":"public","title":"Fundamental limits of two-layer autoencoders, and achieving them with gradient methods","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"14459","oa_version":"Preprint","scopus_import":"1","article_processing_charge":"No","day":"30","page":"31151-31209","citation":{"ama":"Shevchenko A, Kögler K, Hassani H, Mondelli M. Fundamental limits of two-layer autoencoders, and achieving them with gradient methods. In: Proceedings of the 40th International Conference on Machine Learning. Vol 202. ML Research Press; 2023:31151-31209.","ieee":"A. Shevchenko, K. Kögler, H. Hassani, and M. Mondelli, “Fundamental limits of two-layer autoencoders, and achieving them with gradient methods,” in Proceedings of the 40th International Conference on Machine Learning, Honolulu, Hawaii, HI, United States, 2023, vol. 202, pp. 31151–31209.","apa":"Shevchenko, A., Kögler, K., Hassani, H., & Mondelli, M. (2023). Fundamental limits of two-layer autoencoders, and achieving them with gradient methods. In Proceedings of the 40th International Conference on Machine Learning (Vol. 202, pp. 31151–31209). Honolulu, Hawaii, HI, United States: ML Research Press.","ista":"Shevchenko A, Kögler K, Hassani H, Mondelli M. 2023. Fundamental limits of two-layer autoencoders, and achieving them with gradient methods. Proceedings of the 40th International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 202, 31151–31209.","short":"A. Shevchenko, K. Kögler, H. Hassani, M. Mondelli, in:, Proceedings of the 40th International Conference on Machine Learning, ML Research Press, 2023, pp. 31151–31209.","mla":"Shevchenko, Aleksandr, et al. “Fundamental Limits of Two-Layer Autoencoders, and Achieving Them with Gradient Methods.” Proceedings of the 40th International Conference on Machine Learning, vol. 202, ML Research Press, 2023, pp. 31151–209.","chicago":"Shevchenko, Aleksandr, Kevin Kögler, Hamed Hassani, and Marco Mondelli. “Fundamental Limits of Two-Layer Autoencoders, and Achieving Them with Gradient Methods.” In Proceedings of the 40th International Conference on Machine Learning, 202:31151–209. ML Research Press, 2023."},"publication":"Proceedings of the 40th International Conference on Machine Learning","date_published":"2023-07-30T00:00:00Z","department":[{"_id":"MaMo"},{"_id":"DaAl"}],"publisher":"ML Research Press","publication_status":"published","acknowledgement":"Aleksandr Shevchenko, Kevin Kogler and Marco Mondelli are supported by the 2019 Lopez-Loreta Prize. Hamed Hassani acknowledges the support by the NSF CIF award (1910056) and the NSF Institute for CORE Emerging Methods in Data Science (EnCORE).","year":"2023","volume":202,"date_updated":"2023-10-31T08:52:28Z","date_created":"2023-10-29T23:01:17Z","author":[{"first_name":"Aleksandr","last_name":"Shevchenko","id":"F2B06EC2-C99E-11E9-89F0-752EE6697425","full_name":"Shevchenko, Aleksandr"},{"full_name":"Kögler, Kevin","id":"94ec913c-dc85-11ea-9058-e5051ab2428b","first_name":"Kevin","last_name":"Kögler"},{"full_name":"Hassani, Hamed","last_name":"Hassani","first_name":"Hamed"},{"full_name":"Mondelli, Marco","last_name":"Mondelli","first_name":"Marco","orcid":"0000-0002-3242-7020","id":"27EB676C-8706-11E9-9510-7717E6697425"}],"publication_identifier":{"eissn":["2640-3498"]},"month":"07","project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"}],"quality_controlled":"1","external_id":{"arxiv":["2212.13468"]},"main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2212.13468"}],"oa":1,"language":[{"iso":"eng"}],"conference":{"name":"ICML: International Conference on Machine Learning","location":"Honolulu, Hawaii, HI, United States","start_date":"2023-07-23","end_date":"2023-07-29"}},{"abstract":[{"lang":"eng","text":"We provide an efficient implementation of the backpropagation algorithm, specialized to the case where the weights of the neural network being trained are sparse. Our algorithm is general, as it applies to arbitrary (unstructured) sparsity and common layer types (e.g., convolutional or linear). We provide a fast vectorized implementation on commodity CPUs, and show that it can yield speedups in end-to-end runtime experiments, both in transfer learning using already-sparsified networks, and in training sparse networks from scratch. Thus, our results provide the first support for sparse training on commodity hardware."}],"type":"conference","alternative_title":["PMLR"],"oa_version":"Preprint","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"14460","title":"SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge","status":"public","intvolume":" 202","day":"30","article_processing_charge":"No","scopus_import":"1","date_published":"2023-07-30T00:00:00Z","publication":"Proceedings of the 40th International Conference on Machine Learning","citation":{"mla":"Nikdan, Mahdi, et al. “SparseProp: Efficient Sparse Backpropagation for Faster Training of Neural Networks at the Edge.” Proceedings of the 40th International Conference on Machine Learning, vol. 202, ML Research Press, 2023, pp. 26215–27.","short":"M. Nikdan, T. Pegolotti, E.B. Iofinova, E. Kurtic, D.-A. Alistarh, in:, Proceedings of the 40th International Conference on Machine Learning, ML Research Press, 2023, pp. 26215–26227.","chicago":"Nikdan, Mahdi, Tommaso Pegolotti, Eugenia B Iofinova, Eldar Kurtic, and Dan-Adrian Alistarh. “SparseProp: Efficient Sparse Backpropagation for Faster Training of Neural Networks at the Edge.” In Proceedings of the 40th International Conference on Machine Learning, 202:26215–27. ML Research Press, 2023.","ama":"Nikdan M, Pegolotti T, Iofinova EB, Kurtic E, Alistarh D-A. SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge. In: Proceedings of the 40th International Conference on Machine Learning. Vol 202. ML Research Press; 2023:26215-26227.","ista":"Nikdan M, Pegolotti T, Iofinova EB, Kurtic E, Alistarh D-A. 2023. SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge. Proceedings of the 40th International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 202, 26215–26227.","apa":"Nikdan, M., Pegolotti, T., Iofinova, E. B., Kurtic, E., & Alistarh, D.-A. (2023). SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge. In Proceedings of the 40th International Conference on Machine Learning (Vol. 202, pp. 26215–26227). Honolulu, Hawaii, HI, United States: ML Research Press.","ieee":"M. Nikdan, T. Pegolotti, E. B. Iofinova, E. Kurtic, and D.-A. Alistarh, “SparseProp: Efficient sparse backpropagation for faster training of neural networks at the edge,” in Proceedings of the 40th International Conference on Machine Learning, Honolulu, Hawaii, HI, United States, 2023, vol. 202, pp. 26215–26227."},"page":"26215-26227","ec_funded":1,"author":[{"last_name":"Nikdan","first_name":"Mahdi","id":"66374281-f394-11eb-9cf6-869147deecc0","full_name":"Nikdan, Mahdi"},{"first_name":"Tommaso","last_name":"Pegolotti","full_name":"Pegolotti, Tommaso"},{"first_name":"Eugenia B","last_name":"Iofinova","id":"f9a17499-f6e0-11ea-865d-fdf9a3f77117","orcid":"0000-0002-7778-3221","full_name":"Iofinova, Eugenia B"},{"full_name":"Kurtic, Eldar","first_name":"Eldar","last_name":"Kurtic","id":"47beb3a5-07b5-11eb-9b87-b108ec578218"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"date_created":"2023-10-29T23:01:17Z","date_updated":"2023-10-31T09:33:51Z","volume":202,"year":"2023","acknowledgement":"We would like to thank Elias Frantar for his valuable assistance and support at the outset of this project, and the anonymous ICML and SNN reviewers for very constructive feedback. EI was supported in part by the FWF DK VGSCO, grant agreement number W1260-N35. DA acknowledges generous ERC support, via Starting Grant 805223 ScaleML. ","publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"ML Research Press","month":"07","publication_identifier":{"eissn":["2640-3498"]},"conference":{"name":"ICML: International Conference on Machine Learning","start_date":"2023-07-23","location":"Honolulu, Hawaii, HI, United States","end_date":"2023-07-29"},"language":[{"iso":"eng"}],"main_file_link":[{"url":"https://doi.org/10.48550/arXiv.2302.04852","open_access":"1"}],"oa":1,"external_id":{"arxiv":["2302.04852"]},"quality_controlled":"1","project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}]},{"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"quality_controlled":"1","main_file_link":[{"open_access":"1","url":"https://doi.org/10.48550/arXiv.2301.00774"}],"external_id":{"arxiv":["2301.00774"]},"oa":1,"language":[{"iso":"eng"}],"acknowledged_ssus":[{"_id":"ScienComp"}],"conference":{"end_date":"2023-07-29","start_date":"2023-07-23","location":"Honolulu, Hawaii, HI, United States","name":"ICML: International Conference on Machine Learning"},"publication_identifier":{"eissn":["2640-3498"]},"month":"07","publisher":"ML Research Press","department":[{"_id":"DaAl"}],"publication_status":"published","acknowledgement":"The authors gratefully acknowledge funding from the European Research Council (ERC) under the European Union’s Horizon 2020 programme (grant agreement No. 805223 ScaleML), as well as experimental support from Eldar Kurtic, and from the IST Austria IT department, in particular Stefano Elefante, Andrei Hornoiu, and Alois Schloegl.","year":"2023","volume":202,"date_updated":"2023-10-31T09:59:42Z","date_created":"2023-10-29T23:01:16Z","author":[{"full_name":"Frantar, Elias","id":"09a8f98d-ec99-11ea-ae11-c063a7b7fe5f","last_name":"Frantar","first_name":"Elias"},{"orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian"}],"ec_funded":1,"page":"10323-10337","citation":{"ama":"Frantar E, Alistarh D-A. SparseGPT: Massive language models can be accurately pruned in one-shot. In: Proceedings of the 40th International Conference on Machine Learning. Vol 202. ML Research Press; 2023:10323-10337.","ista":"Frantar E, Alistarh D-A. 2023. SparseGPT: Massive language models can be accurately pruned in one-shot. Proceedings of the 40th International Conference on Machine Learning. ICML: International Conference on Machine Learning, PMLR, vol. 202, 10323–10337.","ieee":"E. Frantar and D.-A. Alistarh, “SparseGPT: Massive language models can be accurately pruned in one-shot,” in Proceedings of the 40th International Conference on Machine Learning, Honolulu, Hawaii, HI, United States, 2023, vol. 202, pp. 10323–10337.","apa":"Frantar, E., & Alistarh, D.-A. (2023). SparseGPT: Massive language models can be accurately pruned in one-shot. In Proceedings of the 40th International Conference on Machine Learning (Vol. 202, pp. 10323–10337). Honolulu, Hawaii, HI, United States: ML Research Press.","mla":"Frantar, Elias, and Dan-Adrian Alistarh. “SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot.” Proceedings of the 40th International Conference on Machine Learning, vol. 202, ML Research Press, 2023, pp. 10323–37.","short":"E. Frantar, D.-A. Alistarh, in:, Proceedings of the 40th International Conference on Machine Learning, ML Research Press, 2023, pp. 10323–10337.","chicago":"Frantar, Elias, and Dan-Adrian Alistarh. “SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot.” In Proceedings of the 40th International Conference on Machine Learning, 202:10323–37. ML Research Press, 2023."},"publication":"Proceedings of the 40th International Conference on Machine Learning","date_published":"2023-07-30T00:00:00Z","scopus_import":"1","article_processing_charge":"No","day":"30","intvolume":" 202","title":"SparseGPT: Massive language models can be accurately pruned in one-shot","status":"public","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"14458","oa_version":"Preprint","alternative_title":["PMLR"],"type":"conference","abstract":[{"text":"We show for the first time that large-scale generative pretrained transformer (GPT) family models can be pruned to at least 50% sparsity in one-shot, without any retraining, at minimal loss of accuracy. This is achieved via a new pruning method called SparseGPT, specifically designed to work efficiently and accurately on massive GPT-family models. We can execute SparseGPT on the largest available open-source models, OPT-175B and BLOOM-176B, in under 4.5 hours, and can reach 60% unstructured sparsity with negligible increase in perplexity: remarkably, more than 100 billion weights from these models can be ignored at inference time. SparseGPT generalizes to semi-structured (2:4 and 4:8) patterns, and is compatible with weight quantization approaches. The code is available at: https://github.com/IST-DASLab/sparsegpt.","lang":"eng"}]},{"related_material":{"record":[{"id":"6676","relation":"earlier_version","status":"public"}]},"author":[{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian"},{"first_name":"James","last_name":"Aspnes","full_name":"Aspnes, James"},{"full_name":"Ellen, Faith","first_name":"Faith","last_name":"Ellen"},{"full_name":"Gelashvili, Rati","first_name":"Rati","last_name":"Gelashvili"},{"full_name":"Zhu, Leqi","id":"a2117c59-cee4-11ed-b9d0-874ecf0f8ac5","first_name":"Leqi","last_name":"Zhu"}],"volume":52,"date_updated":"2023-12-13T12:28:29Z","date_created":"2023-09-24T22:01:11Z","acknowledgement":"We would like to thank Valerie King, Toniann Pitassi, and Michael Saks for helpful discussions and Shi Hao Liu for his useful feedback.\r\nThis research was supported by the Natural Science and Engineering Research Council of Canada under grants RGPIN-2015-05080 and RGPIN-2020-04178, a postgraduate scholarship, and a postdoctoral fellowship; a University of Toronto postdoctoral fellowship; the National Science Foundation under grants CCF-1217921, CCF-1301926, CCF-1637385, CCF-1650596, and IIS-1447786; the U.S. Department of Energy under grant ER26116/DE-SC0008923; the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme grant agreement 805223 ScaleML; and the Oracle and Intel corporations. Some of the work on this paper was done while Faith Ellen was visiting IST Austria.","year":"2023","department":[{"_id":"DaAl"}],"publisher":"Society for Industrial and Applied Mathematics","publication_status":"published","ec_funded":1,"doi":"10.1137/20M1375851","language":[{"iso":"eng"}],"oa":1,"external_id":{"isi":["001082972300004"],"arxiv":["1811.01421"]},"main_file_link":[{"url":"https://arxiv.org/abs/1811.01421","open_access":"1"}],"project":[{"_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning"}],"isi":1,"quality_controlled":"1","publication_identifier":{"issn":["0097-5397"],"eissn":["1095-7111"]},"month":"07","oa_version":"Preprint","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"14364","intvolume":" 52","status":"public","title":"Why extension-based proofs fail","issue":"4","abstract":[{"lang":"eng","text":"We introduce extension-based proofs, a class of impossibility proofs that includes valency arguments. They are modelled as an interaction between a prover and a protocol. Using proofs based on combinatorial topology, it has been shown that it is impossible to deterministically solve -set agreement among processes or approximate agreement on a cycle of length 4 among processes in a wait-free manner in asynchronous models where processes communicate using objects that can be constructed from shared registers. However, it was unknown whether proofs based on simpler techniques were possible. We show that these impossibility results cannot be obtained by extension-based proofs in the iterated snapshot model and, hence, extension-based proofs are limited in power."}],"type":"journal_article","date_published":"2023-07-25T00:00:00Z","citation":{"ista":"Alistarh D-A, Aspnes J, Ellen F, Gelashvili R, Zhu L. 2023. Why extension-based proofs fail. SIAM Journal on Computing. 52(4), 913–944.","apa":"Alistarh, D.-A., Aspnes, J., Ellen, F., Gelashvili, R., & Zhu, L. (2023). Why extension-based proofs fail. SIAM Journal on Computing. Society for Industrial and Applied Mathematics. https://doi.org/10.1137/20M1375851","ieee":"D.-A. Alistarh, J. Aspnes, F. Ellen, R. Gelashvili, and L. Zhu, “Why extension-based proofs fail,” SIAM Journal on Computing, vol. 52, no. 4. Society for Industrial and Applied Mathematics, pp. 913–944, 2023.","ama":"Alistarh D-A, Aspnes J, Ellen F, Gelashvili R, Zhu L. Why extension-based proofs fail. SIAM Journal on Computing. 2023;52(4):913-944. doi:10.1137/20M1375851","chicago":"Alistarh, Dan-Adrian, James Aspnes, Faith Ellen, Rati Gelashvili, and Leqi Zhu. “Why Extension-Based Proofs Fail.” SIAM Journal on Computing. Society for Industrial and Applied Mathematics, 2023. https://doi.org/10.1137/20M1375851.","mla":"Alistarh, Dan-Adrian, et al. “Why Extension-Based Proofs Fail.” SIAM Journal on Computing, vol. 52, no. 4, Society for Industrial and Applied Mathematics, 2023, pp. 913–44, doi:10.1137/20M1375851.","short":"D.-A. Alistarh, J. Aspnes, F. Ellen, R. Gelashvili, L. Zhu, SIAM Journal on Computing 52 (2023) 913–944."},"publication":"SIAM Journal on Computing","page":"913-944","article_type":"original","article_processing_charge":"No","day":"25","scopus_import":"1"},{"ec_funded":1,"author":[{"full_name":"Iofinova, Eugenia B","first_name":"Eugenia B","last_name":"Iofinova","id":"f9a17499-f6e0-11ea-865d-fdf9a3f77117","orcid":"0000-0002-7778-3221"},{"full_name":"Peste, Elena-Alexandra","first_name":"Elena-Alexandra","last_name":"Peste","id":"32D78294-F248-11E8-B48F-1D18A9856A87"},{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"}],"related_material":{"link":[{"relation":"software","url":"https://github.com/IST-DASLab/pruned-vision-model-bias"}]},"date_created":"2024-01-10T08:42:40Z","date_updated":"2024-01-10T08:59:26Z","year":"2023","acknowledgement":"The authors would like to sincerely thank Sara Hooker for her feedback during the development of this work. EI was supported in part by the FWF DK VGSCO, grant agreement number W1260-N35. AP and DA acknowledge generous ERC support, via Starting Grant 805223 ScaleML.","publication_status":"published","department":[{"_id":"DaAl"},{"_id":"ChLa"}],"publisher":"IEEE","month":"08","publication_identifier":{"eisbn":["9798350301298"],"eissn":["2575-7075"]},"conference":{"name":"CVPR: Conference on Computer Vision and Pattern Recognition","location":"Vancouver, BC, Canada","start_date":"2023-06-17","end_date":"2023-06-24"},"doi":"10.1109/cvpr52729.2023.02334","language":[{"iso":"eng"}],"external_id":{"arxiv":["2304.12622"],"isi":["001062531308068"]},"main_file_link":[{"url":"https://doi.org/10.48550/arXiv.2304.12622","open_access":"1"}],"oa":1,"quality_controlled":"1","isi":1,"project":[{"_id":"9B9290DE-BA93-11EA-9121-9846C619BF3A","grant_number":" W1260-N35","name":"Vienna Graduate School on Computational Optimization"},{"_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning"}],"abstract":[{"lang":"eng","text":"Pruning—that is, setting a significant subset of the parameters of a neural network to zero—is one of the most popular methods of model compression. Yet, several recent works have raised the issue that pruning may induce or exacerbate bias in the output of the compressed model. Despite existing evidence for this phenomenon, the relationship between neural network pruning and induced bias is not well-understood. In this work, we systematically investigate and characterize this phenomenon in Convolutional Neural Networks for computer vision. First, we show that it is in fact possible to obtain highly-sparse models, e.g. with less than 10% remaining weights, which do not decrease in accuracy nor substantially increase in bias when compared to dense models. At the same time, we also find that, at higher sparsities, pruned models exhibit higher uncertainty in their outputs, as well as increased correlations, which we directly link to increased bias. We propose easy-to-use criteria which, based only on the uncompressed model, establish whether bias will increase with pruning, and identify the samples most susceptible to biased predictions post-compression. Our code can be found at https://github.com/IST-DASLab/pruned-vision-model-bias."}],"type":"conference","oa_version":"Preprint","_id":"14771","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","status":"public","title":"Bias in pruned vision models: In-depth analysis and countermeasures","day":"22","article_processing_charge":"No","date_published":"2023-08-22T00:00:00Z","publication":"2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition","citation":{"apa":"Iofinova, E. B., Peste, E.-A., & Alistarh, D.-A. (2023). Bias in pruned vision models: In-depth analysis and countermeasures. In 2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition (pp. 24364–24373). Vancouver, BC, Canada: IEEE. https://doi.org/10.1109/cvpr52729.2023.02334","ieee":"E. B. Iofinova, E.-A. Peste, and D.-A. Alistarh, “Bias in pruned vision models: In-depth analysis and countermeasures,” in 2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition, Vancouver, BC, Canada, 2023, pp. 24364–24373.","ista":"Iofinova EB, Peste E-A, Alistarh D-A. 2023. Bias in pruned vision models: In-depth analysis and countermeasures. 2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition. CVPR: Conference on Computer Vision and Pattern Recognition, 24364–24373.","ama":"Iofinova EB, Peste E-A, Alistarh D-A. Bias in pruned vision models: In-depth analysis and countermeasures. In: 2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition. IEEE; 2023:24364-24373. doi:10.1109/cvpr52729.2023.02334","chicago":"Iofinova, Eugenia B, Elena-Alexandra Peste, and Dan-Adrian Alistarh. “Bias in Pruned Vision Models: In-Depth Analysis and Countermeasures.” In 2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition, 24364–73. IEEE, 2023. https://doi.org/10.1109/cvpr52729.2023.02334.","short":"E.B. Iofinova, E.-A. Peste, D.-A. Alistarh, in:, 2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition, IEEE, 2023, pp. 24364–24373.","mla":"Iofinova, Eugenia B., et al. “Bias in Pruned Vision Models: In-Depth Analysis and Countermeasures.” 2023 IEEE/CVF Conference on Computer Vision and Pattern Recognition, IEEE, 2023, pp. 24364–73, doi:10.1109/cvpr52729.2023.02334."},"page":"24364-24373"},{"file_date_updated":"2024-01-16T12:13:27Z","publication_status":"published","publisher":"Journal of Machine Learning Research","department":[{"_id":"DaAl"}],"year":"2023","acknowledgement":"The work in Sections 1-5 was conducted while A. Beznosikov was a research intern in the Optimizationand Machine Learning Lab of Peter Richtárik at KAUST; this visit was funded by the KAUST Baseline Research Funding Scheme. The work of A. Beznosikov in Section 6 was conducted in Skoltech and was supported by Ministry of Science and Higher Education grant No. 075-10-2021-068. ","date_created":"2024-01-16T12:13:36Z","date_updated":"2024-01-17T09:14:13Z","volume":24,"author":[{"full_name":"Beznosikov, Aleksandr","last_name":"Beznosikov","first_name":"Aleksandr"},{"first_name":"Samuel","last_name":"Horvath","full_name":"Horvath, Samuel"},{"full_name":"Richtarik, Peter","last_name":"Richtarik","first_name":"Peter"},{"full_name":"Safaryan, Mher","first_name":"Mher","last_name":"Safaryan","id":"dd546b39-0804-11ed-9c55-ef075c39778d"}],"month":"10","publication_identifier":{"eissn":["1533-7928"]},"isi":1,"quality_controlled":"1","tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"external_id":{"arxiv":["2002.12410"],"isi":["001111578500001"]},"language":[{"iso":"eng"}],"type":"journal_article","abstract":[{"text":"In the last few years, various communication compression techniques have emerged as an indispensable tool helping to alleviate the communication bottleneck in distributed learning. However, despite the fact biased compressors often show superior performance in practice when compared to the much more studied and understood unbiased compressors, very little is known about them. In this work we study three classes of biased compression operators, two of which are new, and their performance when applied to (stochastic) gradient descent and distributed (stochastic) gradient descent. We show for the first time that biased compressors can lead to linear convergence rates both in the single node and distributed settings. We prove that distributed compressed SGD method, employed with error feedback mechanism, enjoys the ergodic rate O(δLexp[−μKδL]+(C+δD)Kμ), where δ≥1 is a compression parameter which grows when more compression is applied, L and μ are the smoothness and strong convexity constants, C captures stochastic gradient noise (C=0 if full gradients are computed on each node) and D captures the variance of the gradients at the optimum (D=0 for over-parameterized models). Further, via a theoretical study of several synthetic and empirical distributions of communicated gradients, we shed light on why and by how much biased compressors outperform their unbiased variants. Finally, we propose several new biased compressors with promising theoretical guarantees and practical performance.","lang":"eng"}],"title":"On biased compression for distributed learning","status":"public","ddc":["000"],"intvolume":" 24","_id":"14815","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","file":[{"creator":"dernst","content_type":"application/pdf","file_size":1510993,"access_level":"open_access","file_name":"2023_JMLR_Beznosikov.pdf","success":1,"checksum":"c50f2b9db53938b755e30a085f464059","date_created":"2024-01-16T12:13:27Z","date_updated":"2024-01-16T12:13:27Z","file_id":"14816","relation":"main_file"}],"oa_version":"Published Version","day":"01","article_processing_charge":"Yes (in subscription journal)","has_accepted_license":"1","article_type":"original","page":"1-50","publication":"Journal of Machine Learning Research","citation":{"ieee":"A. Beznosikov, S. Horvath, P. Richtarik, and M. Safaryan, “On biased compression for distributed learning,” Journal of Machine Learning Research, vol. 24. Journal of Machine Learning Research, pp. 1–50, 2023.","apa":"Beznosikov, A., Horvath, S., Richtarik, P., & Safaryan, M. (2023). On biased compression for distributed learning. Journal of Machine Learning Research. Journal of Machine Learning Research.","ista":"Beznosikov A, Horvath S, Richtarik P, Safaryan M. 2023. On biased compression for distributed learning. Journal of Machine Learning Research. 24, 1–50.","ama":"Beznosikov A, Horvath S, Richtarik P, Safaryan M. On biased compression for distributed learning. Journal of Machine Learning Research. 2023;24:1-50.","chicago":"Beznosikov, Aleksandr, Samuel Horvath, Peter Richtarik, and Mher Safaryan. “On Biased Compression for Distributed Learning.” Journal of Machine Learning Research. Journal of Machine Learning Research, 2023.","short":"A. Beznosikov, S. Horvath, P. Richtarik, M. Safaryan, Journal of Machine Learning Research 24 (2023) 1–50.","mla":"Beznosikov, Aleksandr, et al. “On Biased Compression for Distributed Learning.” Journal of Machine Learning Research, vol. 24, Journal of Machine Learning Research, 2023, pp. 1–50."},"date_published":"2023-10-01T00:00:00Z"},{"year":"2023","publisher":"Springer Nature","department":[{"_id":"DaAl"},{"_id":"GradSch"}],"publication_status":"published","related_material":{"record":[{"status":"public","relation":"research_data","id":"14995"}]},"author":[{"full_name":"Koval, Nikita","last_name":"Koval","first_name":"Nikita","id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87"},{"last_name":"Fedorov","first_name":"Alexander","id":"2e711909-896a-11ed-bdf8-eb0f5a2984c6","full_name":"Fedorov, Alexander"},{"full_name":"Sokolova, Maria","first_name":"Maria","last_name":"Sokolova"},{"last_name":"Tsitelov","first_name":"Dmitry","full_name":"Tsitelov, Dmitry"},{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian"}],"volume":13964,"date_updated":"2024-02-27T07:46:52Z","date_created":"2023-09-03T22:01:16Z","file_date_updated":"2023-09-06T08:16:25Z","oa":1,"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"quality_controlled":"1","doi":"10.1007/978-3-031-37706-8_8","conference":{"name":"CAV: Computer Aided Verification","start_date":"2023-07-17","location":"Paris, France","end_date":"2023-07-22"},"language":[{"iso":"eng"}],"publication_identifier":{"issn":["0302-9743"],"isbn":["9783031377051"],"eissn":["1611-3349"]},"month":"07","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"14260","intvolume":" 13964","title":"Lincheck: A practical framework for testing concurrent data structures on JVM","ddc":["000"],"status":"public","file":[{"file_name":"2023_LNCS_Koval.pdf","access_level":"open_access","creator":"dernst","file_size":421408,"content_type":"application/pdf","file_id":"14275","relation":"main_file","date_updated":"2023-09-06T08:16:25Z","date_created":"2023-09-06T08:16:25Z","success":1,"checksum":"c346016393123a0a2338ad4d976f61bc"}],"oa_version":"Published Version","type":"conference","alternative_title":["LNCS"],"abstract":[{"lang":"eng","text":"This paper presents Lincheck, a new practical and user-friendly framework for testing concurrent algorithms on the Java Virtual Machine (JVM). Lincheck provides a simple and declarative way to write concurrent tests: instead of describing how to perform the test, users specify what to test by declaring all the operations to examine; the framework automatically handles the rest. As a result, tests written with Lincheck are concise and easy to understand. The framework automatically generates a set of concurrent scenarios, examines them using stress-testing or bounded model checking, and verifies that the results of each invocation are correct. Notably, if an error is detected via model checking, Lincheck provides an easy-to-follow trace to reproduce it, significantly simplifying the bug investigation.\r\n\r\nTo the best of our knowledge, Lincheck is the first production-ready tool on the JVM that offers such a simple way of writing concurrent tests, without requiring special skills or expertise. We successfully integrated Lincheck in the development process of several large projects, such as Kotlin Coroutines, and identified new bugs in popular concurrency libraries, such as a race in Java’s standard ConcurrentLinkedDeque and a liveliness bug in Java’s AbstractQueuedSynchronizer framework, which is used in most of the synchronization primitives. We believe that Lincheck can significantly improve the quality and productivity of concurrent algorithms research and development and become the state-of-the-art tool for checking their correctness."}],"citation":{"chicago":"Koval, Nikita, Alexander Fedorov, Maria Sokolova, Dmitry Tsitelov, and Dan-Adrian Alistarh. “Lincheck: A Practical Framework for Testing Concurrent Data Structures on JVM.” In 35th International Conference on Computer Aided Verification , 13964:156–69. Springer Nature, 2023. https://doi.org/10.1007/978-3-031-37706-8_8.","mla":"Koval, Nikita, et al. “Lincheck: A Practical Framework for Testing Concurrent Data Structures on JVM.” 35th International Conference on Computer Aided Verification , vol. 13964, Springer Nature, 2023, pp. 156–69, doi:10.1007/978-3-031-37706-8_8.","short":"N. Koval, A. Fedorov, M. Sokolova, D. Tsitelov, D.-A. Alistarh, in:, 35th International Conference on Computer Aided Verification , Springer Nature, 2023, pp. 156–169.","ista":"Koval N, Fedorov A, Sokolova M, Tsitelov D, Alistarh D-A. 2023. Lincheck: A practical framework for testing concurrent data structures on JVM. 35th International Conference on Computer Aided Verification . CAV: Computer Aided Verification, LNCS, vol. 13964, 156–169.","apa":"Koval, N., Fedorov, A., Sokolova, M., Tsitelov, D., & Alistarh, D.-A. (2023). Lincheck: A practical framework for testing concurrent data structures on JVM. In 35th International Conference on Computer Aided Verification (Vol. 13964, pp. 156–169). Paris, France: Springer Nature. https://doi.org/10.1007/978-3-031-37706-8_8","ieee":"N. Koval, A. Fedorov, M. Sokolova, D. Tsitelov, and D.-A. Alistarh, “Lincheck: A practical framework for testing concurrent data structures on JVM,” in 35th International Conference on Computer Aided Verification , Paris, France, 2023, vol. 13964, pp. 156–169.","ama":"Koval N, Fedorov A, Sokolova M, Tsitelov D, Alistarh D-A. Lincheck: A practical framework for testing concurrent data structures on JVM. In: 35th International Conference on Computer Aided Verification . Vol 13964. Springer Nature; 2023:156-169. doi:10.1007/978-3-031-37706-8_8"},"publication":"35th International Conference on Computer Aided Verification ","page":"156-169","date_published":"2023-07-17T00:00:00Z","scopus_import":"1","has_accepted_license":"1","article_processing_charge":"Yes (in subscription journal)","day":"17"},{"day":"28","month":"04","article_processing_charge":"No","citation":{"ama":"Koval N, Fedorov A, Sokolova M, Tsitelov D, Alistarh D-A. Lincheck: A practical framework for testing concurrent data structures on JVM. 2023. doi:10.5281/ZENODO.7877757","ista":"Koval N, Fedorov A, Sokolova M, Tsitelov D, Alistarh D-A. 2023. Lincheck: A practical framework for testing concurrent data structures on JVM, Zenodo, 10.5281/ZENODO.7877757.","ieee":"N. Koval, A. Fedorov, M. Sokolova, D. Tsitelov, and D.-A. Alistarh, “Lincheck: A practical framework for testing concurrent data structures on JVM.” Zenodo, 2023.","apa":"Koval, N., Fedorov, A., Sokolova, M., Tsitelov, D., & Alistarh, D.-A. (2023). Lincheck: A practical framework for testing concurrent data structures on JVM. Zenodo. https://doi.org/10.5281/ZENODO.7877757","mla":"Koval, Nikita, et al. Lincheck: A Practical Framework for Testing Concurrent Data Structures on JVM. Zenodo, 2023, doi:10.5281/ZENODO.7877757.","short":"N. Koval, A. Fedorov, M. Sokolova, D. Tsitelov, D.-A. Alistarh, (2023).","chicago":"Koval, Nikita, Alexander Fedorov, Maria Sokolova, Dmitry Tsitelov, and Dan-Adrian Alistarh. “Lincheck: A Practical Framework for Testing Concurrent Data Structures on JVM.” Zenodo, 2023. https://doi.org/10.5281/ZENODO.7877757."},"oa":1,"main_file_link":[{"open_access":"1","url":"https://doi.org/10.5281/zenodo.7877757"}],"doi":"10.5281/ZENODO.7877757","date_published":"2023-04-28T00:00:00Z","type":"research_data_reference","abstract":[{"lang":"eng","text":"Lincheck is a new practical and user-friendly framework for testing concurrent data structures on the Java Virtual Machine (JVM). It provides a simple and declarative way to write concurrent tests. Instead of describing how to perform the test, users specify what to test by declaring all the operations to examine; the framework automatically handles the rest. As a result, tests written with Lincheck are concise and easy to understand. \r\nThe artifact presents a collection of Lincheck tests that discover new bugs in popular libraries and implementations from the concurrency literature -- they are listed in Table 1, Section 3. To evaluate the performance of Lincheck analysis, the collection of tests also includes those which check correct data structures and, thus, always succeed. Similarly to Table 2, Section 3, the experiments demonstrate the reasonable time to perform a test. Finally, Lincheck provides user-friendly output with an easy-to-follow trace to reproduce a detected error, significantly simplifying further investigation."}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"14995","year":"2023","status":"public","ddc":["000"],"title":"Lincheck: A practical framework for testing concurrent data structures on JVM","department":[{"_id":"DaAl"}],"publisher":"Zenodo","author":[{"last_name":"Koval","first_name":"Nikita","id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87","full_name":"Koval, Nikita"},{"id":"2e711909-896a-11ed-bdf8-eb0f5a2984c6","last_name":"Fedorov","first_name":"Alexander","full_name":"Fedorov, Alexander"},{"full_name":"Sokolova, Maria","first_name":"Maria","last_name":"Sokolova"},{"last_name":"Tsitelov","first_name":"Dmitry","full_name":"Tsitelov, Dmitry"},{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"}],"related_material":{"record":[{"status":"public","relation":"used_in_publication","id":"14260"}]},"date_created":"2024-02-14T15:14:13Z","date_updated":"2024-02-27T07:46:52Z","oa_version":"Published Version"},{"publication_identifier":{"isbn":["9783959772198"],"issn":["1868-8969"]},"month":"02","doi":"10.4230/LIPIcs.OPODIS.2021.14","conference":{"end_date":"2021-12-15","location":"Strasbourg, France","start_date":"2021-12-13","name":"OPODIS"},"language":[{"iso":"eng"}],"oa":1,"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"external_id":{"arxiv":["2102.08808"]},"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"},{"_id":"26A5D39A-B435-11E9-9278-68D0E5697425","grant_number":"840605","name":"Coordination in constrained and natural distributed systems","call_identifier":"H2020"}],"quality_controlled":"1","ec_funded":1,"file_date_updated":"2022-05-02T08:06:33Z","article_number":"14","author":[{"orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian"},{"last_name":"Gelashvili","first_name":"Rati","full_name":"Gelashvili, Rati"},{"id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0002-6432-6646","first_name":"Joel","last_name":"Rybicki","full_name":"Rybicki, Joel"}],"volume":217,"date_created":"2022-04-17T22:01:47Z","date_updated":"2022-05-02T08:09:39Z","acknowledgement":"Dan Alistarh: This project has received funding from the European Research Council (ERC)\r\nunder the European Union’s Horizon 2020 research and innovation programme (grant agreement No.805223 ScaleML).\r\nJoel Rybicki: This project has received from the European Union’s Horizon 2020 research and\r\ninnovation programme under the Marie Skłodowska-Curie grant agreement No. 840605.\r\nAcknowledgements We grateful to Giorgi Nadiradze for pointing out a generalisation of the phase clock construction to non-regular graphs. We also thank anonymous reviewers for their useful comments on earlier versions of this manuscript.","year":"2022","editor":[{"first_name":"Quentin","last_name":"Bramas","full_name":"Bramas, Quentin"},{"last_name":"Gramoli","first_name":"Vincent","full_name":"Gramoli, Vincent"},{"full_name":"Milani, Alessia","first_name":"Alessia","last_name":"Milani"}],"department":[{"_id":"DaAl"}],"publisher":"Schloss Dagstuhl - Leibniz-Zentrum für Informatik","publication_status":"published","article_processing_charge":"No","has_accepted_license":"1","day":"01","scopus_import":"1","date_published":"2022-02-01T00:00:00Z","citation":{"ama":"Alistarh D-A, Gelashvili R, Rybicki J. Fast graphical population protocols. In: Bramas Q, Gramoli V, Milani A, eds. 25th International Conference on Principles of Distributed Systems. Vol 217. Schloss Dagstuhl - Leibniz-Zentrum für Informatik; 2022. doi:10.4230/LIPIcs.OPODIS.2021.14","ista":"Alistarh D-A, Gelashvili R, Rybicki J. 2022. Fast graphical population protocols. 25th International Conference on Principles of Distributed Systems. OPODIS, LIPIcs, vol. 217, 14.","apa":"Alistarh, D.-A., Gelashvili, R., & Rybicki, J. (2022). Fast graphical population protocols. In Q. Bramas, V. Gramoli, & A. Milani (Eds.), 25th International Conference on Principles of Distributed Systems (Vol. 217). Strasbourg, France: Schloss Dagstuhl - Leibniz-Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.OPODIS.2021.14","ieee":"D.-A. Alistarh, R. Gelashvili, and J. Rybicki, “Fast graphical population protocols,” in 25th International Conference on Principles of Distributed Systems, Strasbourg, France, 2022, vol. 217.","mla":"Alistarh, Dan-Adrian, et al. “Fast Graphical Population Protocols.” 25th International Conference on Principles of Distributed Systems, edited by Quentin Bramas et al., vol. 217, 14, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2022, doi:10.4230/LIPIcs.OPODIS.2021.14.","short":"D.-A. Alistarh, R. Gelashvili, J. Rybicki, in:, Q. Bramas, V. Gramoli, A. Milani (Eds.), 25th International Conference on Principles of Distributed Systems, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2022.","chicago":"Alistarh, Dan-Adrian, Rati Gelashvili, and Joel Rybicki. “Fast Graphical Population Protocols.” In 25th International Conference on Principles of Distributed Systems, edited by Quentin Bramas, Vincent Gramoli, and Alessia Milani, Vol. 217. Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2022. https://doi.org/10.4230/LIPIcs.OPODIS.2021.14."},"publication":"25th International Conference on Principles of Distributed Systems","abstract":[{"lang":"eng","text":"Let G be a graph on n nodes. In the stochastic population protocol model, a collection of n indistinguishable, resource-limited nodes collectively solve tasks via pairwise interactions. In each interaction, two randomly chosen neighbors first read each other’s states, and then update their local states. A rich line of research has established tight upper and lower bounds on the complexity of fundamental tasks, such as majority and leader election, in this model, when G is a clique. Specifically, in the clique, these tasks can be solved fast, i.e., in n polylog n pairwise interactions, with high probability, using at most polylog n states per node.\r\nIn this work, we consider the more general setting where G is an arbitrary regular graph, and present a technique for simulating protocols designed for fully-connected networks in any connected regular graph. Our main result is a simulation that is efficient on many interesting graph families: roughly, the simulation overhead is polylogarithmic in the number of nodes, and quadratic in the conductance of the graph. As a sample application, we show that, in any regular graph with conductance φ, both leader election and exact majority can be solved in φ^{-2} ⋅ n polylog n pairwise interactions, with high probability, using at most φ^{-2} ⋅ polylog n states per node. This shows that there are fast and space-efficient population protocols for leader election and exact majority on graphs with good expansion properties. We believe our results will prove generally useful, as they allow efficient technology transfer between the well-mixed (clique) case, and the under-explored spatial setting."}],"type":"conference","alternative_title":["LIPIcs"],"file":[{"content_type":"application/pdf","file_size":959406,"creator":"dernst","access_level":"open_access","file_name":"2022_LIPICs_Alistarh.pdf","checksum":"2c7c982174c6f98c4ca6e92539d15086","success":1,"date_updated":"2022-05-02T08:06:33Z","date_created":"2022-05-02T08:06:33Z","relation":"main_file","file_id":"11346"}],"oa_version":"Published Version","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"11184","intvolume":" 217","title":"Fast graphical population protocols","ddc":["510"],"status":"public"},{"ddc":["510"],"status":"public","title":"Beyond distributed subgraph detection: Induced subgraphs, multicolored problems and graph parameters","intvolume":" 217","_id":"11183","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","oa_version":"Published Version","file":[{"file_name":"2022_LIPICs_Nikabadi.pdf","access_level":"open_access","creator":"dernst","content_type":"application/pdf","file_size":790396,"file_id":"11345","relation":"main_file","date_updated":"2022-05-02T07:53:00Z","date_created":"2022-05-02T07:53:00Z","success":1,"checksum":"626551c14de5d4091573200ed0535752"}],"alternative_title":["LIPIcs"],"type":"conference","abstract":[{"text":"Subgraph detection has recently been one of the most studied problems in the CONGEST model of distributed computing. In this work, we study the distributed complexity of problems closely related to subgraph detection, mainly focusing on induced subgraph detection. The main line of this work presents lower bounds and parameterized algorithms w.r.t structural parameters of the input graph:\r\n- On general graphs, we give unconditional lower bounds for induced detection of cycles and patterns of treewidth 2 in CONGEST. Moreover, by adapting reductions from centralized parameterized complexity, we prove lower bounds in CONGEST for detecting patterns with a 4-clique, and for induced path detection conditional on the hardness of triangle detection in the congested clique.\r\n- On graphs of bounded degeneracy, we show that induced paths can be detected fast in CONGEST using techniques from parameterized algorithms, while detecting cycles and patterns of treewidth 2 is hard.\r\n- On graphs of bounded vertex cover number, we show that induced subgraph detection is easy in CONGEST for any pattern graph. More specifically, we adapt a centralized parameterized algorithm for a more general maximum common induced subgraph detection problem to the distributed setting. In addition to these induced subgraph detection results, we study various related problems in the CONGEST and congested clique models, including for multicolored versions of subgraph-detection-like problems.","lang":"eng"}],"publication":"25th International Conference on Principles of Distributed Systems","citation":{"ista":"Nikabadi A, Korhonen J. 2022. Beyond distributed subgraph detection: Induced subgraphs, multicolored problems and graph parameters. 25th International Conference on Principles of Distributed Systems. OPODIS, LIPIcs, vol. 217, 15.","ieee":"A. Nikabadi and J. Korhonen, “Beyond distributed subgraph detection: Induced subgraphs, multicolored problems and graph parameters,” in 25th International Conference on Principles of Distributed Systems, Strasbourg, France, 2022, vol. 217.","apa":"Nikabadi, A., & Korhonen, J. (2022). Beyond distributed subgraph detection: Induced subgraphs, multicolored problems and graph parameters. In Q. Bramas, V. Gramoli, & A. Milani (Eds.), 25th International Conference on Principles of Distributed Systems (Vol. 217). Strasbourg, France: Schloss Dagstuhl - Leibniz-Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.OPODIS.2021.15","ama":"Nikabadi A, Korhonen J. Beyond distributed subgraph detection: Induced subgraphs, multicolored problems and graph parameters. In: Bramas Q, Gramoli V, Milani A, eds. 25th International Conference on Principles of Distributed Systems. Vol 217. Schloss Dagstuhl - Leibniz-Zentrum für Informatik; 2022. doi:10.4230/LIPIcs.OPODIS.2021.15","chicago":"Nikabadi, Amir, and Janne Korhonen. “Beyond Distributed Subgraph Detection: Induced Subgraphs, Multicolored Problems and Graph Parameters.” In 25th International Conference on Principles of Distributed Systems, edited by Quentin Bramas, Vincent Gramoli, and Alessia Milani, Vol. 217. Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2022. https://doi.org/10.4230/LIPIcs.OPODIS.2021.15.","mla":"Nikabadi, Amir, and Janne Korhonen. “Beyond Distributed Subgraph Detection: Induced Subgraphs, Multicolored Problems and Graph Parameters.” 25th International Conference on Principles of Distributed Systems, edited by Quentin Bramas et al., vol. 217, 15, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2022, doi:10.4230/LIPIcs.OPODIS.2021.15.","short":"A. Nikabadi, J. Korhonen, in:, Q. Bramas, V. Gramoli, A. Milani (Eds.), 25th International Conference on Principles of Distributed Systems, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2022."},"date_published":"2022-02-01T00:00:00Z","scopus_import":"1","day":"01","has_accepted_license":"1","article_processing_charge":"No","publication_status":"published","editor":[{"full_name":"Bramas, Quentin","first_name":"Quentin","last_name":"Bramas"},{"full_name":"Gramoli, Vincent","last_name":"Gramoli","first_name":"Vincent"},{"full_name":"Milani, Alessia","last_name":"Milani","first_name":"Alessia"}],"department":[{"_id":"DaAl"}],"publisher":"Schloss Dagstuhl - Leibniz-Zentrum für Informatik","acknowledgement":"Amir Nikabadi: Supported by the LABEX MILYON (ANR-10-LABX-0070) of Université de Lyon, within the program “Investissements d’Avenir” (ANR-11-IDEX-0007) operated by the French National Research Agency (ANR). Janne H. Korhonen: Supported by the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML).\r\nWe thank François Le Gall and Masayuki Miyamoto for sharing their work on lower bounds for induced subgraph detection [36].","year":"2022","date_updated":"2022-05-02T07:56:35Z","date_created":"2022-04-17T22:01:47Z","volume":217,"author":[{"full_name":"Nikabadi, Amir","last_name":"Nikabadi","first_name":"Amir"},{"full_name":"Korhonen, Janne","first_name":"Janne","last_name":"Korhonen","id":"C5402D42-15BC-11E9-A202-CA2BE6697425"}],"article_number":"15","file_date_updated":"2022-05-02T07:53:00Z","ec_funded":1,"quality_controlled":"1","project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"language":[{"iso":"eng"}],"conference":{"name":"OPODIS","location":"Strasbourg, France","start_date":"2021-12-13","end_date":"2021-12-15"},"doi":"10.4230/LIPIcs.OPODIS.2021.15","month":"02","publication_identifier":{"issn":["1868-8969"],"isbn":["9783959772198"]}},{"scopus_import":"1","day":"01","has_accepted_license":"1","article_processing_charge":"No","publication":"Journal of Machine Learning Research","citation":{"apa":"Shevchenko, A., Kungurtsev, V., & Mondelli, M. (2022). Mean-field analysis of piecewise linear solutions for wide ReLU networks. Journal of Machine Learning Research. Journal of Machine Learning Research.","ieee":"A. Shevchenko, V. Kungurtsev, and M. Mondelli, “Mean-field analysis of piecewise linear solutions for wide ReLU networks,” Journal of Machine Learning Research, vol. 23, no. 130. Journal of Machine Learning Research, pp. 1–55, 2022.","ista":"Shevchenko A, Kungurtsev V, Mondelli M. 2022. Mean-field analysis of piecewise linear solutions for wide ReLU networks. Journal of Machine Learning Research. 23(130), 1–55.","ama":"Shevchenko A, Kungurtsev V, Mondelli M. Mean-field analysis of piecewise linear solutions for wide ReLU networks. Journal of Machine Learning Research. 2022;23(130):1-55.","chicago":"Shevchenko, Aleksandr, Vyacheslav Kungurtsev, and Marco Mondelli. “Mean-Field Analysis of Piecewise Linear Solutions for Wide ReLU Networks.” Journal of Machine Learning Research. Journal of Machine Learning Research, 2022.","short":"A. Shevchenko, V. Kungurtsev, M. Mondelli, Journal of Machine Learning Research 23 (2022) 1–55.","mla":"Shevchenko, Aleksandr, et al. “Mean-Field Analysis of Piecewise Linear Solutions for Wide ReLU Networks.” Journal of Machine Learning Research, vol. 23, no. 130, Journal of Machine Learning Research, 2022, pp. 1–55."},"article_type":"original","page":"1-55","date_published":"2022-04-01T00:00:00Z","type":"journal_article","abstract":[{"lang":"eng","text":"Understanding the properties of neural networks trained via stochastic gradient descent (SGD) is at the heart of the theory of deep learning. In this work, we take a mean-field view, and consider a two-layer ReLU network trained via noisy-SGD for a univariate regularized regression problem. Our main result is that SGD with vanishingly small noise injected in the gradients is biased towards a simple solution: at convergence, the ReLU network implements a piecewise linear map of the inputs, and the number of “knot” points -- i.e., points where the tangent of the ReLU network estimator changes -- between two consecutive training inputs is at most three. In particular, as the number of neurons of the network grows, the SGD dynamics is captured by the solution of a gradient flow and, at convergence, the distribution of the weights approaches the unique minimizer of a related free energy, which has a Gibbs form. Our key technical contribution consists in the analysis of the estimator resulting from this minimizer: we show that its second derivative vanishes everywhere, except at some specific locations which represent the “knot” points. We also provide empirical evidence that knots at locations distinct from the data points might occur, as predicted by our theory."}],"issue":"130","_id":"11420","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","ddc":["000"],"status":"public","title":"Mean-field analysis of piecewise linear solutions for wide ReLU networks","intvolume":" 23","file":[{"relation":"main_file","file_id":"11422","date_updated":"2022-05-30T08:22:55Z","date_created":"2022-05-30T08:22:55Z","checksum":"d4ff5d1affb34848b5c5e4002483fc62","success":1,"file_name":"21-1365.pdf","access_level":"open_access","content_type":"application/pdf","file_size":1521701,"creator":"cchlebak"}],"oa_version":"Published Version","month":"04","publication_identifier":{"issn":["1532-4435"],"eissn":["1533-7928"]},"external_id":{"arxiv":["2111.02278"]},"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"quality_controlled":"1","project":[{"name":"Prix Lopez-Loretta 2019 - Marco Mondelli","_id":"059876FA-7A3F-11EA-A408-12923DDC885E"}],"language":[{"iso":"eng"}],"file_date_updated":"2022-05-30T08:22:55Z","year":"2022","acknowledgement":"We would like to thank Mert Pilanci for several exploratory discussions in the early stage\r\nof the project, Jan Maas for clarifications about Jordan et al. (1998), and Max Zimmer for\r\nsuggestive numerical experiments. A. Shevchenko and M. Mondelli are partially supported\r\nby the 2019 Lopez-Loreta Prize. V. Kungurtsev acknowledges support to the OP VVV\r\nproject CZ.02.1.01/0.0/0.0/16 019/0000765 Research Center for Informatics.\r\n","publication_status":"published","department":[{"_id":"MaMo"},{"_id":"DaAl"}],"publisher":"Journal of Machine Learning Research","author":[{"first_name":"Aleksandr","last_name":"Shevchenko","id":"F2B06EC2-C99E-11E9-89F0-752EE6697425","full_name":"Shevchenko, Aleksandr"},{"full_name":"Kungurtsev, Vyacheslav","first_name":"Vyacheslav","last_name":"Kungurtsev"},{"last_name":"Mondelli","first_name":"Marco","orcid":"0000-0002-3242-7020","id":"27EB676C-8706-11E9-9510-7717E6697425","full_name":"Mondelli, Marco"}],"related_material":{"link":[{"relation":"other","url":"https://www.jmlr.org/papers/v23/21-1365.html"}]},"date_updated":"2022-05-30T08:34:14Z","date_created":"2022-05-29T22:01:54Z","volume":23},{"oa_version":"Published Version","file":[{"date_created":"2023-01-27T06:58:02Z","date_updated":"2023-01-27T06:58:02Z","checksum":"11bbb56f68a00f2cf6bcce6cc7f5c5f9","success":1,"relation":"main_file","file_id":"12409","content_type":"application/pdf","file_size":524804,"creator":"dernst","file_name":"2022_LIPICs_Pacut.pdf","access_level":"open_access"}],"_id":"12182","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","intvolume":" 246","ddc":["000"],"title":"Brief announcement: Temporal locality in online algorithms","status":"public","abstract":[{"lang":"eng","text":"Online algorithms make decisions based on past inputs, with the goal of being competitive against an algorithm that sees also future inputs. In this work, we introduce time-local online algorithms; these are online algorithms in which the output at any given time is a function of only T latest inputs. Our main observation is that time-local online algorithms are closely connected to local distributed graph algorithms: distributed algorithms make decisions based on the local information in the spatial dimension, while time-local online algorithms make decisions based on the local information in the temporal dimension. We formalize this connection, and show how we can directly use the tools developed to study distributed approximability of graph optimization problems to prove upper and lower bounds on the competitive ratio achieved with time-local online algorithms. Moreover, we show how to use computational techniques to synthesize optimal time-local algorithms."}],"type":"conference","date_published":"2022-10-17T00:00:00Z","citation":{"ama":"Pacut M, Parham M, Rybicki J, Schmid S, Suomela J, Tereshchenko A. Brief announcement: Temporal locality in online algorithms. In: 36th International Symposium on Distributed Computing. Vol 246. Schloss Dagstuhl - Leibniz-Zentrum für Informatik; 2022. doi:10.4230/LIPIcs.DISC.2022.52","ista":"Pacut M, Parham M, Rybicki J, Schmid S, Suomela J, Tereshchenko A. 2022. Brief announcement: Temporal locality in online algorithms. 36th International Symposium on Distributed Computing. DISC: Symposium on Distributed Computing vol. 246, 52.","ieee":"M. Pacut, M. Parham, J. Rybicki, S. Schmid, J. Suomela, and A. Tereshchenko, “Brief announcement: Temporal locality in online algorithms,” in 36th International Symposium on Distributed Computing, Augusta, GA, United States, 2022, vol. 246.","apa":"Pacut, M., Parham, M., Rybicki, J., Schmid, S., Suomela, J., & Tereshchenko, A. (2022). Brief announcement: Temporal locality in online algorithms. In 36th International Symposium on Distributed Computing (Vol. 246). Augusta, GA, United States: Schloss Dagstuhl - Leibniz-Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.DISC.2022.52","mla":"Pacut, Maciej, et al. “Brief Announcement: Temporal Locality in Online Algorithms.” 36th International Symposium on Distributed Computing, vol. 246, 52, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2022, doi:10.4230/LIPIcs.DISC.2022.52.","short":"M. Pacut, M. Parham, J. Rybicki, S. Schmid, J. Suomela, A. Tereshchenko, in:, 36th International Symposium on Distributed Computing, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2022.","chicago":"Pacut, Maciej, Mahmoud Parham, Joel Rybicki, Stefan Schmid, Jukka Suomela, and Aleksandr Tereshchenko. “Brief Announcement: Temporal Locality in Online Algorithms.” In 36th International Symposium on Distributed Computing, Vol. 246. Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2022. https://doi.org/10.4230/LIPIcs.DISC.2022.52."},"publication":"36th International Symposium on Distributed Computing","article_processing_charge":"No","has_accepted_license":"1","day":"17","scopus_import":"1","author":[{"last_name":"Pacut","first_name":"Maciej","full_name":"Pacut, Maciej"},{"first_name":"Mahmoud","last_name":"Parham","full_name":"Parham, Mahmoud"},{"full_name":"Rybicki, Joel","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0002-6432-6646","first_name":"Joel","last_name":"Rybicki"},{"last_name":"Schmid","first_name":"Stefan","full_name":"Schmid, Stefan"},{"full_name":"Suomela, Jukka","last_name":"Suomela","first_name":"Jukka"},{"last_name":"Tereshchenko","first_name":"Aleksandr","full_name":"Tereshchenko, Aleksandr"}],"volume":246,"date_created":"2023-01-13T11:06:28Z","date_updated":"2023-01-27T06:59:29Z","acknowledgement":"This research has received funding from the German Research Foundation (DFG), grant\r\n470029389 (FlexNets), 2021-2024, and the Marie Skłodowska-Curie grant agreement No. 840605.","year":"2022","department":[{"_id":"DaAl"}],"publisher":"Schloss Dagstuhl - Leibniz-Zentrum für Informatik","publication_status":"published","ec_funded":1,"file_date_updated":"2023-01-27T06:58:02Z","article_number":"52","doi":"10.4230/LIPIcs.DISC.2022.52","conference":{"name":"DISC: Symposium on Distributed Computing","location":"Augusta, GA, United States","start_date":"2022-10-25","end_date":"2022-10-27"},"language":[{"iso":"eng"}],"oa":1,"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"project":[{"grant_number":"840605","_id":"26A5D39A-B435-11E9-9278-68D0E5697425","name":"Coordination in constrained and natural distributed systems","call_identifier":"H2020"}],"quality_controlled":"1","publication_identifier":{"eisbn":["9783959772556"],"eissn":["1868-8969"]},"month":"10"},{"file_date_updated":"2023-04-03T06:17:58Z","year":"2022","acknowledgement":"The authors sincerely thank Nikoli Dryden, Tal Ben-Nun, Torsten Hoefler and Bapi Chatterjee for useful discussions throughout the development of this project.","publication_status":"published","publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"author":[{"id":"D0CF4148-C985-11E9-8066-0BDEE5697425","last_name":"Markov","first_name":"Ilia","full_name":"Markov, Ilia"},{"full_name":"Ramezanikebrya, Hamidreza","first_name":"Hamidreza","last_name":"Ramezanikebrya"},{"first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","full_name":"Alistarh, Dan-Adrian"}],"date_created":"2023-03-31T06:17:00Z","date_updated":"2023-04-03T06:21:04Z","month":"11","publication_identifier":{"isbn":["9781450393409"]},"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"external_id":{"arxiv":["2111.08617"]},"oa":1,"quality_controlled":"1","conference":{"name":"Middleware: International Middleware Conference","end_date":"2022-11-11","location":"Quebec, QC, Canada","start_date":"2022-11-07"},"doi":"10.1145/3528535.3565248","language":[{"iso":"eng"}],"type":"conference","abstract":[{"lang":"eng","text":"The ability to scale out training workloads has been one of the key performance enablers of deep learning. The main scaling approach is data-parallel GPU-based training, which has been boosted by hardware and software support for highly efficient point-to-point communication, and in particular via hardware bandwidth over-provisioning. Overprovisioning comes at a cost: there is an order of magnitude price difference between \"cloud-grade\" servers with such support, relative to their popular \"consumer-grade\" counterparts, although single server-grade and consumer-grade GPUs can have similar computational envelopes.\r\n\r\nIn this paper, we show that the costly hardware overprovisioning approach can be supplanted via algorithmic and system design, and propose a framework called CGX, which provides efficient software support for compressed communication in ML applications, for both multi-GPU single-node training, as well as larger-scale multi-node training. CGX is based on two technical advances: At the system level, it relies on a re-developed communication stack for ML frameworks, which provides flexible, highly-efficient support for compressed communication. At the application level, it provides seamless, parameter-free integration with popular frameworks, so that end-users do not have to modify training recipes, nor significant training code. This is complemented by a layer-wise adaptive compression technique which dynamically balances compression gains with accuracy preservation. CGX integrates with popular ML frameworks, providing up to 3X speedups for multi-GPU nodes based on commodity hardware, and order-of-magnitude improvements in the multi-node setting, with negligible impact on accuracy."}],"_id":"12780","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","status":"public","ddc":["000"],"title":"CGX: Adaptive system support for communication-efficient deep learning","file":[{"date_updated":"2023-04-03T06:17:58Z","date_created":"2023-04-03T06:17:58Z","success":1,"checksum":"1a397746235f245da5468819247ff663","file_id":"12795","relation":"main_file","creator":"dernst","content_type":"application/pdf","file_size":1514169,"file_name":"2022_ACMMiddleware_Markov.pdf","access_level":"open_access"}],"oa_version":"Published Version","day":"01","has_accepted_license":"1","article_processing_charge":"Yes (via OA deal)","publication":"Proceedings of the 23rd ACM/IFIP International Middleware Conference","citation":{"ama":"Markov I, Ramezanikebrya H, Alistarh D-A. CGX: Adaptive system support for communication-efficient deep learning. In: Proceedings of the 23rd ACM/IFIP International Middleware Conference. Association for Computing Machinery; 2022:241-254. doi:10.1145/3528535.3565248","ista":"Markov I, Ramezanikebrya H, Alistarh D-A. 2022. CGX: Adaptive system support for communication-efficient deep learning. Proceedings of the 23rd ACM/IFIP International Middleware Conference. Middleware: International Middleware Conference, 241–254.","ieee":"I. Markov, H. Ramezanikebrya, and D.-A. Alistarh, “CGX: Adaptive system support for communication-efficient deep learning,” in Proceedings of the 23rd ACM/IFIP International Middleware Conference, Quebec, QC, Canada, 2022, pp. 241–254.","apa":"Markov, I., Ramezanikebrya, H., & Alistarh, D.-A. (2022). CGX: Adaptive system support for communication-efficient deep learning. In Proceedings of the 23rd ACM/IFIP International Middleware Conference (pp. 241–254). Quebec, QC, Canada: Association for Computing Machinery. https://doi.org/10.1145/3528535.3565248","mla":"Markov, Ilia, et al. “CGX: Adaptive System Support for Communication-Efficient Deep Learning.” Proceedings of the 23rd ACM/IFIP International Middleware Conference, Association for Computing Machinery, 2022, pp. 241–54, doi:10.1145/3528535.3565248.","short":"I. Markov, H. Ramezanikebrya, D.-A. Alistarh, in:, Proceedings of the 23rd ACM/IFIP International Middleware Conference, Association for Computing Machinery, 2022, pp. 241–254.","chicago":"Markov, Ilia, Hamidreza Ramezanikebrya, and Dan-Adrian Alistarh. “CGX: Adaptive System Support for Communication-Efficient Deep Learning.” In Proceedings of the 23rd ACM/IFIP International Middleware Conference, 241–54. Association for Computing Machinery, 2022. https://doi.org/10.1145/3528535.3565248."},"page":"241-254","date_published":"2022-11-01T00:00:00Z"},{"date_published":"2022-07-21T00:00:00Z","page":"246-256","citation":{"ama":"Alistarh D-A, Rybicki J, Voitovych S. Near-optimal leader election in population protocols on graphs. In: Proceedings of the Annual ACM Symposium on Principles of Distributed Computing. Association for Computing Machinery; 2022:246-256. doi:10.1145/3519270.3538435","apa":"Alistarh, D.-A., Rybicki, J., & Voitovych, S. (2022). Near-optimal leader election in population protocols on graphs. In Proceedings of the Annual ACM Symposium on Principles of Distributed Computing (pp. 246–256). Salerno, Italy: Association for Computing Machinery. https://doi.org/10.1145/3519270.3538435","ieee":"D.-A. Alistarh, J. Rybicki, and S. Voitovych, “Near-optimal leader election in population protocols on graphs,” in Proceedings of the Annual ACM Symposium on Principles of Distributed Computing, Salerno, Italy, 2022, pp. 246–256.","ista":"Alistarh D-A, Rybicki J, Voitovych S. 2022. Near-optimal leader election in population protocols on graphs. Proceedings of the Annual ACM Symposium on Principles of Distributed Computing. PODC: Symposium on Principles of Distributed Computing, 246–256.","short":"D.-A. Alistarh, J. Rybicki, S. Voitovych, in:, Proceedings of the Annual ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2022, pp. 246–256.","mla":"Alistarh, Dan-Adrian, et al. “Near-Optimal Leader Election in Population Protocols on Graphs.” Proceedings of the Annual ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2022, pp. 246–56, doi:10.1145/3519270.3538435.","chicago":"Alistarh, Dan-Adrian, Joel Rybicki, and Sasha Voitovych. “Near-Optimal Leader Election in Population Protocols on Graphs.” In Proceedings of the Annual ACM Symposium on Principles of Distributed Computing, 246–56. Association for Computing Machinery, 2022. https://doi.org/10.1145/3519270.3538435."},"publication":"Proceedings of the Annual ACM Symposium on Principles of Distributed Computing","article_processing_charge":"Yes (via OA deal)","has_accepted_license":"1","day":"21","scopus_import":"1","file":[{"creator":"cchlebak","file_size":1593474,"content_type":"application/pdf","file_name":"2022_PODC_Alistarh.pdf","access_level":"open_access","date_updated":"2022-08-16T08:05:15Z","date_created":"2022-08-16T08:05:15Z","success":1,"checksum":"4c6b29172b8e355b4fbc364a2e0827b2","file_id":"11854","relation":"main_file"}],"oa_version":"Published Version","status":"public","ddc":["000"],"title":"Near-optimal leader election in population protocols on graphs","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"11844","abstract":[{"lang":"eng","text":"In the stochastic population protocol model, we are given a connected graph with n nodes, and in every time step, a scheduler samples an edge of the graph uniformly at random and the nodes connected by this edge interact. A fundamental task in this model is stable leader election, in which all nodes start in an identical state and the aim is to reach a configuration in which (1) exactly one node is elected as leader and (2) this node remains as the unique leader no matter what sequence of interactions follows. On cliques, the complexity of this problem has recently been settled: time-optimal protocols stabilize in Θ(n log n) expected steps using Θ(log log n) states, whereas protocols that use O(1) states require Θ(n2) expected steps.\r\n\r\nIn this work, we investigate the complexity of stable leader election on general graphs. We provide the first non-trivial time lower bounds for leader election on general graphs, showing that, when moving beyond cliques, the complexity landscape of leader election becomes very diverse: the time required to elect a leader can range from O(1) to Θ(n3) expected steps. On the upper bound side, we first observe that there exists a protocol that is time-optimal on many graph families, but uses polynomially-many states. In contrast, we give a near-time-optimal protocol that uses only O(log2n) states that is at most a factor log n slower. Finally, we show that the constant-state protocol of Beauquier et al. [OPODIS 2013] is at most a factor n log n slower than the fast polynomial-state protocol. Moreover, among constant-state protocols, this protocol has near-optimal average case complexity on dense random graphs."}],"type":"conference","language":[{"iso":"eng"}],"doi":"10.1145/3519270.3538435","conference":{"end_date":"2022-07-29","start_date":"2022-07-25","location":"Salerno, Italy","name":"PODC: Symposium on Principles of Distributed Computing"},"project":[{"grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425","name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020"}],"quality_controlled":"1","tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"external_id":{"arxiv":["2205.12597"]},"publication_identifier":{"isbn":["9781450392624"]},"month":"07","date_updated":"2023-06-14T12:06:01Z","date_created":"2022-08-14T22:01:46Z","author":[{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian"},{"full_name":"Rybicki, Joel","orcid":"0000-0002-6432-6646","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","last_name":"Rybicki","first_name":"Joel"},{"first_name":"Sasha","last_name":"Voitovych","full_name":"Voitovych, Sasha"}],"publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"publication_status":"published","acknowledgement":"We thank the anonymous reviewers for their helpful comments. We gratefully acknowledge funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML).","year":"2022","ec_funded":1,"file_date_updated":"2022-08-16T08:05:15Z"},{"publication_identifier":{"isbn":["9781450392044"]},"month":"04","oa":1,"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"external_id":{"isi":["000883318200027"]},"isi":1,"quality_controlled":"1","doi":"10.1145/3503221.3508410","conference":{"end_date":"2022-04-06","start_date":"2022-04-02","location":"Seoul, Republic of Korea","name":"PPoPP: Sympopsium on Principles and Practice of Parallel Programming"},"language":[{"iso":"eng"}],"file_date_updated":"2022-08-05T09:19:29Z","acknowledgement":"This work was supported by: the Natural Sciences and Engineering Research Council of Canada (NSERC) Collaborative Research and Development grant: CRDPJ 539431-19, the\r\nCanada Foundation for Innovation John R. Evans Leaders Fund with equal support from the Ontario Research Fund CFI Leaders Opportunity Fund: 38512, Waterloo Huawei Joint Innovation Lab project “Scalable Infrastructure for Next Generation Data Management Systems”, NSERC Discovery Launch Supplement: DGECR-2019-00048, NSERC Discovery\r\nProgram under the grants: RGPIN-2019-04227 and RGPIN04512-2018, and the University of Waterloo. We would also like to thank the reviewers for their insightful comments.","year":"2022","publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"publication_status":"published","author":[{"last_name":"Brown","first_name":"Trevor A","id":"3569F0A0-F248-11E8-B48F-1D18A9856A87","full_name":"Brown, Trevor A"},{"first_name":"William","last_name":"Sigouin","full_name":"Sigouin, William"},{"orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian"}],"date_created":"2022-04-17T22:01:46Z","date_updated":"2023-08-03T06:49:20Z","scopus_import":"1","article_processing_charge":"No","has_accepted_license":"1","day":"02","citation":{"chicago":"Brown, Trevor A, William Sigouin, and Dan-Adrian Alistarh. “PathCAS: An Efficient Middle Ground for Concurrent Search Data Structures.” In Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, 385–99. Association for Computing Machinery, 2022. https://doi.org/10.1145/3503221.3508410.","short":"T.A. Brown, W. Sigouin, D.-A. Alistarh, in:, Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2022, pp. 385–399.","mla":"Brown, Trevor A., et al. “PathCAS: An Efficient Middle Ground for Concurrent Search Data Structures.” Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2022, pp. 385–99, doi:10.1145/3503221.3508410.","apa":"Brown, T. A., Sigouin, W., & Alistarh, D.-A. (2022). PathCAS: An efficient middle ground for concurrent search data structures. In Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (pp. 385–399). Seoul, Republic of Korea: Association for Computing Machinery. https://doi.org/10.1145/3503221.3508410","ieee":"T. A. Brown, W. Sigouin, and D.-A. Alistarh, “PathCAS: An efficient middle ground for concurrent search data structures,” in Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Seoul, Republic of Korea, 2022, pp. 385–399.","ista":"Brown TA, Sigouin W, Alistarh D-A. 2022. PathCAS: An efficient middle ground for concurrent search data structures. Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. PPoPP: Sympopsium on Principles and Practice of Parallel Programming, 385–399.","ama":"Brown TA, Sigouin W, Alistarh D-A. PathCAS: An efficient middle ground for concurrent search data structures. In: Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. Association for Computing Machinery; 2022:385-399. doi:10.1145/3503221.3508410"},"publication":"Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming","page":"385-399","date_published":"2022-04-02T00:00:00Z","type":"conference","abstract":[{"lang":"eng","text":"To maximize the performance of concurrent data structures, researchers have often turned to highly complex fine-grained techniques, resulting in efficient and elegant algorithms, which can however be often difficult to understand and prove correct. While simpler techniques exist, such as transactional memory, they can have limited performance or portability relative to their fine-grained counterparts. Approaches at both ends of this complexity-performance spectrum have been extensively explored, but relatively less is known about the middle ground: approaches that are willing to sacrifice some performance for simplicity, while remaining competitive with state-of-the-art handcrafted designs. In this paper, we explore this middle ground, and present PathCAS, a primitive that combines ideas from multi-word CAS (KCAS) and transactional memory approaches, while carefully avoiding overhead. We show how PathCAS can be used to implement efficient search data structures relatively simply, using an internal binary search tree as an example, then extending this to an AVL tree. Our best implementations outperform many handcrafted search trees: in search-heavy workloads, it rivals the BCCO tree [5], the fastest known concurrent binary tree in terms of search performance [3]. Our results suggest that PathCAS can yield concurrent data structures that are relatively easy to build and prove correct, while offering surprisingly high performance."}],"_id":"11181","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","ddc":["000"],"title":"PathCAS: An efficient middle ground for concurrent search data structures","status":"public","file":[{"file_id":"11731","relation":"main_file","date_created":"2022-08-05T09:19:29Z","date_updated":"2022-08-05T09:19:29Z","success":1,"checksum":"8ceea411fa133795cd4903529498eb6b","file_name":"2022_PPoPP_Brown.pdf","access_level":"open_access","creator":"dernst","content_type":"application/pdf","file_size":1128343}],"oa_version":"Published Version"},{"page":"353-367","publication":"Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming","citation":{"short":"A. Postnikova, N. Koval, G. Nadiradze, D.-A. Alistarh, in:, Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2022, pp. 353–367.","mla":"Postnikova, Anastasiia, et al. “Multi-Queues Can Be State-of-the-Art Priority Schedulers.” Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2022, pp. 353–67, doi:10.1145/3503221.3508432.","chicago":"Postnikova, Anastasiia, Nikita Koval, Giorgi Nadiradze, and Dan-Adrian Alistarh. “Multi-Queues Can Be State-of-the-Art Priority Schedulers.” In Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, 353–67. Association for Computing Machinery, 2022. https://doi.org/10.1145/3503221.3508432.","ama":"Postnikova A, Koval N, Nadiradze G, Alistarh D-A. Multi-queues can be state-of-the-art priority schedulers. In: Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. Association for Computing Machinery; 2022:353-367. doi:10.1145/3503221.3508432","ieee":"A. Postnikova, N. Koval, G. Nadiradze, and D.-A. Alistarh, “Multi-queues can be state-of-the-art priority schedulers,” in Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Seoul, Republic of Korea, 2022, pp. 353–367.","apa":"Postnikova, A., Koval, N., Nadiradze, G., & Alistarh, D.-A. (2022). Multi-queues can be state-of-the-art priority schedulers. In Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (pp. 353–367). Seoul, Republic of Korea: Association for Computing Machinery. https://doi.org/10.1145/3503221.3508432","ista":"Postnikova A, Koval N, Nadiradze G, Alistarh D-A. 2022. Multi-queues can be state-of-the-art priority schedulers. Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. PPoPP: Sympopsium on Principles and Practice of Parallel Programming, 353–367."},"date_published":"2022-04-02T00:00:00Z","scopus_import":"1","day":"02","article_processing_charge":"No","status":"public","title":"Multi-queues can be state-of-the-art priority schedulers","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","_id":"11180","oa_version":"Preprint","type":"conference","abstract":[{"text":"Designing and implementing efficient parallel priority schedulers is an active research area. An intriguing proposed design is the Multi-Queue: given n threads and m ≥ n distinct priority queues, task insertions are performed uniformly at random, while, to delete, a thread picks two queues uniformly at random, and removes the observed task of higher priority. This approach scales well, and has probabilistic rank guarantees: roughly, the rank of each task removed, relative to remaining tasks in all other queues, is O (m) in expectation. Yet, the performance of this pattern is below that of well-engineered schedulers, which eschew theoretical guarantees for practical efficiency.\r\n\r\nWe investigate whether it is possible to design and implement a Multi-Queue-based task scheduler that is both highly-efficient and has analytical guarantees. We propose a new variant called the Stealing Multi-Queue (SMQ), a cache-efficient variant of the Multi-Queue, which leverages both queue affinity---each thread has a local queue, from which tasks are usually removed; but, with some probability, threads also attempt to steal higher-priority tasks from the other queues---and task batching, that is, the processing of several tasks in a single insert / remove step. These ideas are well-known for task scheduling without priorities; our theoretical contribution is showing that, despite relaxations, this design can still provide rank guarantees, which in turn implies bounds on total work performed. We provide a general SMQ implementation which can surpass state-of-the-art schedulers such as OBIM and PMOD in terms of performance on popular graph-processing benchmarks. Notably, the performance improvement comes mainly from the superior rank guarantees provided by our scheduler, confirming that analytically-reasoned approaches can still provide performance improvements for priority task scheduling.","lang":"eng"}],"isi":1,"quality_controlled":"1","project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"external_id":{"isi":["000883318200025"],"arxiv":["2109.00657"]},"main_file_link":[{"url":" https://doi.org/10.48550/arXiv.2109.00657","open_access":"1"}],"oa":1,"language":[{"iso":"eng"}],"conference":{"name":"PPoPP: Sympopsium on Principles and Practice of Parallel Programming","start_date":"2022-04-02","location":"Seoul, Republic of Korea","end_date":"2022-04-06"},"doi":"10.1145/3503221.3508432","month":"04","publication_identifier":{"isbn":["9781450392044"]},"publication_status":"published","publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"acknowledgement":"We would like to thank the anonymous reviewers for their useful comments. This project has received funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML).","year":"2022","date_updated":"2023-08-03T06:48:35Z","date_created":"2022-04-17T22:01:46Z","author":[{"full_name":"Postnikova, Anastasiia","last_name":"Postnikova","first_name":"Anastasiia"},{"id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87","last_name":"Koval","first_name":"Nikita","full_name":"Koval, Nikita"},{"id":"3279A00C-F248-11E8-B48F-1D18A9856A87","first_name":"Giorgi","last_name":"Nadiradze","full_name":"Nadiradze, Giorgi"},{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"}],"related_material":{"record":[{"status":"public","relation":"research_data","id":"13076"}]},"ec_funded":1},{"month":"01","day":"03","article_processing_charge":"No","date_published":"2022-01-03T00:00:00Z","doi":"10.5281/ZENODO.5733408","oa":1,"main_file_link":[{"url":"https://doi.org/10.5281/zenodo.5813846","open_access":"1"}],"citation":{"mla":"Postnikova, Anastasiia, et al. Multi-Queues Can Be State-of-the-Art Priority Schedulers. Zenodo, 2022, doi:10.5281/ZENODO.5733408.","short":"A. Postnikova, N. Koval, G. Nadiradze, D.-A. Alistarh, (2022).","chicago":"Postnikova, Anastasiia, Nikita Koval, Giorgi Nadiradze, and Dan-Adrian Alistarh. “Multi-Queues Can Be State-of-the-Art Priority Schedulers.” Zenodo, 2022. https://doi.org/10.5281/ZENODO.5733408.","ama":"Postnikova A, Koval N, Nadiradze G, Alistarh D-A. Multi-queues can be state-of-the-art priority schedulers. 2022. doi:10.5281/ZENODO.5733408","ista":"Postnikova A, Koval N, Nadiradze G, Alistarh D-A. 2022. Multi-queues can be state-of-the-art priority schedulers, Zenodo, 10.5281/ZENODO.5733408.","ieee":"A. Postnikova, N. Koval, G. Nadiradze, and D.-A. Alistarh, “Multi-queues can be state-of-the-art priority schedulers.” Zenodo, 2022.","apa":"Postnikova, A., Koval, N., Nadiradze, G., & Alistarh, D.-A. (2022). Multi-queues can be state-of-the-art priority schedulers. Zenodo. https://doi.org/10.5281/ZENODO.5733408"},"abstract":[{"text":"The source code for replicating experiments presented in the paper.\r\n\r\nThe implementation of the designed priority schedulers can be found in Galois-2.2.1/include/Galois/WorkList/:\r\nStealingMultiQueue.h is the StealingMultiQueue.\r\nMQOptimized/ contains MQ Optimized variants.\r\n\r\nWe provide images that contain all the dependencies and datasets. Images can be pulled from npostnikova/mq-based-schedulers repository, or downloaded from Zenodo. See readme for more detail.","lang":"eng"}],"type":"research_data_reference","author":[{"first_name":"Anastasiia","last_name":"Postnikova","full_name":"Postnikova, Anastasiia"},{"id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87","first_name":"Nikita","last_name":"Koval","full_name":"Koval, Nikita"},{"id":"3279A00C-F248-11E8-B48F-1D18A9856A87","first_name":"Giorgi","last_name":"Nadiradze","full_name":"Nadiradze, Giorgi"},{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X"}],"related_material":{"record":[{"id":"11180","relation":"used_in_publication","status":"public"}],"link":[{"url":"https://github.com/npostnikova/mq-based-schedulers/tree/v1.1","relation":"software"}]},"date_created":"2023-05-23T17:05:40Z","date_updated":"2023-08-03T06:48:34Z","oa_version":"Published Version","_id":"13076","year":"2022","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","ddc":["510"],"status":"public","title":"Multi-queues can be state-of-the-art priority schedulers","publisher":"Zenodo","department":[{"_id":"DaAl"}]},{"editor":[{"last_name":"Parter","first_name":"Merav","full_name":"Parter, Merav"}],"publisher":"Springer Nature","department":[{"_id":"DaAl"}],"publication_status":"published","year":"2022","acknowledgement":"This project has received funding from the European Union’s Horizon 2020 research and innovation programme under the Marie Skłodowska-Curie grant agreement No 840605. This work was supported in part by the Academy of Finland, Grants 314888 and 333837. The authors would also like to thank David Harris, Neven Villani, and the anonymous reviewers for their very helpful comments and feedback on previous versions of this work.","volume":13298,"date_created":"2022-07-31T22:01:49Z","date_updated":"2023-08-03T12:16:29Z","author":[{"last_name":"Balliu","first_name":"Alkida","full_name":"Balliu, Alkida"},{"full_name":"Hirvonen, Juho","first_name":"Juho","last_name":"Hirvonen"},{"full_name":"Melnyk, Darya","first_name":"Darya","last_name":"Melnyk"},{"first_name":"Dennis","last_name":"Olivetti","full_name":"Olivetti, Dennis"},{"full_name":"Rybicki, Joel","orcid":"0000-0002-6432-6646","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","last_name":"Rybicki","first_name":"Joel"},{"full_name":"Suomela, Jukka","first_name":"Jukka","last_name":"Suomela"}],"ec_funded":1,"project":[{"_id":"26A5D39A-B435-11E9-9278-68D0E5697425","grant_number":"840605","name":"Coordination in constrained and natural distributed systems","call_identifier":"H2020"}],"isi":1,"quality_controlled":"1","main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/2102.08703"}],"oa":1,"external_id":{"arxiv":["2102.08703"],"isi":["000876977400001"]},"language":[{"iso":"eng"}],"doi":"10.1007/978-3-031-09993-9_1","conference":{"end_date":"2022-06-29","location":"Paderborn, Germany","start_date":"2022-06-27","name":"SIROCCO: Structural Information and Communication Complexity"},"publication_identifier":{"isbn":["9783031099922"],"eissn":["1611-3349"],"issn":["0302-9743"]},"month":"06","intvolume":" 13298","title":"Local mending","status":"public","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","_id":"11707","oa_version":"Preprint","type":"conference","abstract":[{"text":"In this work we introduce the graph-theoretic notion of mendability: for each locally checkable graph problem we can define its mending radius, which captures the idea of how far one needs to modify a partial solution in order to “patch a hole.” We explore how mendability is connected to the existence of efficient algorithms, especially in distributed, parallel, and fault-tolerant settings. It is easy to see that O(1)-mendable problems are also solvable in O(log∗n) rounds in the LOCAL model of distributed computing. One of the surprises is that in paths and cycles, a converse also holds in the following sense: if a problem Π can be solved in O(log∗n), there is always a restriction Π′⊆Π that is still efficiently solvable but that is also O(1)-mendable. We also explore the structure of the landscape of mendability. For example, we show that in trees, the mending radius of any locally checkable problem is O(1), Θ(logn), or Θ(n), while in general graphs the structure is much more diverse.","lang":"eng"}],"page":"1-20","citation":{"ieee":"A. Balliu, J. Hirvonen, D. Melnyk, D. Olivetti, J. Rybicki, and J. Suomela, “Local mending,” in International Colloquium on Structural Information and Communication Complexity, Paderborn, Germany, 2022, vol. 13298, pp. 1–20.","apa":"Balliu, A., Hirvonen, J., Melnyk, D., Olivetti, D., Rybicki, J., & Suomela, J. (2022). Local mending. In M. Parter (Ed.), International Colloquium on Structural Information and Communication Complexity (Vol. 13298, pp. 1–20). Paderborn, Germany: Springer Nature. https://doi.org/10.1007/978-3-031-09993-9_1","ista":"Balliu A, Hirvonen J, Melnyk D, Olivetti D, Rybicki J, Suomela J. 2022. Local mending. International Colloquium on Structural Information and Communication Complexity. SIROCCO: Structural Information and Communication ComplexityLNCS vol. 13298, 1–20.","ama":"Balliu A, Hirvonen J, Melnyk D, Olivetti D, Rybicki J, Suomela J. Local mending. In: Parter M, ed. International Colloquium on Structural Information and Communication Complexity. Vol 13298. LNCS. Springer Nature; 2022:1-20. doi:10.1007/978-3-031-09993-9_1","chicago":"Balliu, Alkida, Juho Hirvonen, Darya Melnyk, Dennis Olivetti, Joel Rybicki, and Jukka Suomela. “Local Mending.” In International Colloquium on Structural Information and Communication Complexity, edited by Merav Parter, 13298:1–20. LNCS. Springer Nature, 2022. https://doi.org/10.1007/978-3-031-09993-9_1.","short":"A. Balliu, J. Hirvonen, D. Melnyk, D. Olivetti, J. Rybicki, J. Suomela, in:, M. Parter (Ed.), International Colloquium on Structural Information and Communication Complexity, Springer Nature, 2022, pp. 1–20.","mla":"Balliu, Alkida, et al. “Local Mending.” International Colloquium on Structural Information and Communication Complexity, edited by Merav Parter, vol. 13298, Springer Nature, 2022, pp. 1–20, doi:10.1007/978-3-031-09993-9_1."},"publication":"International Colloquium on Structural Information and Communication Complexity","date_published":"2022-06-25T00:00:00Z","series_title":"LNCS","scopus_import":"1","article_processing_charge":"No","day":"25"},{"oa_version":"Preprint","_id":"12299","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","status":"public","title":"How well do sparse ImageNet models transfer?","abstract":[{"text":"Transfer learning is a classic paradigm by which models pretrained on large “upstream” datasets are adapted to yield good results on “downstream” specialized datasets. Generally, more accurate models on the “upstream” dataset tend to provide better transfer accuracy “downstream”. In this work, we perform an in-depth investigation of this phenomenon in the context of convolutional neural networks (CNNs) trained on the ImageNet dataset, which have been pruned-that is, compressed by sparsifiying their connections. We consider transfer using unstructured pruned models obtained by applying several state-of-the-art pruning methods, including magnitude-based, second-order, regrowth, lottery-ticket, and regularization approaches, in the context of twelve standard transfer tasks. In a nutshell, our study shows that sparse models can match or even outperform the transfer performance of dense models, even at high sparsities, and, while doing so, can lead to significant inference and even training speedups. At the same time, we observe and analyze significant differences in the behaviour of different pruning methods. The code is available at: https://github.com/IST-DASLab/sparse-imagenet-transfer.","lang":"eng"}],"type":"conference","date_published":"2022-09-27T00:00:00Z","publication":"2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition","citation":{"ista":"Iofinova EB, Peste E-A, Kurtz M, Alistarh D-A. 2022. How well do sparse ImageNet models transfer? 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition. CVPR: Computer Vision and Pattern Recognition, 12256–12266.","ieee":"E. B. Iofinova, E.-A. Peste, M. Kurtz, and D.-A. Alistarh, “How well do sparse ImageNet models transfer?,” in 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition, New Orleans, LA, United States, 2022, pp. 12256–12266.","apa":"Iofinova, E. B., Peste, E.-A., Kurtz, M., & Alistarh, D.-A. (2022). How well do sparse ImageNet models transfer? In 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (pp. 12256–12266). New Orleans, LA, United States: Institute of Electrical and Electronics Engineers. https://doi.org/10.1109/cvpr52688.2022.01195","ama":"Iofinova EB, Peste E-A, Kurtz M, Alistarh D-A. How well do sparse ImageNet models transfer? In: 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition. Institute of Electrical and Electronics Engineers; 2022:12256-12266. doi:10.1109/cvpr52688.2022.01195","chicago":"Iofinova, Eugenia B, Elena-Alexandra Peste, Mark Kurtz, and Dan-Adrian Alistarh. “How Well Do Sparse ImageNet Models Transfer?” In 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition, 12256–66. Institute of Electrical and Electronics Engineers, 2022. https://doi.org/10.1109/cvpr52688.2022.01195.","mla":"Iofinova, Eugenia B., et al. “How Well Do Sparse ImageNet Models Transfer?” 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition, Institute of Electrical and Electronics Engineers, 2022, pp. 12256–66, doi:10.1109/cvpr52688.2022.01195.","short":"E.B. Iofinova, E.-A. Peste, M. Kurtz, D.-A. Alistarh, in:, 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition, Institute of Electrical and Electronics Engineers, 2022, pp. 12256–12266."},"page":"12256-12266","day":"27","article_processing_charge":"No","scopus_import":"1","author":[{"full_name":"Iofinova, Eugenia B","last_name":"Iofinova","first_name":"Eugenia B","orcid":"0000-0002-7778-3221","id":"f9a17499-f6e0-11ea-865d-fdf9a3f77117"},{"last_name":"Peste","first_name":"Elena-Alexandra","id":"32D78294-F248-11E8-B48F-1D18A9856A87","full_name":"Peste, Elena-Alexandra"},{"full_name":"Kurtz, Mark","first_name":"Mark","last_name":"Kurtz"},{"orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian"}],"related_material":{"record":[{"id":"13074","status":"public","relation":"dissertation_contains"}]},"date_updated":"2023-08-04T10:33:28Z","date_created":"2023-01-16T10:06:00Z","year":"2022","acknowledgement":"he authors would like to sincerely thank Christoph Lampert and Nir Shavit for fruitful discussions during the development of this work, and Eldar Kurtic for experimental support. EI was supported in part by the FWF DK VGSCO, grant agreement number W1260-N35, while AP and DA acknowledge generous support by the ERC, via Starting Grant 805223 ScaleML.","publication_status":"published","publisher":"Institute of Electrical and Electronics Engineers","department":[{"_id":"DaAl"},{"_id":"ChLa"}],"ec_funded":1,"conference":{"name":"CVPR: Computer Vision and Pattern Recognition","end_date":"2022-06-24","start_date":"2022-06-18","location":"New Orleans, LA, United States"},"doi":"10.1109/cvpr52688.2022.01195","language":[{"iso":"eng"}],"oa":1,"external_id":{"isi":["000870759105034"],"arxiv":["2111.13445"]},"main_file_link":[{"url":"https://doi.org/10.48550/arXiv.2111.13445","open_access":"1"}],"quality_controlled":"1","isi":1,"project":[{"_id":"9B9290DE-BA93-11EA-9121-9846C619BF3A","grant_number":" W1260-N35","name":"Vienna Graduate School on Computational Optimization"},{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"month":"09","publication_identifier":{"eissn":["2575-7075"]}},{"date_published":"2021-09-01T00:00:00Z","article_type":"original","page":"1-124","publication":"Journal of Machine Learning Research","citation":{"chicago":"Hoefler, Torsten, Dan-Adrian Alistarh, Tal Ben-Nun, Nikoli Dryden, and Elena-Alexandra Peste. “Sparsity in Deep Learning: Pruning and Growth for Efficient Inference and Training in Neural Networks.” Journal of Machine Learning Research. Journal of Machine Learning Research, 2021.","short":"T. Hoefler, D.-A. Alistarh, T. Ben-Nun, N. Dryden, E.-A. Peste, Journal of Machine Learning Research 22 (2021) 1–124.","mla":"Hoefler, Torsten, et al. “Sparsity in Deep Learning: Pruning and Growth for Efficient Inference and Training in Neural Networks.” Journal of Machine Learning Research, vol. 22, no. 241, Journal of Machine Learning Research, 2021, pp. 1–124.","ieee":"T. Hoefler, D.-A. Alistarh, T. Ben-Nun, N. Dryden, and E.-A. Peste, “Sparsity in deep learning: Pruning and growth for efficient inference and training in neural networks,” Journal of Machine Learning Research, vol. 22, no. 241. Journal of Machine Learning Research, pp. 1–124, 2021.","apa":"Hoefler, T., Alistarh, D.-A., Ben-Nun, T., Dryden, N., & Peste, E.-A. (2021). Sparsity in deep learning: Pruning and growth for efficient inference and training in neural networks. Journal of Machine Learning Research. Journal of Machine Learning Research.","ista":"Hoefler T, Alistarh D-A, Ben-Nun T, Dryden N, Peste E-A. 2021. Sparsity in deep learning: Pruning and growth for efficient inference and training in neural networks. Journal of Machine Learning Research. 22(241), 1–124.","ama":"Hoefler T, Alistarh D-A, Ben-Nun T, Dryden N, Peste E-A. Sparsity in deep learning: Pruning and growth for efficient inference and training in neural networks. Journal of Machine Learning Research. 2021;22(241):1-124."},"day":"01","article_processing_charge":"No","has_accepted_license":"1","scopus_import":"1","file":[{"file_name":"2021_JMachLearnRes_Hoefler.pdf","access_level":"open_access","creator":"cziletti","file_size":3527521,"content_type":"application/pdf","file_id":"10192","relation":"main_file","date_updated":"2021-10-27T15:34:18Z","date_created":"2021-10-27T15:34:18Z","success":1,"checksum":"3389d9d01fc58f8fb4c1a53e14a8abbf"}],"oa_version":"Published Version","status":"public","ddc":["000"],"title":"Sparsity in deep learning: Pruning and growth for efficient inference and training in neural networks","intvolume":" 22","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"10180","abstract":[{"text":"The growing energy and performance costs of deep learning have driven the community to reduce the size of neural networks by selectively pruning components. Similarly to their biological counterparts, sparse networks generalize just as well, sometimes even better than, the original dense networks. Sparsity promises to reduce the memory footprint of regular networks to fit mobile devices, as well as shorten training time for ever growing networks. In this paper, we survey prior work on sparsity in deep learning and provide an extensive tutorial of sparsification for both inference and training. We describe approaches to remove and add elements of neural networks, different training strategies to achieve model sparsity, and mechanisms to exploit sparsity in practice. Our work distills ideas from more than 300 research papers and provides guidance to practitioners who wish to utilize sparsity today, as well as to researchers whose goal is to push the frontier forward. We include the necessary background on mathematical methods in sparsification, describe phenomena such as early structure adaptation, the intricate relations between sparsity and the training process, and show techniques for achieving acceleration on real hardware. We also define a metric of pruned parameter efficiency that could serve as a baseline for comparison of different sparse networks. We close by speculating on how sparsity can improve future workloads and outline major open problems in the field.","lang":"eng"}],"issue":"241","type":"journal_article","language":[{"iso":"eng"}],"quality_controlled":"1","main_file_link":[{"url":"https://www.jmlr.org/papers/v22/21-0366.html","open_access":"1"}],"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"external_id":{"arxiv":["2102.00554"]},"oa":1,"month":"09","publication_identifier":{"issn":["1532-4435"],"eissn":["1533-7928"]},"date_updated":"2022-05-13T09:36:08Z","date_created":"2021-10-24T22:01:34Z","volume":22,"author":[{"first_name":"Torsten","last_name":"Hoefler","full_name":"Hoefler, Torsten"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"},{"full_name":"Ben-Nun, Tal","last_name":"Ben-Nun","first_name":"Tal"},{"last_name":"Dryden","first_name":"Nikoli","full_name":"Dryden, Nikoli"},{"id":"32D78294-F248-11E8-B48F-1D18A9856A87","first_name":"Elena-Alexandra","last_name":"Peste","full_name":"Peste, Elena-Alexandra"}],"publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"Journal of Machine Learning Research","acknowledgement":"We thank Doug Burger, Steve Scott, Marco Heddes, and the respective teams at Microsoft for inspiring discussions on the topic. We thank Angelika Steger for uplifting debates about the connections to biological brains, Sidak Pal Singh for his support regarding experimental results, and Utku Evci as well as Xin Wang for comments on previous versions of this\r\nwork. Special thanks go to Bernhard Schölkopf, our JMLR editor Samy Bengio, and the three anonymous reviewers who provided excellent comprehensive, pointed, and deep review comments that improved the quality of our manuscript significantly.","year":"2021","file_date_updated":"2021-10-27T15:34:18Z"},{"ec_funded":1,"file_date_updated":"2021-11-12T08:16:44Z","article_number":"43","volume":209,"date_updated":"2023-02-21T09:24:08Z","date_created":"2021-11-07T23:01:24Z","author":[{"orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian"},{"first_name":"Rati","last_name":"Gelashvili","full_name":"Gelashvili, Rati"},{"full_name":"Rybicki, Joel","orcid":"0000-0002-6432-6646","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","last_name":"Rybicki","first_name":"Joel"}],"publisher":"Schloss Dagstuhl - Leibniz-Zentrum für Informatik","department":[{"_id":"DaAl"}],"publication_status":"published","acknowledgement":"This project has received funding from the European Union’s Horizon 2020 research and innovation programme under the Marie Skłodowska-Curie grant agreement No 840605.","year":"2021","publication_identifier":{"issn":["1868-8969"],"isbn":["9-783-9597-7210-5"]},"month":"10","language":[{"iso":"eng"}],"doi":"10.4230/LIPIcs.DISC.2021.43","conference":{"name":"DISC: Distributed Computing ","start_date":"2021-10-04","location":"Freiburg, Germany","end_date":"2021-10-08"},"project":[{"call_identifier":"H2020","name":"Coordination in constrained and natural distributed systems","_id":"26A5D39A-B435-11E9-9278-68D0E5697425","grant_number":"840605"}],"quality_controlled":"1","tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"external_id":{"arxiv":["2102.08808"]},"abstract":[{"lang":"eng","text":"Let G be a graph on n nodes. In the stochastic population protocol model, a collection of n indistinguishable, resource-limited nodes collectively solve tasks via pairwise interactions. In each interaction, two randomly chosen neighbors first read each other’s states, and then update their local states. A rich line of research has established tight upper and lower bounds on the complexity of fundamental tasks, such as majority and leader election, in this model, when G is a clique. Specifically, in the clique, these tasks can be solved fast, i.e., in n polylog n pairwise interactions, with high probability, using at most polylog n states per node. In this work, we consider the more general setting where G is an arbitrary graph, and present a technique for simulating protocols designed for fully-connected networks in any connected regular graph. Our main result is a simulation that is efficient on many interesting graph families: roughly, the simulation overhead is polylogarithmic in the number of nodes, and quadratic in the conductance of the graph. As an example, this implies that, in any regular graph with conductance φ, both leader election and exact majority can be solved in φ^{-2} ⋅ n polylog n pairwise interactions, with high probability, using at most φ^{-2} ⋅ polylog n states per node. This shows that there are fast and space-efficient population protocols for leader election and exact majority on graphs with good expansion properties."}],"alternative_title":["LIPIcs"],"type":"conference","file":[{"file_size":534219,"content_type":"application/pdf","creator":"cchlebak","file_name":"2021_LIPIcsDISC_Alistarh.pdf","access_level":"open_access","date_created":"2021-11-12T08:16:44Z","date_updated":"2021-11-12T08:16:44Z","checksum":"fd2a690f6856d21247e9aa952b0e2885","success":1,"relation":"main_file","file_id":"10274"}],"oa_version":"Published Version","intvolume":" 209","title":"Brief announcement: Fast graphical population protocols","ddc":["000"],"status":"public","_id":"10218","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","article_processing_charge":"No","has_accepted_license":"1","day":"04","scopus_import":"1","date_published":"2021-10-04T00:00:00Z","citation":{"ista":"Alistarh D-A, Gelashvili R, Rybicki J. 2021. Brief announcement: Fast graphical population protocols. 35th International Symposium on Distributed Computing. DISC: Distributed Computing , LIPIcs, vol. 209, 43.","apa":"Alistarh, D.-A., Gelashvili, R., & Rybicki, J. (2021). Brief announcement: Fast graphical population protocols. In 35th International Symposium on Distributed Computing (Vol. 209). Freiburg, Germany: Schloss Dagstuhl - Leibniz-Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.DISC.2021.43","ieee":"D.-A. Alistarh, R. Gelashvili, and J. Rybicki, “Brief announcement: Fast graphical population protocols,” in 35th International Symposium on Distributed Computing, Freiburg, Germany, 2021, vol. 209.","ama":"Alistarh D-A, Gelashvili R, Rybicki J. Brief announcement: Fast graphical population protocols. In: 35th International Symposium on Distributed Computing. Vol 209. Schloss Dagstuhl - Leibniz-Zentrum für Informatik; 2021. doi:10.4230/LIPIcs.DISC.2021.43","chicago":"Alistarh, Dan-Adrian, Rati Gelashvili, and Joel Rybicki. “Brief Announcement: Fast Graphical Population Protocols.” In 35th International Symposium on Distributed Computing, Vol. 209. Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2021. https://doi.org/10.4230/LIPIcs.DISC.2021.43.","mla":"Alistarh, Dan-Adrian, et al. “Brief Announcement: Fast Graphical Population Protocols.” 35th International Symposium on Distributed Computing, vol. 209, 43, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2021, doi:10.4230/LIPIcs.DISC.2021.43.","short":"D.-A. Alistarh, R. Gelashvili, J. Rybicki, in:, 35th International Symposium on Distributed Computing, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2021."},"publication":"35th International Symposium on Distributed Computing"},{"publication_identifier":{"issn":["1868-8969"],"isbn":["9-783-9597-7210-5"]},"month":"10","oa":1,"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"quality_controlled":"1","doi":"10.4230/LIPIcs.DISC.2021.4","conference":{"end_date":"2021-10-08","start_date":"2021-10-04","location":"Freiburg, Germany","name":"DISC: Distributed Computing"},"language":[{"iso":"eng"}],"article_number":"4","ec_funded":1,"file_date_updated":"2021-11-12T09:33:26Z","year":"2021","acknowledgement":"Dan Alistarh: Supported in part by the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). Giorgi Nadiradze: Supported in part by the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). The authors would like to thank the DISC anonymous reviewers for their useful\r\nfeedback and comments.","publisher":"Schloss Dagstuhl - Leibniz Zentrum für Informatik","department":[{"_id":"DaAl"}],"publication_status":"published","author":[{"orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian"},{"first_name":"Rati","last_name":"Gelashvili","full_name":"Gelashvili, Rati"},{"last_name":"Nadiradze","first_name":"Giorgi","id":"3279A00C-F248-11E8-B48F-1D18A9856A87","full_name":"Nadiradze, Giorgi"}],"volume":209,"date_created":"2021-11-07T23:01:23Z","date_updated":"2022-08-19T07:23:28Z","scopus_import":"1","has_accepted_license":"1","article_processing_charge":"No","day":"04","citation":{"mla":"Alistarh, Dan-Adrian, et al. “Lower Bounds for Shared-Memory Leader Election under Bounded Write Contention.” 35th International Symposium on Distributed Computing, vol. 209, 4, Schloss Dagstuhl - Leibniz Zentrum für Informatik, 2021, doi:10.4230/LIPIcs.DISC.2021.4.","short":"D.-A. Alistarh, R. Gelashvili, G. Nadiradze, in:, 35th International Symposium on Distributed Computing, Schloss Dagstuhl - Leibniz Zentrum für Informatik, 2021.","chicago":"Alistarh, Dan-Adrian, Rati Gelashvili, and Giorgi Nadiradze. “Lower Bounds for Shared-Memory Leader Election under Bounded Write Contention.” In 35th International Symposium on Distributed Computing, Vol. 209. Schloss Dagstuhl - Leibniz Zentrum für Informatik, 2021. https://doi.org/10.4230/LIPIcs.DISC.2021.4.","ama":"Alistarh D-A, Gelashvili R, Nadiradze G. Lower bounds for shared-memory leader election under bounded write contention. In: 35th International Symposium on Distributed Computing. Vol 209. Schloss Dagstuhl - Leibniz Zentrum für Informatik; 2021. doi:10.4230/LIPIcs.DISC.2021.4","ista":"Alistarh D-A, Gelashvili R, Nadiradze G. 2021. Lower bounds for shared-memory leader election under bounded write contention. 35th International Symposium on Distributed Computing. DISC: Distributed Computing, LIPIcs, vol. 209, 4.","apa":"Alistarh, D.-A., Gelashvili, R., & Nadiradze, G. (2021). Lower bounds for shared-memory leader election under bounded write contention. In 35th International Symposium on Distributed Computing (Vol. 209). Freiburg, Germany: Schloss Dagstuhl - Leibniz Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.DISC.2021.4","ieee":"D.-A. Alistarh, R. Gelashvili, and G. Nadiradze, “Lower bounds for shared-memory leader election under bounded write contention,” in 35th International Symposium on Distributed Computing, Freiburg, Germany, 2021, vol. 209."},"publication":"35th International Symposium on Distributed Computing","date_published":"2021-10-04T00:00:00Z","type":"conference","alternative_title":["LIPIcs"],"abstract":[{"text":"This paper gives tight logarithmic lower bounds on the solo step complexity of leader election in an asynchronous shared-memory model with single-writer multi-reader (SWMR) registers, for both deterministic and randomized obstruction-free algorithms. The approach extends to lower bounds for deterministic and randomized obstruction-free algorithms using multi-writer registers under bounded write concurrency, showing a trade-off between the solo step complexity of a leader election algorithm, and the worst-case number of stalls incurred by a processor in an execution.","lang":"eng"}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"10217","intvolume":" 209","ddc":["000"],"status":"public","title":"Lower bounds for shared-memory leader election under bounded write contention","oa_version":"Published Version","file":[{"access_level":"open_access","file_name":"2021_LIPIcsDISC_Alistarh.pdf","creator":"cchlebak","file_size":706791,"content_type":"application/pdf","file_id":"10277","relation":"main_file","success":1,"checksum":"b4cdc6668c899a601c5e6a96b8ca54d9","date_created":"2021-11-12T09:33:26Z","date_updated":"2021-11-12T09:33:26Z"}]},{"article_processing_charge":"No","has_accepted_license":"1","day":"04","scopus_import":"1","date_published":"2021-10-04T00:00:00Z","citation":{"ista":"Chatterjee B, Peri S, Sa M. 2021. Brief announcement: Non-blocking dynamic unbounded graphs with worst-case amortized bounds. 35th International Symposium on Distributed Computing. DISC: Distributed Computing, LIPIcs, vol. 209, 52.","apa":"Chatterjee, B., Peri, S., & Sa, M. (2021). Brief announcement: Non-blocking dynamic unbounded graphs with worst-case amortized bounds. In 35th International Symposium on Distributed Computing (Vol. 209). Freiburg, Germany: Schloss Dagstuhl - Leibniz Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.DISC.2021.52","ieee":"B. Chatterjee, S. Peri, and M. Sa, “Brief announcement: Non-blocking dynamic unbounded graphs with worst-case amortized bounds,” in 35th International Symposium on Distributed Computing, Freiburg, Germany, 2021, vol. 209.","ama":"Chatterjee B, Peri S, Sa M. Brief announcement: Non-blocking dynamic unbounded graphs with worst-case amortized bounds. In: 35th International Symposium on Distributed Computing. Vol 209. Schloss Dagstuhl - Leibniz Zentrum für Informatik; 2021. doi:10.4230/LIPIcs.DISC.2021.52","chicago":"Chatterjee, Bapi, Sathya Peri, and Muktikanta Sa. “Brief Announcement: Non-Blocking Dynamic Unbounded Graphs with Worst-Case Amortized Bounds.” In 35th International Symposium on Distributed Computing, Vol. 209. Schloss Dagstuhl - Leibniz Zentrum für Informatik, 2021. https://doi.org/10.4230/LIPIcs.DISC.2021.52.","mla":"Chatterjee, Bapi, et al. “Brief Announcement: Non-Blocking Dynamic Unbounded Graphs with Worst-Case Amortized Bounds.” 35th International Symposium on Distributed Computing, vol. 209, 52, Schloss Dagstuhl - Leibniz Zentrum für Informatik, 2021, doi:10.4230/LIPIcs.DISC.2021.52.","short":"B. Chatterjee, S. Peri, M. Sa, in:, 35th International Symposium on Distributed Computing, Schloss Dagstuhl - Leibniz Zentrum für Informatik, 2021."},"publication":"35th International Symposium on Distributed Computing","abstract":[{"lang":"eng","text":"This paper reports a new concurrent graph data structure that supports updates of both edges and vertices and queries: Breadth-first search, Single-source shortest-path, and Betweenness centrality. The operations are provably linearizable and non-blocking."}],"alternative_title":["LIPIcs"],"type":"conference","file":[{"date_created":"2021-11-12T09:23:22Z","date_updated":"2021-11-12T09:23:22Z","success":1,"checksum":"76546df112a0ba1166c864d33d7834e2","file_id":"10276","relation":"main_file","creator":"cchlebak","content_type":"application/pdf","file_size":795860,"file_name":"2021_LIPIcsDISC_BChatterjee.pdf","access_level":"open_access"}],"oa_version":"Published Version","intvolume":" 209","ddc":["000"],"status":"public","title":"Brief announcement: Non-blocking dynamic unbounded graphs with worst-case amortized bounds","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","_id":"10216","publication_identifier":{"isbn":["9-783-9597-7210-5"],"issn":["1868-8969"]},"month":"10","language":[{"iso":"eng"}],"doi":"10.4230/LIPIcs.DISC.2021.52","conference":{"name":"DISC: Distributed Computing","location":"Freiburg, Germany","start_date":"2021-10-04","end_date":"2021-10-08"},"quality_controlled":"1","oa":1,"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"external_id":{"arxiv":["2003.01697"]},"file_date_updated":"2021-11-12T09:23:22Z","article_number":"52","volume":209,"date_created":"2021-11-07T23:01:23Z","date_updated":"2021-11-12T09:42:55Z","author":[{"id":"3C41A08A-F248-11E8-B48F-1D18A9856A87","first_name":"Bapi","last_name":"Chatterjee","full_name":"Chatterjee, Bapi"},{"full_name":"Peri, Sathya","last_name":"Peri","first_name":"Sathya"},{"full_name":"Sa, Muktikanta","last_name":"Sa","first_name":"Muktikanta"}],"publisher":"Schloss Dagstuhl - Leibniz Zentrum für Informatik","department":[{"_id":"DaAl"}],"publication_status":"published","year":"2021","acknowledgement":"This work was partially funded by National Supercomputing Mission, Govt. of India under the project “Concurrent and Distributed Programming primitives and algorithms for Temporal Graphs”(DST/NSM/R&D_Exascale/2021/16).\r\n"},{"scopus_import":"1","article_processing_charge":"No","has_accepted_license":"1","day":"04","citation":{"ama":"Korhonen J, Paz A, Rybicki J, Schmid S, Suomela J. Brief announcement: Sinkless orientation is hard also in the supported LOCAL model. In: 35th International Symposium on Distributed Computing. Vol 209. Schloss Dagstuhl - Leibniz Zentrum für Informatik; 2021. doi:10.4230/LIPIcs.DISC.2021.58","ieee":"J. Korhonen, A. Paz, J. Rybicki, S. Schmid, and J. Suomela, “Brief announcement: Sinkless orientation is hard also in the supported LOCAL model,” in 35th International Symposium on Distributed Computing, Freiburg, Germany, 2021, vol. 209.","apa":"Korhonen, J., Paz, A., Rybicki, J., Schmid, S., & Suomela, J. (2021). Brief announcement: Sinkless orientation is hard also in the supported LOCAL model. In 35th International Symposium on Distributed Computing (Vol. 209). Freiburg, Germany: Schloss Dagstuhl - Leibniz Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.DISC.2021.58","ista":"Korhonen J, Paz A, Rybicki J, Schmid S, Suomela J. 2021. Brief announcement: Sinkless orientation is hard also in the supported LOCAL model. 35th International Symposium on Distributed Computing. DISC: Distributed Computing , LIPIcs, vol. 209, 58.","short":"J. Korhonen, A. Paz, J. Rybicki, S. Schmid, J. Suomela, in:, 35th International Symposium on Distributed Computing, Schloss Dagstuhl - Leibniz Zentrum für Informatik, 2021.","mla":"Korhonen, Janne, et al. “Brief Announcement: Sinkless Orientation Is Hard Also in the Supported LOCAL Model.” 35th International Symposium on Distributed Computing, vol. 209, 58, Schloss Dagstuhl - Leibniz Zentrum für Informatik, 2021, doi:10.4230/LIPIcs.DISC.2021.58.","chicago":"Korhonen, Janne, Ami Paz, Joel Rybicki, Stefan Schmid, and Jukka Suomela. “Brief Announcement: Sinkless Orientation Is Hard Also in the Supported LOCAL Model.” In 35th International Symposium on Distributed Computing, Vol. 209. Schloss Dagstuhl - Leibniz Zentrum für Informatik, 2021. https://doi.org/10.4230/LIPIcs.DISC.2021.58."},"publication":"35th International Symposium on Distributed Computing","date_published":"2021-10-04T00:00:00Z","type":"conference","alternative_title":["LIPIcs"],"abstract":[{"text":"We show that any algorithm that solves the sinkless orientation problem in the supported LOCAL model requires Ω(log n) rounds, and this is tight. The supported LOCAL is at least as strong as the usual LOCAL model, and as a corollary this also gives a new, short and elementary proof that shows that the round complexity of the sinkless orientation problem in the deterministic LOCAL model is Ω(log n).","lang":"eng"}],"_id":"10219","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","intvolume":" 209","status":"public","title":"Brief announcement: Sinkless orientation is hard also in the supported LOCAL model","ddc":["000"],"file":[{"creator":"cchlebak","content_type":"application/pdf","file_size":474242,"file_name":"2021_LIPIcsDISC_Korhonen.pdf","access_level":"open_access","date_updated":"2021-11-12T08:27:42Z","date_created":"2021-11-12T08:27:42Z","success":1,"checksum":"c43188dc2070bbd2bf5fd6fdaf9ce36d","file_id":"10275","relation":"main_file"}],"oa_version":"Published Version","publication_identifier":{"issn":["1868-8969"],"isbn":["9-783-9597-7210-5"]},"month":"10","external_id":{"arxiv":["2108.02655"]},"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"quality_controlled":"1","doi":"10.4230/LIPIcs.DISC.2021.58","conference":{"end_date":"2021-10-08","start_date":"2021-10-04","location":"Freiburg, Germany","name":"DISC: Distributed Computing "},"language":[{"iso":"eng"}],"article_number":"58","ec_funded":1,"file_date_updated":"2021-11-12T08:27:42Z","acknowledgement":"Janne H. Korhonen: Project has received funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). Ami Paz: We acknowledge the Austrian Science Fund (FWF) and netIDEE SCIENCE project P 33775-N. Stefan Schmid: Research supported by the Austrian Science Fund (FWF) project ADVISE, I 4800-N, 2020-2023.\r\n","year":"2021","department":[{"_id":"DaAl"}],"publisher":"Schloss Dagstuhl - Leibniz Zentrum für Informatik","publication_status":"published","author":[{"full_name":"Korhonen, Janne","last_name":"Korhonen","first_name":"Janne","id":"C5402D42-15BC-11E9-A202-CA2BE6697425"},{"full_name":"Paz, Ami","first_name":"Ami","last_name":"Paz"},{"full_name":"Rybicki, Joel","orcid":"0000-0002-6432-6646","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","last_name":"Rybicki","first_name":"Joel"},{"full_name":"Schmid, Stefan","first_name":"Stefan","last_name":"Schmid"},{"full_name":"Suomela, Jukka","last_name":"Suomela","first_name":"Jukka"}],"volume":209,"date_updated":"2021-11-12T09:37:18Z","date_created":"2021-11-07T23:01:24Z"},{"page":"208-220","quality_controlled":"1","external_id":{"arxiv":["2105.08098"]},"oa":1,"main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/2105.08098"}],"citation":{"short":"A. Fedorov, N. Koval, D.-A. Alistarh, in:, Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures, Association for Computing Machinery, 2021, pp. 208–220.","mla":"Fedorov, Alexander, et al. “A Scalable Concurrent Algorithm for Dynamic Connectivity.” Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures, Association for Computing Machinery, 2021, pp. 208–20, doi:10.1145/3409964.3461810.","chicago":"Fedorov, Alexander, Nikita Koval, and Dan-Adrian Alistarh. “A Scalable Concurrent Algorithm for Dynamic Connectivity.” In Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures, 208–20. Association for Computing Machinery, 2021. https://doi.org/10.1145/3409964.3461810.","ama":"Fedorov A, Koval N, Alistarh D-A. A scalable concurrent algorithm for dynamic connectivity. In: Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures. Association for Computing Machinery; 2021:208-220. doi:10.1145/3409964.3461810","ieee":"A. Fedorov, N. Koval, and D.-A. Alistarh, “A scalable concurrent algorithm for dynamic connectivity,” in Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures, Virtual, Online, 2021, pp. 208–220.","apa":"Fedorov, A., Koval, N., & Alistarh, D.-A. (2021). A scalable concurrent algorithm for dynamic connectivity. In Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures (pp. 208–220). Virtual, Online: Association for Computing Machinery. https://doi.org/10.1145/3409964.3461810","ista":"Fedorov A, Koval N, Alistarh D-A. 2021. A scalable concurrent algorithm for dynamic connectivity. Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures. SPAA: Symposium on Parallelism in Algorithms and Architectures, 208–220."},"publication":"Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures","language":[{"iso":"eng"}],"date_published":"2021-07-01T00:00:00Z","doi":"10.1145/3409964.3461810","conference":{"location":"Virtual, Online","start_date":"2021-07-06","end_date":"2021-07-08","name":"SPAA: Symposium on Parallelism in Algorithms and Architectures"},"scopus_import":"1","publication_identifier":{"isbn":["9781450380706"]},"article_processing_charge":"No","day":"01","month":"07","department":[{"_id":"DaAl"}],"publisher":"Association for Computing Machinery","publication_status":"published","title":"A scalable concurrent algorithm for dynamic connectivity","status":"public","_id":"10853","year":"2021","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","oa_version":"Preprint","date_updated":"2022-03-18T08:45:46Z","date_created":"2022-03-18T08:21:47Z","author":[{"full_name":"Fedorov, Alexander","first_name":"Alexander","last_name":"Fedorov"},{"full_name":"Koval, Nikita","first_name":"Nikita","last_name":"Koval"},{"full_name":"Alistarh, Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh"}],"type":"conference","abstract":[{"lang":"eng","text":"Dynamic Connectivity is a fundamental algorithmic graph problem, motivated by a wide range of applications to social and communication networks and used as a building block in various other algorithms, such as the bi-connectivity and the dynamic minimal spanning tree problems. In brief, we wish to maintain the connected components of the graph under dynamic edge insertions and deletions. In the sequential case, the problem has been well-studied from both theoretical and practical perspectives. However, much less is known about efficient concurrent solutions to this problem. This is the gap we address in this paper. We start from one of the classic data structures used to solve this problem, the Euler Tour Tree. Our first contribution is a non-blocking single-writer implementation of it. We leverage this data structure to obtain the first truly concurrent generalization of dynamic connectivity, which preserves the time complexity of its sequential counterpart, but is also scalable in practice. To achieve this, we rely on three main techniques. The first is to ensure that connectivity queries, which usually dominate real-world workloads, are non-blocking. The second non-trivial technique expands the above idea by making all queries that do not change the connectivity structure non-blocking. The third ingredient is applying fine-grained locking for updating the connected components, which allows operations on disjoint components to occur in parallel. We evaluate the resulting algorithm on various workloads, executing on both real and synthetic graphs. The results show the efficiency of each of the proposed optimizations; the most efficient variant improves the performance of a coarse-grained based implementation on realistic scenarios up to 6x on average and up to 30x when connectivity queries dominate."}]},{"scopus_import":"1","day":"18","article_processing_charge":"No","page":"8209-8216","publication":"35th AAAI Conference on Artificial Intelligence, AAAI 2021","citation":{"short":"V. Kungurtsev, M. Egan, B. Chatterjee, D.-A. Alistarh, in:, 35th AAAI Conference on Artificial Intelligence, AAAI 2021, AAAI Press, 2021, pp. 8209–8216.","mla":"Kungurtsev, Vyacheslav, et al. “Asynchronous Optimization Methods for Efficient Training of Deep Neural Networks with Guarantees.” 35th AAAI Conference on Artificial Intelligence, AAAI 2021, vol. 35, no. 9B, AAAI Press, 2021, pp. 8209–16.","chicago":"Kungurtsev, Vyacheslav, Malcolm Egan, Bapi Chatterjee, and Dan-Adrian Alistarh. “Asynchronous Optimization Methods for Efficient Training of Deep Neural Networks with Guarantees.” In 35th AAAI Conference on Artificial Intelligence, AAAI 2021, 35:8209–16. AAAI Press, 2021.","ama":"Kungurtsev V, Egan M, Chatterjee B, Alistarh D-A. Asynchronous optimization methods for efficient training of deep neural networks with guarantees. In: 35th AAAI Conference on Artificial Intelligence, AAAI 2021. Vol 35. AAAI Press; 2021:8209-8216.","apa":"Kungurtsev, V., Egan, M., Chatterjee, B., & Alistarh, D.-A. (2021). Asynchronous optimization methods for efficient training of deep neural networks with guarantees. In 35th AAAI Conference on Artificial Intelligence, AAAI 2021 (Vol. 35, pp. 8209–8216). Virtual, Online: AAAI Press.","ieee":"V. Kungurtsev, M. Egan, B. Chatterjee, and D.-A. Alistarh, “Asynchronous optimization methods for efficient training of deep neural networks with guarantees,” in 35th AAAI Conference on Artificial Intelligence, AAAI 2021, Virtual, Online, 2021, vol. 35, no. 9B, pp. 8209–8216.","ista":"Kungurtsev V, Egan M, Chatterjee B, Alistarh D-A. 2021. Asynchronous optimization methods for efficient training of deep neural networks with guarantees. 35th AAAI Conference on Artificial Intelligence, AAAI 2021. AAAI: Conference on Artificial Intelligence vol. 35, 8209–8216."},"date_published":"2021-05-18T00:00:00Z","type":"conference","abstract":[{"lang":"eng","text":"Asynchronous distributed algorithms are a popular way to reduce synchronization costs in large-scale optimization, and in particular for neural network training. However, for nonsmooth and nonconvex objectives, few convergence guarantees exist beyond cases where closed-form proximal operator solutions are available. As training most popular deep neural networks corresponds to optimizing nonsmooth and nonconvex objectives, there is a pressing need for such convergence guarantees. In this paper, we analyze for the first time the convergence of stochastic asynchronous optimization for this general class of objectives. In particular, we focus on stochastic subgradient methods allowing for block variable partitioning, where the shared model is asynchronously updated by concurrent processes. To this end, we use a probabilistic model which captures key features of real asynchronous scheduling between concurrent processes. Under this model, we establish convergence with probability one to an invariant set for stochastic subgradient methods with momentum. From a practical perspective, one issue with the family of algorithms that we consider is that they are not efficiently supported by machine learning frameworks, which mostly focus on distributed data-parallel strategies. To address this, we propose a new implementation strategy for shared-memory based training of deep neural networks for a partitioned but shared model in single- and multi-GPU settings. Based on this implementation, we achieve on average1.2x speed-up in comparison to state-of-the-art training methods for popular image classification tasks, without compromising accuracy."}],"issue":"9B","status":"public","title":"Asynchronous optimization methods for efficient training of deep neural networks with guarantees","intvolume":" 35","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"11436","oa_version":"Preprint","month":"05","publication_identifier":{"eissn":["2374-3468"],"isbn":["9781713835974"],"issn":["2159-5399"]},"quality_controlled":"1","project":[{"_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411","call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships"},{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"oa":1,"main_file_link":[{"open_access":"1","url":" https://doi.org/10.48550/arXiv.1905.11845"}],"external_id":{"arxiv":["1905.11845"]},"language":[{"iso":"eng"}],"conference":{"name":"AAAI: Conference on Artificial Intelligence","start_date":"2021-02-02","location":"Virtual, Online","end_date":"2021-02-09"},"ec_funded":1,"publication_status":"published","publisher":"AAAI Press","department":[{"_id":"DaAl"}],"acknowledgement":"Vyacheslav Kungurtsev was supported by the OP VVV project CZ.02.1.01/0.0/0.0/16 019/0000765 “Research Center for Informatics. Bapi Chatterjee was supported by the European Union’s Horizon 2020 research and innovation programme under the Marie Sklodowska-Curie grant agreement No. 754411 (ISTPlus). Dan Alistarh has received funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML).","year":"2021","date_created":"2022-06-05T22:01:52Z","date_updated":"2022-06-07T06:53:36Z","volume":35,"author":[{"first_name":"Vyacheslav","last_name":"Kungurtsev","full_name":"Kungurtsev, Vyacheslav"},{"first_name":"Malcolm","last_name":"Egan","full_name":"Egan, Malcolm"},{"id":"3C41A08A-F248-11E8-B48F-1D18A9856A87","first_name":"Bapi","last_name":"Chatterjee","full_name":"Chatterjee, Bapi"},{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X"}]},{"month":"12","publication_identifier":{"isbn":["9781713845393"],"issn":["1049-5258"]},"quality_controlled":"1","project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"},{"name":"ISTplus - Postdoctoral Fellowships","call_identifier":"H2020","_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411"}],"main_file_link":[{"open_access":"1","url":"https://proceedings.neurips.cc/paper/2021/file/1680e9fa7b4dd5d62ece800239bb53bd-Paper.pdf"}],"external_id":{"arxiv":["2110.14391"]},"oa":1,"language":[{"iso":"eng"}],"conference":{"name":"NeurIPS: Neural Information Processing Systems","location":"Virtual, Online","start_date":"2021-12-06","end_date":"2021-12-14"},"ec_funded":1,"publication_status":"published","publisher":"Neural Information Processing Systems Foundation","department":[{"_id":"DaAl"}],"year":"2021","acknowledgement":"We would like to thank the anonymous reviewers for helpful comments and suggestions. We also thank Aurelien Lucchi and Antonio Orvieto for fruitful discussions at an early stage of this work. FA is partially supported by the SNSF under research project No. 192363 and conducted part of this work while at IST Austria under the European Union’s Horizon 2020 research and innovation programme (grant agreement No. 805223 ScaleML). PD partly conducted this work while at IST Austria and was supported by the European Union’s Horizon 2020 programme under the Marie Skłodowska-Curie grant agreement No. 754411.","date_created":"2022-06-19T22:01:58Z","date_updated":"2022-06-20T08:31:52Z","volume":4,"author":[{"full_name":"Alimisis, Foivos","first_name":"Foivos","last_name":"Alimisis"},{"full_name":"Davies, Peter","orcid":"0000-0002-5646-9524","id":"11396234-BB50-11E9-B24C-90FCE5697425","last_name":"Davies","first_name":"Peter"},{"full_name":"Vandereycken, Bart","last_name":"Vandereycken","first_name":"Bart"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"}],"scopus_import":"1","day":"01","article_processing_charge":"No","page":"2823-2834","publication":"Advances in Neural Information Processing Systems - 35th Conference on Neural Information Processing Systems","citation":{"mla":"Alimisis, Foivos, et al. “Distributed Principal Component Analysis with Limited Communication.” Advances in Neural Information Processing Systems - 35th Conference on Neural Information Processing Systems, vol. 4, Neural Information Processing Systems Foundation, 2021, pp. 2823–34.","short":"F. Alimisis, P. Davies, B. Vandereycken, D.-A. Alistarh, in:, Advances in Neural Information Processing Systems - 35th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2021, pp. 2823–2834.","chicago":"Alimisis, Foivos, Peter Davies, Bart Vandereycken, and Dan-Adrian Alistarh. “Distributed Principal Component Analysis with Limited Communication.” In Advances in Neural Information Processing Systems - 35th Conference on Neural Information Processing Systems, 4:2823–34. Neural Information Processing Systems Foundation, 2021.","ama":"Alimisis F, Davies P, Vandereycken B, Alistarh D-A. Distributed principal component analysis with limited communication. In: Advances in Neural Information Processing Systems - 35th Conference on Neural Information Processing Systems. Vol 4. Neural Information Processing Systems Foundation; 2021:2823-2834.","ista":"Alimisis F, Davies P, Vandereycken B, Alistarh D-A. 2021. Distributed principal component analysis with limited communication. Advances in Neural Information Processing Systems - 35th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems vol. 4, 2823–2834.","ieee":"F. Alimisis, P. Davies, B. Vandereycken, and D.-A. Alistarh, “Distributed principal component analysis with limited communication,” in Advances in Neural Information Processing Systems - 35th Conference on Neural Information Processing Systems, Virtual, Online, 2021, vol. 4, pp. 2823–2834.","apa":"Alimisis, F., Davies, P., Vandereycken, B., & Alistarh, D.-A. (2021). Distributed principal component analysis with limited communication. In Advances in Neural Information Processing Systems - 35th Conference on Neural Information Processing Systems (Vol. 4, pp. 2823–2834). Virtual, Online: Neural Information Processing Systems Foundation."},"date_published":"2021-12-01T00:00:00Z","type":"conference","abstract":[{"text":"We study efficient distributed algorithms for the fundamental problem of principal component analysis and leading eigenvector computation on the sphere, when the data are randomly distributed among a set of computational nodes. We propose a new quantized variant of Riemannian gradient descent to solve this problem, and prove that the algorithm converges with high probability under a set of necessary spherical-convexity properties. We give bounds on the number of bits transmitted by the algorithm under common initialization schemes, and investigate the dependency on the problem dimension in each case.","lang":"eng"}],"title":"Distributed principal component analysis with limited communication","status":"public","intvolume":" 4","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"11452","oa_version":"Published Version"},{"scopus_import":"1","article_processing_charge":"No","day":"06","citation":{"ama":"Frantar E, Kurtic E, Alistarh D-A. M-FAC: Efficient matrix-free approximations of second-order information. In: 35th Conference on Neural Information Processing Systems. Vol 34. Curran Associates; 2021:14873-14886.","ieee":"E. Frantar, E. Kurtic, and D.-A. Alistarh, “M-FAC: Efficient matrix-free approximations of second-order information,” in 35th Conference on Neural Information Processing Systems, Virtual, Online, 2021, vol. 34, pp. 14873–14886.","apa":"Frantar, E., Kurtic, E., & Alistarh, D.-A. (2021). M-FAC: Efficient matrix-free approximations of second-order information. In 35th Conference on Neural Information Processing Systems (Vol. 34, pp. 14873–14886). Virtual, Online: Curran Associates.","ista":"Frantar E, Kurtic E, Alistarh D-A. 2021. M-FAC: Efficient matrix-free approximations of second-order information. 35th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems vol. 34, 14873–14886.","short":"E. Frantar, E. Kurtic, D.-A. Alistarh, in:, 35th Conference on Neural Information Processing Systems, Curran Associates, 2021, pp. 14873–14886.","mla":"Frantar, Elias, et al. “M-FAC: Efficient Matrix-Free Approximations of Second-Order Information.” 35th Conference on Neural Information Processing Systems, vol. 34, Curran Associates, 2021, pp. 14873–86.","chicago":"Frantar, Elias, Eldar Kurtic, and Dan-Adrian Alistarh. “M-FAC: Efficient Matrix-Free Approximations of Second-Order Information.” In 35th Conference on Neural Information Processing Systems, 34:14873–86. Curran Associates, 2021."},"publication":"35th Conference on Neural Information Processing Systems","page":"14873-14886","date_published":"2021-12-06T00:00:00Z","type":"conference","abstract":[{"lang":"eng","text":"Efficiently approximating local curvature information of the loss function is a key tool for optimization and compression of deep neural networks. Yet, most existing methods to approximate second-order information have high computational\r\nor storage costs, which limits their practicality. In this work, we investigate matrix-free, linear-time approaches for estimating Inverse-Hessian Vector Products (IHVPs) for the case when the Hessian can be approximated as a sum of rank-one matrices, as in the classic approximation of the Hessian by the empirical Fisher matrix. We propose two new algorithms: the first is tailored towards network compression and can compute the IHVP for dimension d, if the Hessian is given as a sum of m rank-one matrices, using O(dm2) precomputation, O(dm) cost for computing the IHVP, and query cost O(m) for any single element of the inverse Hessian. The second algorithm targets an optimization setting, where we wish to compute the product between the inverse Hessian, estimated over a sliding window of optimization steps, and a given gradient direction, as required for preconditioned SGD. We give an algorithm with cost O(dm + m2) for computing the IHVP and O(dm + m3) for adding or removing any gradient from the sliding window. These\r\ntwo algorithms yield state-of-the-art results for network pruning and optimization with lower computational overhead relative to existing second-order methods. Implementations are available at [9] and [17]."}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"11463","intvolume":" 34","title":"M-FAC: Efficient matrix-free approximations of second-order information","status":"public","oa_version":"Published Version","publication_identifier":{"isbn":["9781713845393"],"issn":["1049-5258"]},"month":"12","external_id":{"arxiv":["2010.08222"]},"main_file_link":[{"open_access":"1","url":"https://proceedings.neurips.cc/paper/2021/file/7cfd5df443b4eb0d69886a583b33de4c-Paper.pdf"}],"oa":1,"project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"quality_controlled":"1","conference":{"end_date":"2021-12-14","start_date":"2021-12-06","location":"Virtual, Online","name":"NeurIPS: Neural Information Processing Systems"},"language":[{"iso":"eng"}],"ec_funded":1,"acknowledgement":"We gratefully acknowledge funding the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML), as well as computational support from Amazon Web Services (AWS) EC2.","year":"2021","department":[{"_id":"DaAl"}],"publisher":"Curran Associates","publication_status":"published","author":[{"id":"09a8f98d-ec99-11ea-ae11-c063a7b7fe5f","first_name":"Elias","last_name":"Frantar","full_name":"Frantar, Elias"},{"full_name":"Kurtic, Eldar","last_name":"Kurtic","first_name":"Eldar","id":"47beb3a5-07b5-11eb-9b87-b108ec578218"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"volume":34,"date_updated":"2022-06-27T07:05:12Z","date_created":"2022-06-26T22:01:35Z"},{"abstract":[{"text":"We consider a standard distributed optimisation setting where N machines, each holding a d-dimensional function\r\nfi, aim to jointly minimise the sum of the functions ∑Ni=1fi(x). This problem arises naturally in large-scale distributed optimisation, where a standard solution is to apply variants of (stochastic) gradient descent. We focus on the communication complexity of this problem: our main result provides the first fully unconditional bounds on total number of bits which need to be sent and received by the N machines to solve this problem under point-to-point communication, within a given error-tolerance. Specifically, we show that Ω(Ndlogd/Nε) total bits need to be communicated between the machines to find an additive ϵ-approximation to the minimum of ∑Ni=1fi(x). The result holds for both deterministic and randomised algorithms, and, importantly, requires no assumptions on the algorithm structure. The lower bound is tight under certain restrictions on parameter values, and is matched within constant factors for quadratic objectives by a new variant of quantised gradient descent, which we describe and analyse. Our results bring over tools from communication complexity to distributed optimisation, which has potential for further applications.","lang":"eng"}],"type":"conference","oa_version":"Published Version","_id":"11464","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","status":"public","title":"Towards tight communication lower bounds for distributed optimisation","intvolume":" 34","day":"06","article_processing_charge":"No","scopus_import":"1","date_published":"2021-12-06T00:00:00Z","publication":"35th Conference on Neural Information Processing Systems","citation":{"chicago":"Alistarh, Dan-Adrian, and Janne Korhonen. “Towards Tight Communication Lower Bounds for Distributed Optimisation.” In 35th Conference on Neural Information Processing Systems, 34:7254–66. Curran Associates, 2021.","short":"D.-A. Alistarh, J. Korhonen, in:, 35th Conference on Neural Information Processing Systems, Curran Associates, 2021, pp. 7254–7266.","mla":"Alistarh, Dan-Adrian, and Janne Korhonen. “Towards Tight Communication Lower Bounds for Distributed Optimisation.” 35th Conference on Neural Information Processing Systems, vol. 34, Curran Associates, 2021, pp. 7254–66.","ieee":"D.-A. Alistarh and J. Korhonen, “Towards tight communication lower bounds for distributed optimisation,” in 35th Conference on Neural Information Processing Systems, Virtual, Online, 2021, vol. 34, pp. 7254–7266.","apa":"Alistarh, D.-A., & Korhonen, J. (2021). Towards tight communication lower bounds for distributed optimisation. In 35th Conference on Neural Information Processing Systems (Vol. 34, pp. 7254–7266). Virtual, Online: Curran Associates.","ista":"Alistarh D-A, Korhonen J. 2021. Towards tight communication lower bounds for distributed optimisation. 35th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems vol. 34, 7254–7266.","ama":"Alistarh D-A, Korhonen J. Towards tight communication lower bounds for distributed optimisation. In: 35th Conference on Neural Information Processing Systems. Vol 34. Curran Associates; 2021:7254-7266."},"page":"7254-7266","ec_funded":1,"author":[{"full_name":"Alistarh, Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh"},{"id":"C5402D42-15BC-11E9-A202-CA2BE6697425","last_name":"Korhonen","first_name":"Janne","full_name":"Korhonen, Janne"}],"date_created":"2022-06-26T22:01:35Z","date_updated":"2022-06-27T06:54:31Z","volume":34,"acknowledgement":"We thank the NeurIPS reviewers for insightful comments that helped us improve the positioning of our results, as well as for pointing out the subsampling approach for complementing the randomised lower bound. We also thank Foivos Alimisis and Peter Davies for useful discussions. This project has received funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML).","year":"2021","publication_status":"published","publisher":"Curran Associates","department":[{"_id":"DaAl"}],"month":"12","publication_identifier":{"issn":["1049-5258"],"isbn":["9781713845393"]},"conference":{"end_date":"2021-12-14","start_date":"2021-12-06","location":"Virtual, Online","name":"NeurIPS: Neural Information Processing Systems"},"language":[{"iso":"eng"}],"oa":1,"external_id":{"arxiv":["2010.08222"]},"main_file_link":[{"url":"https://proceedings.neurips.cc/paper/2021/file/3b92d18aa7a6176dd37d372bc2f1eb71-Paper.pdf","open_access":"1"}],"quality_controlled":"1","project":[{"_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning"}]},{"publication":"9th International Conference on Learning Representations","citation":{"ieee":"P. Davies, V. Gurunanthan, N. Moshrefi, S. Ashkboos, and D.-A. Alistarh, “New bounds for distributed mean estimation and variance reduction,” in 9th International Conference on Learning Representations, Virtual, 2021.","apa":"Davies, P., Gurunanthan, V., Moshrefi, N., Ashkboos, S., & Alistarh, D.-A. (2021). New bounds for distributed mean estimation and variance reduction. In 9th International Conference on Learning Representations. Virtual.","ista":"Davies P, Gurunanthan V, Moshrefi N, Ashkboos S, Alistarh D-A. 2021. New bounds for distributed mean estimation and variance reduction. 9th International Conference on Learning Representations. ICLR: International Conference on Learning Representations.","ama":"Davies P, Gurunanthan V, Moshrefi N, Ashkboos S, Alistarh D-A. New bounds for distributed mean estimation and variance reduction. In: 9th International Conference on Learning Representations. ; 2021.","chicago":"Davies, Peter, Vijaykrishna Gurunanthan, Niusha Moshrefi, Saleh Ashkboos, and Dan-Adrian Alistarh. “New Bounds for Distributed Mean Estimation and Variance Reduction.” In 9th International Conference on Learning Representations, 2021.","short":"P. Davies, V. Gurunanthan, N. Moshrefi, S. Ashkboos, D.-A. Alistarh, in:, 9th International Conference on Learning Representations, 2021.","mla":"Davies, Peter, et al. “New Bounds for Distributed Mean Estimation and Variance Reduction.” 9th International Conference on Learning Representations, 2021."},"oa":1,"main_file_link":[{"url":"https://openreview.net/pdf?id=t86MwoUCCNe","open_access":"1"}],"external_id":{"arxiv":["2002.09268"]},"quality_controlled":"1","project":[{"_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411","name":"ISTplus - Postdoctoral Fellowships","call_identifier":"H2020"}],"conference":{"end_date":"2021-05-07","location":"Virtual","start_date":"2021-05-03","name":" ICLR: International Conference on Learning Representations"},"date_published":"2021-05-01T00:00:00Z","language":[{"iso":"eng"}],"day":"01","month":"05","article_processing_charge":"No","year":"2021","_id":"9543","user_id":"D865714E-FA4E-11E9-B85B-F5C5E5697425","title":"New bounds for distributed mean estimation and variance reduction","status":"public","publication_status":"published","department":[{"_id":"DaAl"}],"author":[{"id":"11396234-BB50-11E9-B24C-90FCE5697425","orcid":"0000-0002-5646-9524","first_name":"Peter","last_name":"Davies","full_name":"Davies, Peter"},{"first_name":"Vijaykrishna","last_name":"Gurunanthan","full_name":"Gurunanthan, Vijaykrishna"},{"id":"4db776ff-ce15-11eb-96e3-bc2b90b01c16","first_name":"Niusha ","last_name":"Moshrefi","full_name":"Moshrefi, Niusha "},{"full_name":"Ashkboos, Saleh","id":"0D0A9058-257B-11EA-A937-9341C3D8BC8A","first_name":"Saleh","last_name":"Ashkboos"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"}],"date_updated":"2023-02-23T14:00:40Z","date_created":"2021-06-10T19:46:08Z","oa_version":"Published Version","type":"conference","abstract":[{"lang":"eng","text":"We consider the problem ofdistributed mean estimation (DME), in which n machines are each given a local d-dimensional vector xv∈Rd, and must cooperate to estimate the mean of their inputs μ=1n∑nv=1xv, while minimizing total communication cost. DME is a fundamental construct in distributed machine learning, and there has been considerable work on variants of this problem, especially in the context of distributed variance reduction for stochastic gradients in parallel SGD. Previous work typically assumes an upper bound on the norm of the input vectors, and achieves an error bound in terms of this norm. However, in many real applications, the input vectors are concentrated around the correct output μ, but μ itself has large norm. In such cases, previous output error bounds perform poorly. In this paper, we show that output error bounds need not depend on input norm. We provide a method of quantization which allows distributed mean estimation to be performed with solution quality dependent only on the distance between inputs, not on input norm, and show an analogous result for distributed variance reduction. The technique is based on a new connection with lattice theory. We also provide lower bounds showing that the communication to error trade-off of our algorithms is asymptotically optimal. As the lattices achieving optimal bounds under l2-norm can be computationally impractical, we also present an extension which leverages easy-to-use cubic lattices, and is loose only up to a logarithmic factor ind. We show experimentally that our method yields practical improvements for common applications, relative to prior approaches."}],"ec_funded":1},{"intvolume":" 12810","status":"public","ddc":["000"],"title":"Collecting coupons is faster with friends","user_id":"D865714E-FA4E-11E9-B85B-F5C5E5697425","_id":"9620","file":[{"date_updated":"2021-07-01T11:21:40Z","date_created":"2021-07-01T11:21:40Z","checksum":"fe37fb9af3f5016c1084af9d6e7109bd","file_id":"9621","relation":"main_file","creator":"pdavies","file_size":319728,"content_type":"application/pdf","file_name":"Population_Coupon_Collector.pdf","access_level":"open_access"}],"oa_version":"Preprint","alternative_title":["LNCS"],"type":"conference","abstract":[{"lang":"eng","text":"In this note, we introduce a distributed twist on the classic coupon collector problem: a set of m collectors wish to each obtain a set of n coupons; for this, they can each sample coupons uniformly at random, but can also meet in pairwise interactions, during which they can exchange coupons. By doing so, they hope to reduce the number of coupons that must be sampled by each collector in order to obtain a full set. This extension is natural when considering real-world manifestations of the coupon collector phenomenon, and has been remarked upon and studied empirically (Hayes and Hannigan 2006, Ahmad et al. 2014, Delmarcelle 2019).\r\n\r\nWe provide the first theoretical analysis for such a scenario. We find that “coupon collecting with friends” can indeed significantly reduce the number of coupons each collector must sample, and raises interesting connections to the more traditional variants of the problem. While our analysis is in most cases asymptotically tight, there are several open questions raised, regarding finer-grained analysis of both “coupon collecting with friends,” and of a long-studied variant of the original problem in which a collector requires multiple full sets of coupons."}],"page":"3-12","citation":{"ama":"Alistarh D-A, Davies P. Collecting coupons is faster with friends. In: Structural Information and Communication Complexity. Vol 12810. Springer Nature; 2021:3-12. doi:10.1007/978-3-030-79527-6_1","ieee":"D.-A. Alistarh and P. Davies, “Collecting coupons is faster with friends,” in Structural Information and Communication Complexity, Wrocław, Poland, 2021, vol. 12810, pp. 3–12.","apa":"Alistarh, D.-A., & Davies, P. (2021). Collecting coupons is faster with friends. In Structural Information and Communication Complexity (Vol. 12810, pp. 3–12). Wrocław, Poland: Springer Nature. https://doi.org/10.1007/978-3-030-79527-6_1","ista":"Alistarh D-A, Davies P. 2021. Collecting coupons is faster with friends. Structural Information and Communication Complexity. SIROCCO: International Colloquium on Structural Information and Communication Complexity, LNCS, vol. 12810, 3–12.","short":"D.-A. Alistarh, P. Davies, in:, Structural Information and Communication Complexity, Springer Nature, 2021, pp. 3–12.","mla":"Alistarh, Dan-Adrian, and Peter Davies. “Collecting Coupons Is Faster with Friends.” Structural Information and Communication Complexity, vol. 12810, Springer Nature, 2021, pp. 3–12, doi:10.1007/978-3-030-79527-6_1.","chicago":"Alistarh, Dan-Adrian, and Peter Davies. “Collecting Coupons Is Faster with Friends.” In Structural Information and Communication Complexity, 12810:3–12. Springer Nature, 2021. https://doi.org/10.1007/978-3-030-79527-6_1."},"publication":"Structural Information and Communication Complexity","date_published":"2021-06-20T00:00:00Z","has_accepted_license":"1","article_processing_charge":"No","day":"20","publisher":"Springer Nature","department":[{"_id":"DaAl"}],"publication_status":"published","year":"2021","acknowledgement":"Peter Davies is supported by the European Union’s Horizon2020 research and innovation programme under the Marie Skłodowska-Curie grant agreement No. 754411.","volume":12810,"date_updated":"2023-02-23T14:02:46Z","date_created":"2021-07-01T11:04:43Z","author":[{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"},{"orcid":"0000-0002-5646-9524","id":"11396234-BB50-11E9-B24C-90FCE5697425","last_name":"Davies","first_name":"Peter","full_name":"Davies, Peter"}],"ec_funded":1,"file_date_updated":"2021-07-01T11:21:40Z","project":[{"name":"ISTplus - Postdoctoral Fellowships","call_identifier":"H2020","grant_number":"754411","_id":"260C2330-B435-11E9-9278-68D0E5697425"}],"quality_controlled":"1","oa":1,"language":[{"iso":"eng"}],"doi":"10.1007/978-3-030-79527-6_1","conference":{"location":"Wrocław, Poland","start_date":"2021-06-28","end_date":"2021-07-01","name":" SIROCCO: International Colloquium on Structural Information and Communication Complexity"},"publication_identifier":{"eissn":["1611-3349"],"isbn":["9783030795269"],"issn":["0302-9743"],"eisbn":["9783030795276"]},"month":"06"},{"type":"conference","alternative_title":["LNCS"],"abstract":[{"text":"Approximate agreement is one of the few variants of consensus that can be solved in a wait-free manner in asynchronous systems where processes communicate by reading and writing to shared memory. In this work, we consider a natural generalisation of approximate agreement on arbitrary undirected connected graphs. Each process is given a vertex of the graph as input and, if non-faulty, must output a vertex such that\r\nall the outputs are within distance 1 of one another, and\r\n\r\neach output value lies on a shortest path between two input values.\r\n\r\nFrom prior work, it is known that there is no wait-free algorithm among 𝑛≥3 processes for this problem on any cycle of length 𝑐≥4 , by reduction from 2-set agreement (Castañeda et al. 2018).\r\n\r\nIn this work, we investigate the solvability and complexity of this task on general graphs. We give a new, direct proof of the impossibility of approximate agreement on cycles of length 𝑐≥4 , via a generalisation of Sperner’s Lemma to convex polygons. We also extend the reduction from 2-set agreement to a larger class of graphs, showing that approximate agreement on these graphs is unsolvable. On the positive side, we present a wait-free algorithm for a class of graphs that properly contains the class of chordal graphs.","lang":"eng"}],"_id":"9823","user_id":"6785fbc1-c503-11eb-8a32-93094b40e1cf","status":"public","title":"Wait-free approximate agreement on graphs","intvolume":" 12810","oa_version":"Preprint","scopus_import":"1","day":"20","article_processing_charge":"No","publication":"Structural Information and Communication Complexity","citation":{"short":"D.-A. Alistarh, F. Ellen, J. Rybicki, in:, Structural Information and Communication Complexity, Springer Nature, 2021, pp. 87–105.","mla":"Alistarh, Dan-Adrian, et al. “Wait-Free Approximate Agreement on Graphs.” Structural Information and Communication Complexity, vol. 12810, Springer Nature, 2021, pp. 87–105, doi:10.1007/978-3-030-79527-6_6.","chicago":"Alistarh, Dan-Adrian, Faith Ellen, and Joel Rybicki. “Wait-Free Approximate Agreement on Graphs.” In Structural Information and Communication Complexity, 12810:87–105. Springer Nature, 2021. https://doi.org/10.1007/978-3-030-79527-6_6.","ama":"Alistarh D-A, Ellen F, Rybicki J. Wait-free approximate agreement on graphs. In: Structural Information and Communication Complexity. Vol 12810. Springer Nature; 2021:87-105. doi:10.1007/978-3-030-79527-6_6","ieee":"D.-A. Alistarh, F. Ellen, and J. Rybicki, “Wait-free approximate agreement on graphs,” in Structural Information and Communication Complexity, Wrocław, Poland, 2021, vol. 12810, pp. 87–105.","apa":"Alistarh, D.-A., Ellen, F., & Rybicki, J. (2021). Wait-free approximate agreement on graphs. In Structural Information and Communication Complexity (Vol. 12810, pp. 87–105). Wrocław, Poland: Springer Nature. https://doi.org/10.1007/978-3-030-79527-6_6","ista":"Alistarh D-A, Ellen F, Rybicki J. 2021. Wait-free approximate agreement on graphs. Structural Information and Communication Complexity. SIROCCO: Structural Information and Communication Complexity, LNCS, vol. 12810, 87–105."},"page":"87-105","date_published":"2021-06-20T00:00:00Z","year":"2021","publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"Springer Nature","author":[{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"},{"full_name":"Ellen, Faith","first_name":"Faith","last_name":"Ellen"},{"last_name":"Rybicki","first_name":"Joel","orcid":"0000-0002-6432-6646","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","full_name":"Rybicki, Joel"}],"date_created":"2021-08-08T22:01:29Z","date_updated":"2023-02-23T14:09:49Z","volume":12810,"month":"06","publication_identifier":{"issn":["03029743"],"isbn":["9783030795269"],"eissn":["16113349"]},"oa":1,"main_file_link":[{"url":"https://arxiv.org/abs/2103.08949","open_access":"1"}],"external_id":{"arxiv":["2103.08949"]},"quality_controlled":"1","conference":{"end_date":"2021-07-01","start_date":"2021-06-28","location":"Wrocław, Poland","name":"SIROCCO: Structural Information and Communication Complexity"},"doi":"10.1007/978-3-030-79527-6_6","language":[{"iso":"eng"}]},{"scopus_import":"1","article_processing_charge":"No","day":"6","citation":{"ama":"Peste E-A, Iofinova EB, Vladu A, Alistarh D-A. AC/DC: Alternating Compressed/DeCompressed training of deep neural networks. In: 35th Conference on Neural Information Processing Systems. Vol 34. Curran Associates; 2021:8557-8570.","ista":"Peste E-A, Iofinova EB, Vladu A, Alistarh D-A. 2021. AC/DC: Alternating Compressed/DeCompressed training of deep neural networks. 35th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems vol. 34, 8557–8570.","ieee":"E.-A. Peste, E. B. Iofinova, A. Vladu, and D.-A. Alistarh, “AC/DC: Alternating Compressed/DeCompressed training of deep neural networks,” in 35th Conference on Neural Information Processing Systems, Virtual, Online, 2021, vol. 34, pp. 8557–8570.","apa":"Peste, E.-A., Iofinova, E. B., Vladu, A., & Alistarh, D.-A. (2021). AC/DC: Alternating Compressed/DeCompressed training of deep neural networks. In 35th Conference on Neural Information Processing Systems (Vol. 34, pp. 8557–8570). Virtual, Online: Curran Associates.","mla":"Peste, Elena-Alexandra, et al. “AC/DC: Alternating Compressed/DeCompressed Training of Deep Neural Networks.” 35th Conference on Neural Information Processing Systems, vol. 34, Curran Associates, 2021, pp. 8557–70.","short":"E.-A. Peste, E.B. Iofinova, A. Vladu, D.-A. Alistarh, in:, 35th Conference on Neural Information Processing Systems, Curran Associates, 2021, pp. 8557–8570.","chicago":"Peste, Elena-Alexandra, Eugenia B Iofinova, Adrian Vladu, and Dan-Adrian Alistarh. “AC/DC: Alternating Compressed/DeCompressed Training of Deep Neural Networks.” In 35th Conference on Neural Information Processing Systems, 34:8557–70. Curran Associates, 2021."},"publication":"35th Conference on Neural Information Processing Systems","page":"8557-8570","date_published":"2021-12-06T00:00:00Z","type":"conference","abstract":[{"lang":"eng","text":"The increasing computational requirements of deep neural networks (DNNs) have led to significant interest in obtaining DNN models that are sparse, yet accurate. Recent work has investigated the even harder case of sparse training, where the DNN weights are, for as much as possible, already sparse to reduce computational costs during training. Existing sparse training methods are often empirical and can have lower accuracy relative to the dense baseline. In this paper, we present a general approach called Alternating Compressed/DeCompressed (AC/DC) training of DNNs, demonstrate convergence for a variant of the algorithm, and show that AC/DC outperforms existing sparse training methods in accuracy at similar computational budgets; at high sparsity levels, AC/DC even outperforms existing methods that rely on accurate pre-trained dense models. An important property of AC/DC is that it allows co-training of dense and sparse models, yielding accurate sparse–dense model pairs at the end of the training process. This is useful in practice, where compressed variants may be desirable for deployment in resource-constrained settings without re-doing the entire training flow, and also provides us with insights into the accuracy gap between dense and compressed models. The code is available at: https://github.com/IST-DASLab/ACDC."}],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"11458","intvolume":" 34","title":"AC/DC: Alternating Compressed/DeCompressed training of deep neural networks","status":"public","oa_version":"Published Version","publication_identifier":{"issn":["1049-5258"],"isbn":["9781713845393"]},"month":"12","main_file_link":[{"url":"https://proceedings.neurips.cc/paper/2021/file/48000647b315f6f00f913caa757a70b3-Paper.pdf","open_access":"1"}],"external_id":{"arxiv":["2106.12379"]},"oa":1,"project":[{"_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020"}],"quality_controlled":"1","conference":{"start_date":"2021-12-06","location":"Virtual, Online","end_date":"2021-12-14","name":"NeurIPS: Neural Information Processing Systems"},"language":[{"iso":"eng"}],"acknowledged_ssus":[{"_id":"ScienComp"}],"ec_funded":1,"acknowledgement":"This project has received funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML), and a CNRS PEPS grant. This research was supported by the Scientific Service Units (SSU) of IST Austria through resources provided by Scientific Computing (SciComp). We would also like to thank Christoph Lampert for his feedback on an earlier version of this work, as well as for providing hardware for the Transformer-XL experiments.","year":"2021","department":[{"_id":"GradSch"},{"_id":"DaAl"}],"publisher":"Curran Associates","publication_status":"published","related_material":{"record":[{"relation":"dissertation_contains","status":"public","id":"13074"}]},"author":[{"full_name":"Peste, Elena-Alexandra","id":"32D78294-F248-11E8-B48F-1D18A9856A87","last_name":"Peste","first_name":"Elena-Alexandra"},{"orcid":"0000-0002-7778-3221","id":"f9a17499-f6e0-11ea-865d-fdf9a3f77117","last_name":"Iofinova","first_name":"Eugenia B","full_name":"Iofinova, Eugenia B"},{"last_name":"Vladu","first_name":"Adrian","full_name":"Vladu, Adrian"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"volume":34,"date_updated":"2023-06-01T12:54:45Z","date_created":"2022-06-20T12:11:53Z"},{"ddc":["000"],"status":"public","title":"Communication-efficient distributed optimization with quantized preconditioners","intvolume":" 139","_id":"13147","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","oa_version":"Published Version","file":[{"relation":"main_file","file_id":"13154","checksum":"7ec0d59bac268b49c76bf2e036dedd7a","success":1,"date_created":"2023-06-19T10:41:05Z","date_updated":"2023-06-19T10:41:05Z","access_level":"open_access","file_name":"2021_PMLR_Alimisis.pdf","content_type":"application/pdf","file_size":429087,"creator":"dernst"}],"type":"conference","abstract":[{"text":"We investigate fast and communication-efficient algorithms for the classic problem of minimizing a sum of strongly convex and smooth functions that are distributed among n\r\n different nodes, which can communicate using a limited number of bits. Most previous communication-efficient approaches for this problem are limited to first-order optimization, and therefore have \\emph{linear} dependence on the condition number in their communication complexity. We show that this dependence is not inherent: communication-efficient methods can in fact have sublinear dependence on the condition number. For this, we design and analyze the first communication-efficient distributed variants of preconditioned gradient descent for Generalized Linear Models, and for Newton’s method. Our results rely on a new technique for quantizing both the preconditioner and the descent direction at each step of the algorithms, while controlling their convergence rate. We also validate our findings experimentally, showing faster convergence and reduced communication relative to previous methods.","lang":"eng"}],"page":"196-206","publication":"Proceedings of the 38th International Conference on Machine Learning","citation":{"ista":"Alimisis F, Davies P, Alistarh D-A. 2021. Communication-efficient distributed optimization with quantized preconditioners. Proceedings of the 38th International Conference on Machine Learning. International Conference on Machine Learning vol. 139, 196–206.","ieee":"F. Alimisis, P. Davies, and D.-A. Alistarh, “Communication-efficient distributed optimization with quantized preconditioners,” in Proceedings of the 38th International Conference on Machine Learning, Virtual, 2021, vol. 139, pp. 196–206.","apa":"Alimisis, F., Davies, P., & Alistarh, D.-A. (2021). Communication-efficient distributed optimization with quantized preconditioners. In Proceedings of the 38th International Conference on Machine Learning (Vol. 139, pp. 196–206). Virtual: ML Research Press.","ama":"Alimisis F, Davies P, Alistarh D-A. Communication-efficient distributed optimization with quantized preconditioners. In: Proceedings of the 38th International Conference on Machine Learning. Vol 139. ML Research Press; 2021:196-206.","chicago":"Alimisis, Foivos, Peter Davies, and Dan-Adrian Alistarh. “Communication-Efficient Distributed Optimization with Quantized Preconditioners.” In Proceedings of the 38th International Conference on Machine Learning, 139:196–206. ML Research Press, 2021.","mla":"Alimisis, Foivos, et al. “Communication-Efficient Distributed Optimization with Quantized Preconditioners.” Proceedings of the 38th International Conference on Machine Learning, vol. 139, ML Research Press, 2021, pp. 196–206.","short":"F. Alimisis, P. Davies, D.-A. Alistarh, in:, Proceedings of the 38th International Conference on Machine Learning, ML Research Press, 2021, pp. 196–206."},"date_published":"2021-07-01T00:00:00Z","scopus_import":"1","day":"01","has_accepted_license":"1","article_processing_charge":"No","publication_status":"published","publisher":"ML Research Press","department":[{"_id":"DaAl"}],"year":"2021","acknowledgement":"The authors would like to thank Janne Korhonen, Aurelien Lucchi, Celestine MendlerDunner and Antonio Orvieto for helpful discussions. FA ¨and DA were supported during this work by the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). PD was supported by the European Union’s Horizon 2020 programme under the Marie Skłodowska-Curie grant agreement No. 754411.","date_created":"2023-06-18T22:00:48Z","date_updated":"2023-06-19T10:44:38Z","volume":139,"author":[{"full_name":"Alimisis, Foivos","last_name":"Alimisis","first_name":"Foivos"},{"full_name":"Davies, Peter","id":"11396234-BB50-11E9-B24C-90FCE5697425","orcid":"0000-0002-5646-9524","first_name":"Peter","last_name":"Davies"},{"first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","full_name":"Alistarh, Dan-Adrian"}],"file_date_updated":"2023-06-19T10:41:05Z","ec_funded":1,"quality_controlled":"1","project":[{"_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020"},{"_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411","call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships"}],"external_id":{"arxiv":["2102.07214"]},"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"language":[{"iso":"eng"}],"conference":{"name":"International Conference on Machine Learning","end_date":"2021-07-24","start_date":"2021-07-18","location":"Virtual"},"month":"07","publication_identifier":{"isbn":["9781713845065"],"eissn":["2640-3498"]}},{"type":"journal_article","abstract":[{"lang":"eng","text":"Deep learning at scale is dominated by communication time. Distributing samples across nodes usually yields the best performance, but poses scaling challenges due to global information dissemination and load imbalance across uneven sample lengths. State-of-the-art decentralized optimizers mitigate the problem, but require more iterations to achieve the same accuracy as their globally-communicating counterparts. We present Wait-Avoiding Group Model Averaging (WAGMA) SGD, a wait-avoiding stochastic optimizer that reduces global communication via subgroup weight exchange. The key insight is a combination of algorithmic changes to the averaging scheme and the use of a group allreduce operation. We prove the convergence of WAGMA-SGD, and empirically show that it retains convergence rates similar to Allreduce-SGD. For evaluation, we train ResNet-50 on ImageNet; Transformer for machine translation; and deep reinforcement learning for navigation at scale. Compared with state-of-the-art decentralized SGD variants, WAGMA-SGD significantly improves training throughput (e.g., 2.1× on 1,024 GPUs for reinforcement learning), and achieves the fastest time-to-solution (e.g., the highest score using the shortest training time for Transformer)."}],"issue":"7","title":"Breaking (global) barriers in parallel stochastic optimization with wait-avoiding group averaging","status":"public","intvolume":" 32","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","_id":"8723","oa_version":"Preprint","scopus_import":"1","day":"01","article_processing_charge":"No","article_type":"original","publication":"IEEE Transactions on Parallel and Distributed Systems","citation":{"ama":"Li S, Tal Ben-Nun TB-N, Nadiradze G, et al. Breaking (global) barriers in parallel stochastic optimization with wait-avoiding group averaging. IEEE Transactions on Parallel and Distributed Systems. 2021;32(7). doi:10.1109/TPDS.2020.3040606","ista":"Li S, Tal Ben-Nun TB-N, Nadiradze G, Girolamo SD, Dryden N, Alistarh D-A, Hoefler T. 2021. Breaking (global) barriers in parallel stochastic optimization with wait-avoiding group averaging. IEEE Transactions on Parallel and Distributed Systems. 32(7), 9271898.","ieee":"S. Li et al., “Breaking (global) barriers in parallel stochastic optimization with wait-avoiding group averaging,” IEEE Transactions on Parallel and Distributed Systems, vol. 32, no. 7. IEEE, 2021.","apa":"Li, S., Tal Ben-Nun, T. B.-N., Nadiradze, G., Girolamo, S. D., Dryden, N., Alistarh, D.-A., & Hoefler, T. (2021). Breaking (global) barriers in parallel stochastic optimization with wait-avoiding group averaging. IEEE Transactions on Parallel and Distributed Systems. IEEE. https://doi.org/10.1109/TPDS.2020.3040606","mla":"Li, Shigang, et al. “Breaking (Global) Barriers in Parallel Stochastic Optimization with Wait-Avoiding Group Averaging.” IEEE Transactions on Parallel and Distributed Systems, vol. 32, no. 7, 9271898, IEEE, 2021, doi:10.1109/TPDS.2020.3040606.","short":"S. Li, T.B.-N. Tal Ben-Nun, G. Nadiradze, S.D. Girolamo, N. Dryden, D.-A. Alistarh, T. Hoefler, IEEE Transactions on Parallel and Distributed Systems 32 (2021).","chicago":"Li, Shigang, Tal Ben-Nun Tal Ben-Nun, Giorgi Nadiradze, Salvatore Di Girolamo, Nikoli Dryden, Dan-Adrian Alistarh, and Torsten Hoefler. “Breaking (Global) Barriers in Parallel Stochastic Optimization with Wait-Avoiding Group Averaging.” IEEE Transactions on Parallel and Distributed Systems. IEEE, 2021. https://doi.org/10.1109/TPDS.2020.3040606."},"date_published":"2021-07-01T00:00:00Z","article_number":"9271898","ec_funded":1,"publication_status":"published","publisher":"IEEE","department":[{"_id":"DaAl"}],"year":"2021","acknowledgement":"This project has received funding from the European Research Council (ERC) under the European Union’s Hori-\r\nzon 2020 programme under Grant DAPP, Grant 678880; EPi-GRAM-HS, Grant 801039; and ERC Starting Grant ScaleML, Grant 805223. The work of Tal Ben-Nun is supported by the Swiss National Science Foundation (Ambizione Project No. 185778). The work of Nikoli Dryden is supported by the ETH Postdoctoral Fellowship. The authors would like to thank the Swiss National Supercomputing Center for providing the computing resources and technical support.","date_created":"2020-11-05T15:25:43Z","date_updated":"2023-08-04T11:08:52Z","volume":32,"author":[{"last_name":"Li","first_name":"Shigang","full_name":"Li, Shigang"},{"full_name":"Tal Ben-Nun, Tal Ben-Nun","last_name":"Tal Ben-Nun","first_name":"Tal Ben-Nun"},{"full_name":"Nadiradze, Giorgi","first_name":"Giorgi","last_name":"Nadiradze","id":"3279A00C-F248-11E8-B48F-1D18A9856A87"},{"full_name":"Girolamo, Salvatore Di","first_name":"Salvatore Di","last_name":"Girolamo"},{"first_name":"Nikoli","last_name":"Dryden","full_name":"Dryden, Nikoli"},{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X"},{"first_name":"Torsten","last_name":"Hoefler","full_name":"Hoefler, Torsten"}],"month":"07","publication_identifier":{"issn":["10459219"]},"isi":1,"quality_controlled":"1","project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"oa":1,"external_id":{"isi":["000621405200019"],"arxiv":["2005.00124"]},"main_file_link":[{"url":"https://arxiv.org/abs/2005.00124","open_access":"1"}],"language":[{"iso":"eng"}],"doi":"10.1109/TPDS.2020.3040606"},{"keyword":["Concurrent data structure","kD-tree","Nearest neighbor search","Similarity search","Lock-free","Linearizability"],"scopus_import":"1","day":"13","article_processing_charge":"No","article_type":"original","page":"27-48","publication":"Theoretical Computer Science","citation":{"apa":"Chatterjee, B., Walulya, I., & Tsigas, P. (2021). Concurrent linearizable nearest neighbour search in LockFree-kD-tree. Theoretical Computer Science. Elsevier. https://doi.org/10.1016/j.tcs.2021.06.041","ieee":"B. Chatterjee, I. Walulya, and P. Tsigas, “Concurrent linearizable nearest neighbour search in LockFree-kD-tree,” Theoretical Computer Science, vol. 886. Elsevier, pp. 27–48, 2021.","ista":"Chatterjee B, Walulya I, Tsigas P. 2021. Concurrent linearizable nearest neighbour search in LockFree-kD-tree. Theoretical Computer Science. 886, 27–48.","ama":"Chatterjee B, Walulya I, Tsigas P. Concurrent linearizable nearest neighbour search in LockFree-kD-tree. Theoretical Computer Science. 2021;886:27-48. doi:10.1016/j.tcs.2021.06.041","chicago":"Chatterjee, Bapi, Ivan Walulya, and Philippas Tsigas. “Concurrent Linearizable Nearest Neighbour Search in LockFree-KD-Tree.” Theoretical Computer Science. Elsevier, 2021. https://doi.org/10.1016/j.tcs.2021.06.041.","short":"B. Chatterjee, I. Walulya, P. Tsigas, Theoretical Computer Science 886 (2021) 27–48.","mla":"Chatterjee, Bapi, et al. “Concurrent Linearizable Nearest Neighbour Search in LockFree-KD-Tree.” Theoretical Computer Science, vol. 886, Elsevier, 2021, pp. 27–48, doi:10.1016/j.tcs.2021.06.041."},"date_published":"2021-09-13T00:00:00Z","type":"journal_article","abstract":[{"text":"The Nearest neighbour search (NNS) is a fundamental problem in many application domains dealing with multidimensional data. In a concurrent setting, where dynamic modifications are allowed, a linearizable implementation of the NNS is highly desirable.This paper introduces the LockFree-kD-tree (LFkD-tree ): a lock-free concurrent kD-tree, which implements an abstract data type (ADT) that provides the operations Add, Remove, Contains, and NNS. Our implementation is linearizable. The operations in the LFkD-tree use single-word read and compare-and-swap (Image 1 ) atomic primitives, which are readily supported on available multi-core processors. We experimentally evaluate the LFkD-tree using several benchmarks comprising real-world and synthetic datasets. The experiments show that the presented design is scalable and achieves significant speed-up compared to the implementations of an existing sequential kD-tree and a recently proposed multidimensional indexing structure, PH-tree.","lang":"eng"}],"status":"public","title":"Concurrent linearizable nearest neighbour search in LockFree-kD-tree","intvolume":" 886","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","_id":"9827","oa_version":"Submitted Version","month":"09","publication_identifier":{"issn":["0304-3975"]},"isi":1,"quality_controlled":"1","external_id":{"isi":["000694718900004"]},"main_file_link":[{"open_access":"1","url":"https://publications.lib.chalmers.se/records/fulltext/232185/232185.pdf"}],"oa":1,"language":[{"iso":"eng"}],"doi":"10.1016/j.tcs.2021.06.041","publication_status":"published","publisher":"Elsevier","department":[{"_id":"DaAl"}],"year":"2021","date_updated":"2023-08-10T14:27:43Z","date_created":"2021-08-08T22:01:31Z","volume":886,"author":[{"first_name":"Bapi","last_name":"Chatterjee","id":"3C41A08A-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0002-2742-4028","full_name":"Chatterjee, Bapi"},{"full_name":"Walulya, Ivan","first_name":"Ivan","last_name":"Walulya"},{"full_name":"Tsigas, Philippas","first_name":"Philippas","last_name":"Tsigas"}]},{"oa_version":"None","date_updated":"2023-08-11T10:56:04Z","date_created":"2021-08-22T22:01:20Z","author":[{"full_name":"Alistarh, Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh"},{"full_name":"Töpfer, Martin","id":"4B865388-F248-11E8-B48F-1D18A9856A87","first_name":"Martin","last_name":"Töpfer"},{"last_name":"Uznański","first_name":"Przemysław","full_name":"Uznański, Przemysław"}],"publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"title":"Comparison dynamics in population protocols","status":"public","publication_status":"published","_id":"9951","year":"2021","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","acknowledgement":"We would like to thank Rati Gelashvili for very useful discussions, and the PODC anonymous reviewers for their careful reading of our paper, and for their useful remarks. This work is partially supported by the Polish National Science Center (NCN) grant UMO2017/25/B/ST6/02010.","abstract":[{"text":"There has recently been a surge of interest in the computational and complexity properties of the population model, which assumes n anonymous, computationally-bounded nodes, interacting at random, with the goal of jointly computing global predicates. Significant work has gone towards investigating majority or consensus dynamics in this model: that is, assuming that every node is initially in one of two states X or Y, determine which state had higher initial count.\r\n\r\nIn this paper, we consider a natural generalization of majority/consensus, which we call comparison : in its simplest formulation, we are given two baseline states, X and Y, present in any initial configuration in fixed, but possibly small counts. One of these states has higher count than the other: we will assume |X_0| > C |Y_0| for some constant C > 1. The challenge is to design a protocol by which nodes can quickly and reliably decide on which of the baseline states X_0 and Y_0 has higher initial count. We begin by analyzing a simple and general dynamics solving the above comparison problem, which uses O( log n ) states per node, and converges in O(log n) (parallel) time, with high probability, to a state where the whole population votes on opinions X or Y at rates proportional to the initial concentrations of |X_0| vs. |Y_0|. We then describe how this procedure can be bootstrapped to solve comparison, i.e. have every node in the population reach the \"correct'' decision, with probability 1 - o(1), at the cost of O (log log n) additional states. Further, we prove that this dynamics is self-stabilizing, in the sense that it converges to the correct decision from arbitrary initial states, and leak-robust, in the sense that it can withstand spurious faulty reactions, which are known to occur in practical implementations of population protocols. Our analysis is based on a new martingale concentration result relating the discrete-time evolution of a population protocol to its expected (steady-state) analysis, which should be a useful tool when analyzing opinion dynamics and epidemic dissemination in the population model.","lang":"eng"}],"type":"conference","language":[{"iso":"eng"}],"doi":"10.1145/3465084.3467915","date_published":"2021-07-21T00:00:00Z","conference":{"location":"Virtual, Italy","start_date":"2021-07-26","end_date":"2021-07-30","name":"PODC: Symposium on Principles of Distributed Computing"},"page":"55-65","isi":1,"quality_controlled":"1","citation":{"ama":"Alistarh D-A, Töpfer M, Uznański P. Comparison dynamics in population protocols. In: Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing. Association for Computing Machinery; 2021:55-65. doi:10.1145/3465084.3467915","ieee":"D.-A. Alistarh, M. Töpfer, and P. Uznański, “Comparison dynamics in population protocols,” in Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, Virtual, Italy, 2021, pp. 55–65.","apa":"Alistarh, D.-A., Töpfer, M., & Uznański, P. (2021). Comparison dynamics in population protocols. In Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing (pp. 55–65). Virtual, Italy: Association for Computing Machinery. https://doi.org/10.1145/3465084.3467915","ista":"Alistarh D-A, Töpfer M, Uznański P. 2021. Comparison dynamics in population protocols. Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing. PODC: Symposium on Principles of Distributed Computing, 55–65.","short":"D.-A. Alistarh, M. Töpfer, P. Uznański, in:, Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2021, pp. 55–65.","mla":"Alistarh, Dan-Adrian, et al. “Comparison Dynamics in Population Protocols.” Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2021, pp. 55–65, doi:10.1145/3465084.3467915.","chicago":"Alistarh, Dan-Adrian, Martin Töpfer, and Przemysław Uznański. “Comparison Dynamics in Population Protocols.” In Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, 55–65. Association for Computing Machinery, 2021. https://doi.org/10.1145/3465084.3467915."},"external_id":{"isi":["000744439800005"]},"publication":"Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing","publication_identifier":{"isbn":["9781450385480"]},"article_processing_charge":"No","month":"07","day":"21","scopus_import":"1"},{"type":"conference","abstract":[{"text":"We present a deterministic O(log log log n)-round low-space Massively Parallel Computation (MPC) algorithm for the classical problem of (Δ+1)-coloring on n-vertex graphs. In this model, every machine has sublinear local space of size n^φ for any arbitrary constant φ \\in (0,1). Our algorithm works under the relaxed setting where each machine is allowed to perform exponential local computations, while respecting the n^φ space and bandwidth limitations.\r\n\r\nOur key technical contribution is a novel derandomization of the ingenious (Δ+1)-coloring local algorithm by Chang-Li-Pettie (STOC 2018, SIAM J. Comput. 2020). The Chang-Li-Pettie algorithm runs in T_local =poly(loglog n) rounds, which sets the state-of-the-art randomized round complexity for the problem in the local model. Our derandomization employs a combination of tools, notably pseudorandom generators (PRG) and bounded-independence hash functions.\r\n\r\nThe achieved round complexity of O(logloglog n) rounds matches the bound of log(T_local ), which currently serves an upper bound barrier for all known randomized algorithms for locally-checkable problems in this model. Furthermore, no deterministic sublogarithmic low-space MPC algorithms for the (Δ+1)-coloring problem have been known before.","lang":"eng"}],"user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","_id":"9935","status":"public","title":"Improved deterministic (Δ+1) coloring in low-space MPC","oa_version":"Submitted Version","article_processing_charge":"No","day":"21","citation":{"ista":"Czumaj A, Davies P, Parter M. 2021. Improved deterministic (Δ+1) coloring in low-space MPC. Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing. PODC: Symposium on Principles of Distributed Computing, 469–479.","ieee":"A. Czumaj, P. Davies, and M. Parter, “Improved deterministic (Δ+1) coloring in low-space MPC,” in Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, Virtual, Italy, 2021, pp. 469–479.","apa":"Czumaj, A., Davies, P., & Parter, M. (2021). Improved deterministic (Δ+1) coloring in low-space MPC. In Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing (pp. 469–479). Virtual, Italy: Association for Computing Machinery. https://doi.org/10.1145/3465084.3467937","ama":"Czumaj A, Davies P, Parter M. Improved deterministic (Δ+1) coloring in low-space MPC. In: Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing. Association for Computing Machinery; 2021:469–479. doi:10.1145/3465084.3467937","chicago":"Czumaj, Artur, Peter Davies, and Merav Parter. “Improved Deterministic (Δ+1) Coloring in Low-Space MPC.” In Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, 469–479. Association for Computing Machinery, 2021. https://doi.org/10.1145/3465084.3467937.","mla":"Czumaj, Artur, et al. “Improved Deterministic (Δ+1) Coloring in Low-Space MPC.” Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2021, pp. 469–479, doi:10.1145/3465084.3467937.","short":"A. Czumaj, P. Davies, M. Parter, in:, Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2021, pp. 469–479."},"publication":"Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing","page":"469–479","date_published":"2021-07-21T00:00:00Z","ec_funded":1,"acknowledgement":"This work is partially supported by a Weizmann-UK Making Connections Grant, the Centre for Discrete Mathematics and its Applications (DIMAP), IBM Faculty Award, EPSRC award EP/V01305X/1, European Research Council (ERC) Grant No. 949083, the Minerva foundation with funding from the Federal German Ministry for Education and Research No. 713238, and the European Union’s Horizon 2020 programme under the Marie Skłodowska-Curie grant agreement No 754411.","year":"2021","publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"publication_status":"published","author":[{"full_name":"Czumaj, Artur","last_name":"Czumaj","first_name":"Artur"},{"orcid":"0000-0002-5646-9524","id":"11396234-BB50-11E9-B24C-90FCE5697425","last_name":"Davies","first_name":"Peter","full_name":"Davies, Peter"},{"last_name":"Parter","first_name":"Merav","full_name":"Parter, Merav"}],"date_created":"2021-08-17T18:14:15Z","date_updated":"2023-08-17T07:11:03Z","publication_identifier":{"isbn":["978-1-4503-8548-0"]},"month":"07","external_id":{"isi":["000744439800048"]},"main_file_link":[{"open_access":"1","url":"http://wrap.warwick.ac.uk/153753"}],"oa":1,"project":[{"_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411","name":"ISTplus - Postdoctoral Fellowships","call_identifier":"H2020"}],"isi":1,"quality_controlled":"1","doi":"10.1145/3465084.3467937","conference":{"name":"PODC: Symposium on Principles of Distributed Computing","end_date":"2021-07-30","location":"Virtual, Italy","start_date":"2021-07-26"},"language":[{"iso":"eng"}]},{"title":"Component stability in low-space massively parallel computation","status":"public","_id":"9933","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","oa_version":"Submitted Version","type":"conference","abstract":[{"lang":"eng","text":"In this paper, we study the power and limitations of component-stable algorithms in the low-space model of Massively Parallel Computation (MPC). Recently Ghaffari, Kuhn and Uitto (FOCS 2019) introduced the class of component-stable low-space MPC algorithms, which are, informally, defined as algorithms for which the outputs reported by the nodes in different connected components are required to be independent. This very natural notion was introduced to capture most (if not all) of the known efficient MPC algorithms to date, and it was the first general class of MPC algorithms for which one can show non-trivial conditional lower bounds. In this paper we enhance the framework of component-stable algorithms and investigate its effect on the complexity of randomized and deterministic low-space MPC. Our key contributions include: 1) We revise and formalize the lifting approach of Ghaffari, Kuhn and Uitto. This requires a very delicate amendment of the notion of component stability, which allows us to fill in gaps in the earlier arguments. 2) We also extend the framework to obtain conditional lower bounds for deterministic algorithms and fine-grained lower bounds that depend on the maximum degree Δ. 3) We demonstrate a collection of natural graph problems for which non-component-stable algorithms break the conditional lower bound obtained for component-stable algorithms. This implies that, for both deterministic and randomized algorithms, component-stable algorithms are conditionally weaker than the non-component-stable ones.\r\n\r\nAltogether our results imply that component-stability might limit the computational power of the low-space MPC model, paving the way for improved upper bounds that escape the conditional lower bound setting of Ghaffari, Kuhn, and Uitto."}],"page":"481–491","citation":{"mla":"Czumaj, Artur, et al. “Component Stability in Low-Space Massively Parallel Computation.” Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2021, pp. 481–491, doi:10.1145/3465084.3467903.","short":"A. Czumaj, P. Davies, M. Parter, in:, Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2021, pp. 481–491.","chicago":"Czumaj, Artur, Peter Davies, and Merav Parter. “Component Stability in Low-Space Massively Parallel Computation.” In Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, 481–491. Association for Computing Machinery, 2021. https://doi.org/10.1145/3465084.3467903.","ama":"Czumaj A, Davies P, Parter M. Component stability in low-space massively parallel computation. In: Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing. Association for Computing Machinery; 2021:481–491. doi:10.1145/3465084.3467903","ista":"Czumaj A, Davies P, Parter M. 2021. Component stability in low-space massively parallel computation. Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing. PODC: Principles of Distributed Computing, 481–491.","apa":"Czumaj, A., Davies, P., & Parter, M. (2021). Component stability in low-space massively parallel computation. In Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing (pp. 481–491). Virtual, Italy: Association for Computing Machinery. https://doi.org/10.1145/3465084.3467903","ieee":"A. Czumaj, P. Davies, and M. Parter, “Component stability in low-space massively parallel computation,” in Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing, Virtual, Italy, 2021, pp. 481–491."},"publication":"Proceedings of the 2021 ACM Symposium on Principles of Distributed Computing","date_published":"2021-07-21T00:00:00Z","article_processing_charge":"No","day":"21","publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"publication_status":"published","acknowledgement":"This work is partially supported by a Weizmann-UK Making Connections Grant, the Centre for Discrete Mathematics and its Applications (DIMAP), IBM Faculty Award, EPSRC award EP/V01305X/1, European Research Council (ERC) Grant No. 949083, the Minerva foundation with funding from the Federal German Ministry for Education and Research No. 713238, and the European Union’s Horizon 2020 programme under the Marie Skłodowska-Curie grant agreement No 754411.","year":"2021","date_created":"2021-08-17T18:11:16Z","date_updated":"2023-08-17T07:11:32Z","author":[{"full_name":"Czumaj, Artur","last_name":"Czumaj","first_name":"Artur"},{"id":"11396234-BB50-11E9-B24C-90FCE5697425","orcid":"0000-0002-5646-9524","first_name":"Peter","last_name":"Davies","full_name":"Davies, Peter"},{"first_name":"Merav","last_name":"Parter","full_name":"Parter, Merav"}],"ec_funded":1,"project":[{"grant_number":"754411","_id":"260C2330-B435-11E9-9278-68D0E5697425","name":"ISTplus - Postdoctoral Fellowships","call_identifier":"H2020"}],"quality_controlled":"1","isi":1,"main_file_link":[{"url":"https://arxiv.org/abs/2106.01880","open_access":"1"}],"oa":1,"external_id":{"isi":["000744439800049"],"arxiv":["2106.01880"]},"language":[{"iso":"eng"}],"doi":"10.1145/3465084.3467903","conference":{"name":"PODC: Principles of Distributed Computing","end_date":"2021-07-30","start_date":"2021-07-26","location":"Virtual, Italy"},"publication_identifier":{"isbn":["9781450385480"]},"month":"07"},{"date_published":"2021-05-18T00:00:00Z","page":"9037-9045","publication":"Proceedings of the AAAI Conference on Artificial Intelligence","citation":{"ista":"Nadiradze G, Markov I, Chatterjee B, Kungurtsev V, Alistarh D-A. 2021. Elastic consistency: A practical consistency model for distributed stochastic gradient descent. Proceedings of the AAAI Conference on Artificial Intelligence. AAAI: Association for the Advancement of Artificial Intelligence vol. 35, 9037–9045.","ieee":"G. Nadiradze, I. Markov, B. Chatterjee, V. Kungurtsev, and D.-A. Alistarh, “Elastic consistency: A practical consistency model for distributed stochastic gradient descent,” in Proceedings of the AAAI Conference on Artificial Intelligence, Virtual, 2021, vol. 35, no. 10, pp. 9037–9045.","apa":"Nadiradze, G., Markov, I., Chatterjee, B., Kungurtsev, V., & Alistarh, D.-A. (2021). Elastic consistency: A practical consistency model for distributed stochastic gradient descent. In Proceedings of the AAAI Conference on Artificial Intelligence (Vol. 35, pp. 9037–9045). Virtual.","ama":"Nadiradze G, Markov I, Chatterjee B, Kungurtsev V, Alistarh D-A. Elastic consistency: A practical consistency model for distributed stochastic gradient descent. In: Proceedings of the AAAI Conference on Artificial Intelligence. Vol 35. ; 2021:9037-9045.","chicago":"Nadiradze, Giorgi, Ilia Markov, Bapi Chatterjee, Vyacheslav Kungurtsev, and Dan-Adrian Alistarh. “Elastic Consistency: A Practical Consistency Model for Distributed Stochastic Gradient Descent.” In Proceedings of the AAAI Conference on Artificial Intelligence, 35:9037–45, 2021.","mla":"Nadiradze, Giorgi, et al. “Elastic Consistency: A Practical Consistency Model for Distributed Stochastic Gradient Descent.” Proceedings of the AAAI Conference on Artificial Intelligence, vol. 35, no. 10, 2021, pp. 9037–45.","short":"G. Nadiradze, I. Markov, B. Chatterjee, V. Kungurtsev, D.-A. Alistarh, in:, Proceedings of the AAAI Conference on Artificial Intelligence, 2021, pp. 9037–9045."},"day":"18","article_processing_charge":"No","oa_version":"Published Version","title":"Elastic consistency: A practical consistency model for distributed stochastic gradient descent","status":"public","intvolume":" 35","_id":"10432","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","abstract":[{"lang":"eng","text":"One key element behind the recent progress of machine learning has been the ability to train machine learning models in large-scale distributed shared-memory and message-passing environments. Most of these models are trained employing variants of stochastic gradient descent (SGD) based optimization, but most methods involve some type of consistency relaxation relative to sequential SGD, to mitigate its large communication or synchronization costs at scale. In this paper, we introduce a general consistency condition covering communication-reduced and asynchronous distributed SGD implementations. Our framework, called elastic consistency, decouples the system-specific aspects of the implementation from the SGD convergence requirements, giving a general way to obtain convergence bounds for a wide variety of distributed SGD methods used in practice. Elastic consistency can be used to re-derive or improve several previous convergence bounds in message-passing and shared-memory settings, but also to analyze new models and distribution schemes. As a direct application, we propose and analyze a new synchronization-avoiding scheduling scheme for distributed SGD, and show that it can be used to efficiently train deep convolutional models for image classification."}],"issue":"10","type":"conference","language":[{"iso":"eng"}],"conference":{"end_date":"2021-02-09","start_date":"2021-02-02","location":"Virtual","name":"AAAI: Association for the Advancement of Artificial Intelligence"},"quality_controlled":"1","project":[{"_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411","call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships"},{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"oa":1,"main_file_link":[{"url":"https://ojs.aaai.org/index.php/AAAI/article/view/17092","open_access":"1"}],"external_id":{"arxiv":["2001.05918"]},"month":"05","date_created":"2021-12-09T09:21:35Z","date_updated":"2023-09-07T13:31:39Z","volume":35,"author":[{"first_name":"Giorgi","last_name":"Nadiradze","id":"3279A00C-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0001-5634-0731","full_name":"Nadiradze, Giorgi"},{"last_name":"Markov","first_name":"Ilia","id":"D0CF4148-C985-11E9-8066-0BDEE5697425","full_name":"Markov, Ilia"},{"full_name":"Chatterjee, Bapi","last_name":"Chatterjee","first_name":"Bapi","orcid":"0000-0002-2742-4028","id":"3C41A08A-F248-11E8-B48F-1D18A9856A87"},{"full_name":"Kungurtsev, Vyacheslav ","last_name":"Kungurtsev","first_name":"Vyacheslav "},{"full_name":"Alistarh, Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh"}],"related_material":{"record":[{"status":"public","relation":"dissertation_contains","id":"10429"}]},"publication_status":"published","department":[{"_id":"DaAl"}],"acknowledgement":"We would like to thank Christopher De Sa for his feedback on an earlier draft of this paper, as well as the anonymous AAAI reviewers for their useful comments. This project has received\r\nfunding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). Bapi\r\nChatterjee was supported by the European Union’s Horizon 2020 research and innovation programme under the Marie Sklodowska-Curie grant agreement No. 754411 (ISTPlus).","year":"2021","ec_funded":1},{"article_processing_charge":"No","day":"26","citation":{"mla":"Klein, Karen, et al. “Keep the Dirt: Tainted TreeKEM, Adaptively and Actively Secure Continuous Group Key Agreement.” 2021 IEEE Symposium on Security and Privacy , IEEE, 2021, pp. 268–84, doi:10.1109/sp40001.2021.00035.","short":"K. Klein, G. Pascual Perez, M. Walter, C. Kamath Hosdurg, M. Capretto, M. Cueto Noval, I. Markov, M.X. Yeo, J.F. Alwen, K.Z. Pietrzak, in:, 2021 IEEE Symposium on Security and Privacy , IEEE, 2021, pp. 268–284.","chicago":"Klein, Karen, Guillermo Pascual Perez, Michael Walter, Chethan Kamath Hosdurg, Margarita Capretto, Miguel Cueto Noval, Ilia Markov, Michelle X Yeo, Joel F Alwen, and Krzysztof Z Pietrzak. “Keep the Dirt: Tainted TreeKEM, Adaptively and Actively Secure Continuous Group Key Agreement.” In 2021 IEEE Symposium on Security and Privacy , 268–84. IEEE, 2021. https://doi.org/10.1109/sp40001.2021.00035.","ama":"Klein K, Pascual Perez G, Walter M, et al. Keep the dirt: tainted TreeKEM, adaptively and actively secure continuous group key agreement. In: 2021 IEEE Symposium on Security and Privacy . IEEE; 2021:268-284. doi:10.1109/sp40001.2021.00035","ista":"Klein K, Pascual Perez G, Walter M, Kamath Hosdurg C, Capretto M, Cueto Noval M, Markov I, Yeo MX, Alwen JF, Pietrzak KZ. 2021. Keep the dirt: tainted TreeKEM, adaptively and actively secure continuous group key agreement. 2021 IEEE Symposium on Security and Privacy . SP: Symposium on Security and Privacy, 268–284.","apa":"Klein, K., Pascual Perez, G., Walter, M., Kamath Hosdurg, C., Capretto, M., Cueto Noval, M., … Pietrzak, K. Z. (2021). Keep the dirt: tainted TreeKEM, adaptively and actively secure continuous group key agreement. In 2021 IEEE Symposium on Security and Privacy (pp. 268–284). San Francisco, CA, United States: IEEE. https://doi.org/10.1109/sp40001.2021.00035","ieee":"K. Klein et al., “Keep the dirt: tainted TreeKEM, adaptively and actively secure continuous group key agreement,” in 2021 IEEE Symposium on Security and Privacy , San Francisco, CA, United States, 2021, pp. 268–284."},"publication":"2021 IEEE Symposium on Security and Privacy ","page":"268-284","date_published":"2021-08-26T00:00:00Z","type":"conference","abstract":[{"text":"While messaging systems with strong security guarantees are widely used in practice, designing a protocol that scales efficiently to large groups and enjoys similar security guarantees remains largely open. The two existing proposals to date are ART (Cohn-Gordon et al., CCS18) and TreeKEM (IETF, The Messaging Layer Security Protocol, draft). TreeKEM is the currently considered candidate by the IETF MLS working group, but dynamic group operations (i.e. adding and removing users) can cause efficiency issues. In this paper we formalize and analyze a variant of TreeKEM which we term Tainted TreeKEM (TTKEM for short). The basic idea underlying TTKEM was suggested by Millican (MLS mailing list, February 2018). This version is more efficient than TreeKEM for some natural distributions of group operations, we quantify this through simulations.Our second contribution is two security proofs for TTKEM which establish post compromise and forward secrecy even against adaptive attackers. The security loss (to the underlying PKE) in the Random Oracle Model is a polynomial factor, and a quasipolynomial one in the Standard Model. Our proofs can be adapted to TreeKEM as well. Before our work no security proof for any TreeKEM-like protocol establishing tight security against an adversary who can adaptively choose the sequence of operations was known. We also are the first to prove (or even formalize) active security where the server can arbitrarily deviate from the protocol specification. Proving fully active security – where also the users can arbitrarily deviate – remains open.","lang":"eng"}],"_id":"10049","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","status":"public","title":"Keep the dirt: tainted TreeKEM, adaptively and actively secure continuous group key agreement","oa_version":"Preprint","month":"08","main_file_link":[{"open_access":"1","url":"https://eprint.iacr.org/2019/1489"}],"oa":1,"project":[{"name":"International IST Doctoral Program","call_identifier":"H2020","grant_number":"665385","_id":"2564DBCA-B435-11E9-9278-68D0E5697425"},{"name":"Teaching Old Crypto New Tricks","call_identifier":"H2020","_id":"258AA5B2-B435-11E9-9278-68D0E5697425","grant_number":"682815"}],"quality_controlled":"1","doi":"10.1109/sp40001.2021.00035","conference":{"name":"SP: Symposium on Security and Privacy","end_date":"2021-05-27","start_date":"2021-05-24","location":"San Francisco, CA, United States"},"language":[{"iso":"eng"}],"ec_funded":1,"acknowledgement":"The first three authors contributed equally to this work. Funded by the European Research Council (ERC) under the European Union’s Horizon2020 research and innovation programme (682815-TOCNeT). Funded by the European Union’s Horizon 2020 research and innovation programme under the Marie Skłodowska-Curie Grant Agreement No.665385.","year":"2021","publisher":"IEEE","department":[{"_id":"KrPi"},{"_id":"DaAl"}],"publication_status":"published","related_material":{"record":[{"status":"public","relation":"dissertation_contains","id":"10035"}]},"author":[{"first_name":"Karen","last_name":"Klein","id":"3E83A2F8-F248-11E8-B48F-1D18A9856A87","full_name":"Klein, Karen"},{"id":"2D7ABD02-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0001-8630-415X","first_name":"Guillermo","last_name":"Pascual Perez","full_name":"Pascual Perez, Guillermo"},{"orcid":"0000-0003-3186-2482","id":"488F98B0-F248-11E8-B48F-1D18A9856A87","last_name":"Walter","first_name":"Michael","full_name":"Walter, Michael"},{"full_name":"Kamath Hosdurg, Chethan","id":"4BD3F30E-F248-11E8-B48F-1D18A9856A87","last_name":"Kamath Hosdurg","first_name":"Chethan"},{"full_name":"Capretto, Margarita","last_name":"Capretto","first_name":"Margarita"},{"last_name":"Cueto Noval","first_name":"Miguel","id":"ffc563a3-f6e0-11ea-865d-e3cce03d17cc","full_name":"Cueto Noval, Miguel"},{"first_name":"Ilia","last_name":"Markov","id":"D0CF4148-C985-11E9-8066-0BDEE5697425","full_name":"Markov, Ilia"},{"id":"2D82B818-F248-11E8-B48F-1D18A9856A87","first_name":"Michelle X","last_name":"Yeo","full_name":"Yeo, Michelle X"},{"full_name":"Alwen, Joel F","last_name":"Alwen","first_name":"Joel F","id":"2A8DFA8C-F248-11E8-B48F-1D18A9856A87"},{"last_name":"Pietrzak","first_name":"Krzysztof Z","orcid":"0000-0002-9139-1654","id":"3E04A7AA-F248-11E8-B48F-1D18A9856A87","full_name":"Pietrzak, Krzysztof Z"}],"date_created":"2021-09-27T13:46:27Z","date_updated":"2023-09-07T13:32:11Z"},{"date_created":"2022-03-18T08:48:41Z","date_updated":"2023-09-26T10:40:55Z","related_material":{"record":[{"status":"public","relation":"extended_version","id":"10855"}]},"author":[{"first_name":"Klaus-Tycho","last_name":"Foerster","full_name":"Foerster, Klaus-Tycho"},{"id":"C5402D42-15BC-11E9-A202-CA2BE6697425","first_name":"Janne","last_name":"Korhonen","full_name":"Korhonen, Janne"},{"full_name":"Paz, Ami","first_name":"Ami","last_name":"Paz"},{"id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0002-6432-6646","first_name":"Joel","last_name":"Rybicki","full_name":"Rybicki, Joel"},{"last_name":"Schmid","first_name":"Stefan","full_name":"Schmid, Stefan"}],"department":[{"_id":"DaAl"}],"publisher":"Association for Computing Machinery","publication_status":"published","year":"2021","acknowledgement":"We thank Jukka Suomela for discussions. We also thank our shepherd Mohammad Hajiesmaili and the reviewers for their time and suggestions on how to improve the paper. This project has received funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML), from the European Union’s Horizon 2020 research and innovation programme under the Marie Skłodowska–Curie grant agreement No. 840605, from the Vienna Science and Technology Fund (WWTF) project WHATIF, ICT19-045, 2020-2024, and from the Austrian Science Fund (FWF) and netIDEE SCIENCE project P 33775-N.","ec_funded":1,"language":[{"iso":"eng"}],"doi":"10.1145/3410220.3453923","conference":{"name":"SIGMETRICS: International Conference on Measurement and Modeling of Computer Systems","location":"Virtual, Online","start_date":"2021-06-14","end_date":"2021-06-18"},"project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"},{"call_identifier":"H2020","name":"Coordination in constrained and natural distributed systems","_id":"26A5D39A-B435-11E9-9278-68D0E5697425","grant_number":"840605"}],"quality_controlled":"1","external_id":{"arxiv":["2005.07637"]},"oa":1,"main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/2005.07637"}],"publication_identifier":{"isbn":["9781450380720"]},"month":"05","oa_version":"Preprint","status":"public","title":"Input-dynamic distributed algorithms for communication networks","_id":"10854","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","abstract":[{"text":"Consider a distributed task where the communication network is fixed but the local inputs given to the nodes of the distributed system may change over time. In this work, we explore the following question: if some of the local inputs change, can an existing solution be updated efficiently, in a dynamic and distributed manner?\r\nTo address this question, we define the batch dynamic CONGEST model in which we are given a bandwidth-limited communication network and a dynamic edge labelling defines the problem input. The task is to maintain a solution to a graph problem on the labelled graph under batch changes. We investigate, when a batch of alpha edge label changes arrive, - how much time as a function of alpha we need to update an existing solution, and - how much information the nodes have to keep in local memory between batches in order to update the solution quickly.\r\nOur work lays the foundations for the theory of input-dynamic distributed network algorithms. We give a general picture of the complexity landscape in this model, design both universal algorithms and algorithms for concrete problems, and present a general framework for lower bounds. The diverse time complexity of our model spans from constant time, through time polynomial in alpha, and to alpha time, which we show to be enough for any task.","lang":"eng"}],"type":"conference","date_published":"2021-05-01T00:00:00Z","page":"71-72","citation":{"ama":"Foerster K-T, Korhonen J, Paz A, Rybicki J, Schmid S. Input-dynamic distributed algorithms for communication networks. In: Abstract Proceedings of the 2021 ACM SIGMETRICS / International Conference on Measurement and Modeling of Computer Systems. Association for Computing Machinery; 2021:71-72. doi:10.1145/3410220.3453923","ieee":"K.-T. Foerster, J. Korhonen, A. Paz, J. Rybicki, and S. Schmid, “Input-dynamic distributed algorithms for communication networks,” in Abstract Proceedings of the 2021 ACM SIGMETRICS / International Conference on Measurement and Modeling of Computer Systems, Virtual, Online, 2021, pp. 71–72.","apa":"Foerster, K.-T., Korhonen, J., Paz, A., Rybicki, J., & Schmid, S. (2021). Input-dynamic distributed algorithms for communication networks. In Abstract Proceedings of the 2021 ACM SIGMETRICS / International Conference on Measurement and Modeling of Computer Systems (pp. 71–72). Virtual, Online: Association for Computing Machinery. https://doi.org/10.1145/3410220.3453923","ista":"Foerster K-T, Korhonen J, Paz A, Rybicki J, Schmid S. 2021. Input-dynamic distributed algorithms for communication networks. Abstract Proceedings of the 2021 ACM SIGMETRICS / International Conference on Measurement and Modeling of Computer Systems. SIGMETRICS: International Conference on Measurement and Modeling of Computer Systems, 71–72.","short":"K.-T. Foerster, J. Korhonen, A. Paz, J. Rybicki, S. Schmid, in:, Abstract Proceedings of the 2021 ACM SIGMETRICS / International Conference on Measurement and Modeling of Computer Systems, Association for Computing Machinery, 2021, pp. 71–72.","mla":"Foerster, Klaus-Tycho, et al. “Input-Dynamic Distributed Algorithms for Communication Networks.” Abstract Proceedings of the 2021 ACM SIGMETRICS / International Conference on Measurement and Modeling of Computer Systems, Association for Computing Machinery, 2021, pp. 71–72, doi:10.1145/3410220.3453923.","chicago":"Foerster, Klaus-Tycho, Janne Korhonen, Ami Paz, Joel Rybicki, and Stefan Schmid. “Input-Dynamic Distributed Algorithms for Communication Networks.” In Abstract Proceedings of the 2021 ACM SIGMETRICS / International Conference on Measurement and Modeling of Computer Systems, 71–72. Association for Computing Machinery, 2021. https://doi.org/10.1145/3410220.3453923."},"publication":"Abstract Proceedings of the 2021 ACM SIGMETRICS / International Conference on Measurement and Modeling of Computer Systems","article_processing_charge":"No","day":"01","scopus_import":"1"},{"date_published":"2021-03-01T00:00:00Z","article_type":"original","page":"1-33","publication":"Proceedings of the ACM on Measurement and Analysis of Computing Systems","citation":{"ama":"Foerster K-T, Korhonen J, Paz A, Rybicki J, Schmid S. Input-dynamic distributed algorithms for communication networks. Proceedings of the ACM on Measurement and Analysis of Computing Systems. 2021;5(1):1-33. doi:10.1145/3447384","ista":"Foerster K-T, Korhonen J, Paz A, Rybicki J, Schmid S. 2021. Input-dynamic distributed algorithms for communication networks. Proceedings of the ACM on Measurement and Analysis of Computing Systems. 5(1), 1–33.","ieee":"K.-T. Foerster, J. Korhonen, A. Paz, J. Rybicki, and S. Schmid, “Input-dynamic distributed algorithms for communication networks,” Proceedings of the ACM on Measurement and Analysis of Computing Systems, vol. 5, no. 1. Association for Computing Machinery, pp. 1–33, 2021.","apa":"Foerster, K.-T., Korhonen, J., Paz, A., Rybicki, J., & Schmid, S. (2021). Input-dynamic distributed algorithms for communication networks. Proceedings of the ACM on Measurement and Analysis of Computing Systems. Association for Computing Machinery. https://doi.org/10.1145/3447384","mla":"Foerster, Klaus-Tycho, et al. “Input-Dynamic Distributed Algorithms for Communication Networks.” Proceedings of the ACM on Measurement and Analysis of Computing Systems, vol. 5, no. 1, Association for Computing Machinery, 2021, pp. 1–33, doi:10.1145/3447384.","short":"K.-T. Foerster, J. Korhonen, A. Paz, J. Rybicki, S. Schmid, Proceedings of the ACM on Measurement and Analysis of Computing Systems 5 (2021) 1–33.","chicago":"Foerster, Klaus-Tycho, Janne Korhonen, Ami Paz, Joel Rybicki, and Stefan Schmid. “Input-Dynamic Distributed Algorithms for Communication Networks.” Proceedings of the ACM on Measurement and Analysis of Computing Systems. Association for Computing Machinery, 2021. https://doi.org/10.1145/3447384."},"day":"01","article_processing_charge":"No","keyword":["Computer Networks and Communications","Hardware and Architecture","Safety","Risk","Reliability and Quality","Computer Science (miscellaneous)"],"scopus_import":"1","oa_version":"Preprint","title":"Input-dynamic distributed algorithms for communication networks","status":"public","intvolume":" 5","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"10855","abstract":[{"text":"Consider a distributed task where the communication network is fixed but the local inputs given to the nodes of the distributed system may change over time. In this work, we explore the following question: if some of the local inputs change, can an existing solution be updated efficiently, in a dynamic and distributed manner? To address this question, we define the batch dynamic \\congest model in which we are given a bandwidth-limited communication network and a dynamic edge labelling defines the problem input. The task is to maintain a solution to a graph problem on the labeled graph under batch changes. We investigate, when a batch of α edge label changes arrive, \\beginitemize \\item how much time as a function of α we need to update an existing solution, and \\item how much information the nodes have to keep in local memory between batches in order to update the solution quickly. \\enditemize Our work lays the foundations for the theory of input-dynamic distributed network algorithms. We give a general picture of the complexity landscape in this model, design both universal algorithms and algorithms for concrete problems, and present a general framework for lower bounds. In particular, we derive non-trivial upper bounds for two selected, contrasting problems: maintaining a minimum spanning tree and detecting cliques.","lang":"eng"}],"issue":"1","type":"journal_article","language":[{"iso":"eng"}],"doi":"10.1145/3447384","quality_controlled":"1","project":[{"call_identifier":"H2020","name":"Coordination in constrained and natural distributed systems","grant_number":"840605","_id":"26A5D39A-B435-11E9-9278-68D0E5697425"},{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/2005.07637"}],"external_id":{"arxiv":["2005.07637"]},"oa":1,"month":"03","publication_identifier":{"issn":["2476-1249"]},"date_updated":"2023-09-26T10:40:55Z","date_created":"2022-03-18T09:10:27Z","volume":5,"author":[{"first_name":"Klaus-Tycho","last_name":"Foerster","full_name":"Foerster, Klaus-Tycho"},{"full_name":"Korhonen, Janne","last_name":"Korhonen","first_name":"Janne","id":"C5402D42-15BC-11E9-A202-CA2BE6697425"},{"first_name":"Ami","last_name":"Paz","full_name":"Paz, Ami"},{"last_name":"Rybicki","first_name":"Joel","orcid":"0000-0002-6432-6646","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","full_name":"Rybicki, Joel"},{"last_name":"Schmid","first_name":"Stefan","full_name":"Schmid, Stefan"}],"related_material":{"record":[{"id":"10854","relation":"shorter_version","status":"public"}]},"publication_status":"published","publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"year":"2021","acknowledgement":"We thank Jukka Suomela for discussions. We also thank our shepherd Mohammad Hajiesmaili\r\nand the reviewers for their time and suggestions on how to improve the paper. This project\r\nhas received funding from the European Research Council (ERC) under the European Union’s\r\nHorizon 2020 research and innovation programme (grant agreement No 805223 ScaleML), from the European Union’s Horizon 2020 research and innovation programme under the Marie\r\nSk lodowska–Curie grant agreement No. 840605, from the Vienna Science and Technology Fund (WWTF) project WHATIF, ICT19-045, 2020-2024, and from the Austrian Science Fund (FWF) and netIDEE SCIENCE project P 33775-N.","ec_funded":1},{"abstract":[{"text":"The scalability of concurrent data structures and distributed algorithms strongly depends on\r\nreducing the contention for shared resources and the costs of synchronization and communication. We show how such cost reductions can be attained by relaxing the strict consistency conditions required by sequential implementations. In the first part of the thesis, we consider relaxation in the context of concurrent data structures. Specifically, in data structures \r\nsuch as priority queues, imposing strong semantics renders scalability impossible, since a correct implementation of the remove operation should return only the element with highest priority. Intuitively, attempting to invoke remove operations concurrently creates a race condition. This bottleneck can be circumvented by relaxing semantics of the affected data structure, thus allowing removal of the elements which are no longer required to have the highest priority. We prove that the randomized implementations of relaxed data structures provide provable guarantees on the priority of the removed elements even under concurrency. Additionally, we show that in some cases the relaxed data structures can be used to scale the classical algorithms which are usually implemented with the exact ones. In the second part, we study parallel variants of the stochastic gradient descent (SGD) algorithm, which distribute computation among the multiple processors, thus reducing the running time. Unfortunately, in order for standard parallel SGD to succeed, each processor has to maintain a local copy of the necessary model parameter, which is identical to the local copies of other processors; the overheads from this perfect consistency in terms of communication and synchronization can negate the speedup gained by distributing the computation. We show that the consistency conditions required by SGD can be relaxed, allowing the algorithm to be more flexible in terms of tolerating quantized communication, asynchrony, or even crash faults, while its convergence remains asymptotically the same.","lang":"eng"}],"alternative_title":["ISTA Thesis"],"type":"dissertation","file":[{"checksum":"6bf14e9a523387328f016c0689f5e10e","success":1,"date_updated":"2021-12-09T17:47:49Z","date_created":"2021-12-09T17:47:49Z","relation":"main_file","file_id":"10436","file_size":2370859,"content_type":"application/pdf","creator":"gnadirad","access_level":"open_access","file_name":"Thesis_Final_09_12_2021.pdf"},{"file_name":"Thesis_Final_09_12_2021.zip","access_level":"closed","file_size":2596924,"content_type":"application/zip","creator":"gnadirad","relation":"source_file","file_id":"10437","date_updated":"2022-03-28T12:55:12Z","date_created":"2021-12-09T17:47:49Z","checksum":"914d6c5ca86bd0add471971a8f4c4341"}],"oa_version":"Published Version","title":"On achieving scalability through relaxation","ddc":["000"],"status":"public","_id":"10429","user_id":"c635000d-4b10-11ee-a964-aac5a93f6ac1","article_processing_charge":"No","has_accepted_license":"1","day":"09","date_published":"2021-12-09T00:00:00Z","page":"132","citation":{"chicago":"Nadiradze, Giorgi. “On Achieving Scalability through Relaxation.” Institute of Science and Technology Austria, 2021. https://doi.org/10.15479/at:ista:10429.","short":"G. Nadiradze, On Achieving Scalability through Relaxation, Institute of Science and Technology Austria, 2021.","mla":"Nadiradze, Giorgi. On Achieving Scalability through Relaxation. Institute of Science and Technology Austria, 2021, doi:10.15479/at:ista:10429.","apa":"Nadiradze, G. (2021). On achieving scalability through relaxation. Institute of Science and Technology Austria. https://doi.org/10.15479/at:ista:10429","ieee":"G. Nadiradze, “On achieving scalability through relaxation,” Institute of Science and Technology Austria, 2021.","ista":"Nadiradze G. 2021. On achieving scalability through relaxation. Institute of Science and Technology Austria.","ama":"Nadiradze G. On achieving scalability through relaxation. 2021. doi:10.15479/at:ista:10429"},"ec_funded":1,"file_date_updated":"2022-03-28T12:55:12Z","date_created":"2021-12-08T21:52:28Z","date_updated":"2023-10-17T11:48:55Z","related_material":{"record":[{"id":"10432","relation":"part_of_dissertation","status":"public"},{"id":"6673","status":"public","relation":"part_of_dissertation"},{"id":"5965","status":"public","relation":"part_of_dissertation"},{"id":"10435","status":"public","relation":"part_of_dissertation"}]},"author":[{"last_name":"Nadiradze","first_name":"Giorgi","orcid":"0000-0001-5634-0731","id":"3279A00C-F248-11E8-B48F-1D18A9856A87","full_name":"Nadiradze, Giorgi"}],"publisher":"Institute of Science and Technology Austria","department":[{"_id":"GradSch"},{"_id":"DaAl"}],"publication_status":"published","year":"2021","publication_identifier":{"issn":["2663-337X"]},"month":"12","language":[{"iso":"eng"}],"degree_awarded":"PhD","supervisor":[{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian"}],"doi":"10.15479/at:ista:10429","project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"oa":1},{"month":"12","day":"01","article_processing_charge":"No","language":[{"iso":"eng"}],"conference":{"location":"Sydney, Australia","start_date":"2021-12-06","end_date":"2021-12-14","name":"NeurIPS: Neural Information Processing Systems"},"date_published":"2021-12-01T00:00:00Z","quality_controlled":"1","project":[{"call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships","grant_number":"754411","_id":"260C2330-B435-11E9-9278-68D0E5697425"},{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"publication":"35th Conference on Neural Information Processing Systems","citation":{"ama":"Nadiradze G, Sabour A, Davies P, Li S, Alistarh D-A. Asynchronous decentralized SGD with quantized and local updates. In: 35th Conference on Neural Information Processing Systems. Neural Information Processing Systems Foundation; 2021.","apa":"Nadiradze, G., Sabour, A., Davies, P., Li, S., & Alistarh, D.-A. (2021). Asynchronous decentralized SGD with quantized and local updates. In 35th Conference on Neural Information Processing Systems. Sydney, Australia: Neural Information Processing Systems Foundation.","ieee":"G. Nadiradze, A. Sabour, P. Davies, S. Li, and D.-A. Alistarh, “Asynchronous decentralized SGD with quantized and local updates,” in 35th Conference on Neural Information Processing Systems, Sydney, Australia, 2021.","ista":"Nadiradze G, Sabour A, Davies P, Li S, Alistarh D-A. 2021. Asynchronous decentralized SGD with quantized and local updates. 35th Conference on Neural Information Processing Systems. NeurIPS: Neural Information Processing Systems.","short":"G. Nadiradze, A. Sabour, P. Davies, S. Li, D.-A. Alistarh, in:, 35th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2021.","mla":"Nadiradze, Giorgi, et al. “Asynchronous Decentralized SGD with Quantized and Local Updates.” 35th Conference on Neural Information Processing Systems, Neural Information Processing Systems Foundation, 2021.","chicago":"Nadiradze, Giorgi, Amirmojtaba Sabour, Peter Davies, Shigang Li, and Dan-Adrian Alistarh. “Asynchronous Decentralized SGD with Quantized and Local Updates.” In 35th Conference on Neural Information Processing Systems. Neural Information Processing Systems Foundation, 2021."},"oa":1,"main_file_link":[{"open_access":"1","url":"https://papers.nips.cc/paper/2021/hash/362c99307cdc3f2d8b410652386a9dd1-Abstract.html"}],"external_id":{"arxiv":["1910.12308"]},"abstract":[{"text":"Decentralized optimization is emerging as a viable alternative for scalable distributed machine learning, but also introduces new challenges in terms of synchronization costs. To this end, several communication-reduction techniques, such as non-blocking communication, quantization, and local steps, have been explored in the decentralized setting. Due to the complexity of analyzing optimization in such a relaxed setting, this line of work often assumes \\emph{global} communication rounds, which require additional synchronization. In this paper, we consider decentralized optimization in the simpler, but harder to analyze, \\emph{asynchronous gossip} model, in which communication occurs in discrete, randomly chosen pairings among nodes. Perhaps surprisingly, we show that a variant of SGD called \\emph{SwarmSGD} still converges in this setting, even if \\emph{non-blocking communication}, \\emph{quantization}, and \\emph{local steps} are all applied \\emph{in conjunction}, and even if the node data distributions and underlying graph topology are both \\emph{heterogenous}. Our analysis is based on a new connection with multi-dimensional load-balancing processes. We implement this algorithm and deploy it in a super-computing environment, showing that it can outperform previous decentralized methods in terms of end-to-end training time, and that it can even rival carefully-tuned large-batch SGD for certain tasks.","lang":"eng"}],"ec_funded":1,"type":"conference","date_updated":"2023-10-17T11:48:56Z","date_created":"2021-12-09T10:59:12Z","oa_version":"Published Version","author":[{"full_name":"Nadiradze, Giorgi","last_name":"Nadiradze","first_name":"Giorgi","orcid":"0000-0001-5634-0731","id":"3279A00C-F248-11E8-B48F-1D18A9856A87"},{"first_name":"Amirmojtaba","last_name":"Sabour","id":"bcc145fd-e77f-11ea-ae8b-80d661dbff67","full_name":"Sabour, Amirmojtaba"},{"first_name":"Peter","last_name":"Davies","id":"11396234-BB50-11E9-B24C-90FCE5697425","orcid":"0000-0002-5646-9524","full_name":"Davies, Peter"},{"last_name":"Li","first_name":"Shigang","full_name":"Li, Shigang"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"}],"related_material":{"record":[{"status":"public","relation":"dissertation_contains","id":"10429"}]},"status":"public","title":"Asynchronous decentralized SGD with quantized and local updates","publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"Neural Information Processing Systems Foundation","_id":"10435","year":"2021","acknowledgement":"We gratefully acknowledge funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). PD partly conducted this work while at IST Austria and was supported by the European Union’s Horizon 2020 programme under the Marie Skłodowska-Curie grant agreement No. 754411. SL was funded in part by European Research Council (ERC) under the European Union’s Horizon 2020 programme (grant agreement DAPP, No. 678880, and EPiGRAM-HS, No. 801039).\r\n","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87"},{"date_published":"2021-06-01T00:00:00Z","publication":"ACM Transactions on Algorithms","citation":{"mla":"Czumaj, Artur, et al. “Graph Sparsification for Derandomizing Massively Parallel Computation with Low Space.” ACM Transactions on Algorithms, vol. 17, no. 2, 16, Association for Computing Machinery, 2021, doi:10.1145/3451992.","short":"A. Czumaj, P. Davies, M. Parter, ACM Transactions on Algorithms 17 (2021).","chicago":"Czumaj, Artur, Peter Davies, and Merav Parter. “Graph Sparsification for Derandomizing Massively Parallel Computation with Low Space.” ACM Transactions on Algorithms. Association for Computing Machinery, 2021. https://doi.org/10.1145/3451992.","ama":"Czumaj A, Davies P, Parter M. Graph sparsification for derandomizing massively parallel computation with low space. ACM Transactions on Algorithms. 2021;17(2). doi:10.1145/3451992","ista":"Czumaj A, Davies P, Parter M. 2021. Graph sparsification for derandomizing massively parallel computation with low space. ACM Transactions on Algorithms. 17(2), 16.","apa":"Czumaj, A., Davies, P., & Parter, M. (2021). Graph sparsification for derandomizing massively parallel computation with low space. ACM Transactions on Algorithms. Association for Computing Machinery. https://doi.org/10.1145/3451992","ieee":"A. Czumaj, P. Davies, and M. Parter, “Graph sparsification for derandomizing massively parallel computation with low space,” ACM Transactions on Algorithms, vol. 17, no. 2. Association for Computing Machinery, 2021."},"article_type":"original","day":"01","article_processing_charge":"No","has_accepted_license":"1","oa_version":"Submitted Version","file":[{"date_updated":"2021-06-10T19:33:56Z","date_created":"2021-06-10T19:33:56Z","success":1,"checksum":"a21c627683890c309a68f6389302c408","file_id":"9542","relation":"main_file","creator":"pdavies","content_type":"application/pdf","file_size":587404,"file_name":"MISMM-arxiv.pdf","access_level":"open_access"}],"_id":"9541","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","status":"public","title":"Graph sparsification for derandomizing massively parallel computation with low space","ddc":["000"],"intvolume":" 17","abstract":[{"lang":"eng","text":"The Massively Parallel Computation (MPC) model is an emerging model that distills core aspects of distributed and parallel computation, developed as a tool to solve combinatorial (typically graph) problems in systems of many machines with limited space. Recent work has focused on the regime in which machines have sublinear (in n, the number of nodes in the input graph) space, with randomized algorithms presented for the fundamental problems of Maximal Matching and Maximal Independent Set. However, there have been no prior corresponding deterministic algorithms. A major challenge underlying the sublinear space setting is that the local space of each machine might be too small to store all edges incident to a single node. This poses a considerable obstacle compared to classical models in which each node is assumed to know and have easy access to its incident edges. To overcome this barrier, we introduce a new graph sparsification technique that deterministically computes a low-degree subgraph, with the additional property that solving the problem on this subgraph provides significant progress towards solving the problem for the original input graph. Using this framework to derandomize the well-known algorithm of Luby [SICOMP’86], we obtain O(log Δ + log log n)-round deterministic MPC algorithms for solving the problems of Maximal Matching and Maximal Independent Set with O(nɛ) space on each machine for any constant ɛ > 0. These algorithms also run in O(log Δ) rounds in the closely related model of CONGESTED CLIQUE, improving upon the state-of-the-art bound of O(log 2Δ) rounds by Censor-Hillel et al. [DISC’17]."}],"issue":"2","type":"journal_article","doi":"10.1145/3451992","language":[{"iso":"eng"}],"oa":1,"external_id":{"arxiv":["1912.05390"],"isi":["000661311300006"]},"main_file_link":[{"url":"https://arxiv.org/abs/1912.05390","open_access":"1"}],"isi":1,"quality_controlled":"1","project":[{"_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411","name":"ISTplus - Postdoctoral Fellowships","call_identifier":"H2020"}],"month":"06","publication_identifier":{"issn":["1549-6325"],"eissn":["1549-6333"]},"author":[{"first_name":"Artur","last_name":"Czumaj","full_name":"Czumaj, Artur"},{"id":"11396234-BB50-11E9-B24C-90FCE5697425","orcid":"0000-0002-5646-9524","first_name":"Peter","last_name":"Davies","full_name":"Davies, Peter"},{"full_name":"Parter, Merav","last_name":"Parter","first_name":"Merav"}],"related_material":{"record":[{"id":"7802","relation":"earlier_version","status":"public"}]},"date_created":"2021-06-10T19:31:05Z","date_updated":"2024-02-28T12:53:09Z","volume":17,"acknowledgement":"Institute of Science and Technology Austria (IST Austria). Email: peter.davies@ist.ac.at. Work partially\r\ndone at the Department of Computer Science and Centre for Discrete Mathematics and its Applications (DIMAP),University of Warwick. Research partially supported by the European Union’s Horizon 2020 research and innovation programme under the Marie Skłodowska-Curie grant agreement No 754411, the Centre for Discrete Mathematics and its Applications, a Weizmann-UK Making Connections Grant, and EPSRC award EP/N011163/1.","year":"2021","publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"Association for Computing Machinery","file_date_updated":"2021-06-10T19:33:56Z","ec_funded":1,"article_number":"16"},{"oa_version":"Preprint","status":"public","title":"Efficient load-balancing through distributed token dropping","user_id":"D865714E-FA4E-11E9-B85B-F5C5E5697425","_id":"9678","abstract":[{"lang":"eng","text":"We introduce a new graph problem, the token dropping game, and we show how to solve it efficiently in a distributed setting. We use the token dropping game as a tool to design an efficient distributed algorithm for stable orientations and more generally for locally optimal semi-matchings. The prior work by Czygrinow et al. (DISC 2012) finds a stable orientation in O(Δ^5) rounds in graphs of maximum degree Δ, while we improve it to O(Δ^4) and also prove a lower bound of Ω(Δ). For the more general problem of locally optimal semi-matchings, the prior upper bound is O(S^5) and our new algorithm runs in O(C · S^4) rounds, which is an improvement for C = o(S); here C and S are the maximum degrees of customers and servers, respectively."}],"type":"conference","date_published":"2021-07-06T00:00:00Z","page":"129-139","citation":{"chicago":"Brandt, Sebastian, Barbara Keller, Joel Rybicki, Jukka Suomela, and Jara Uitto. “Efficient Load-Balancing through Distributed Token Dropping.” In Annual ACM Symposium on Parallelism in Algorithms and Architectures, 129–39, 2021. https://doi.org/10.1145/3409964.3461785.","mla":"Brandt, Sebastian, et al. “Efficient Load-Balancing through Distributed Token Dropping.” Annual ACM Symposium on Parallelism in Algorithms and Architectures, 2021, pp. 129–39, doi:10.1145/3409964.3461785.","short":"S. Brandt, B. Keller, J. Rybicki, J. Suomela, J. Uitto, in:, Annual ACM Symposium on Parallelism in Algorithms and Architectures, 2021, pp. 129–139.","ista":"Brandt S, Keller B, Rybicki J, Suomela J, Uitto J. 2021. Efficient load-balancing through distributed token dropping. Annual ACM Symposium on Parallelism in Algorithms and Architectures. SPAA: Symposium on Parallelism in Algorithms and Architectures , 129–139.","ieee":"S. Brandt, B. Keller, J. Rybicki, J. Suomela, and J. Uitto, “Efficient load-balancing through distributed token dropping,” in Annual ACM Symposium on Parallelism in Algorithms and Architectures, Virtual Event, United States, 2021, pp. 129–139.","apa":"Brandt, S., Keller, B., Rybicki, J., Suomela, J., & Uitto, J. (2021). Efficient load-balancing through distributed token dropping. In Annual ACM Symposium on Parallelism in Algorithms and Architectures (pp. 129–139). Virtual Event, United States. https://doi.org/10.1145/3409964.3461785","ama":"Brandt S, Keller B, Rybicki J, Suomela J, Uitto J. Efficient load-balancing through distributed token dropping. In: Annual ACM Symposium on Parallelism in Algorithms and Architectures. ; 2021:129-139. doi:10.1145/3409964.3461785"},"publication":"Annual ACM Symposium on Parallelism in Algorithms and Architectures","article_processing_charge":"No","day":"06","scopus_import":"1","date_updated":"2024-03-05T07:13:12Z","date_created":"2021-07-18T22:01:22Z","related_material":{"record":[{"status":"public","relation":"earlier_version","id":"15074"}]},"author":[{"full_name":"Brandt, Sebastian","last_name":"Brandt","first_name":"Sebastian"},{"full_name":"Keller, Barbara","first_name":"Barbara","last_name":"Keller"},{"first_name":"Joel","last_name":"Rybicki","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0002-6432-6646","full_name":"Rybicki, Joel"},{"full_name":"Suomela, Jukka","first_name":"Jukka","last_name":"Suomela"},{"last_name":"Uitto","first_name":"Jara","full_name":"Uitto, Jara"}],"department":[{"_id":"DaAl"}],"publication_status":"published","acknowledgement":"We thank Orr Fischer, Juho Hirvonen, and Tuomo Lempiäinen for valuable discussions. This project has received funding from the European Union’s Horizon 2020 research and innovation programme under the Marie Skłodowska-Curie grant agreement No. 840605.","year":"2021","ec_funded":1,"language":[{"iso":"eng"}],"doi":"10.1145/3409964.3461785","conference":{"name":"SPAA: Symposium on Parallelism in Algorithms and Architectures ","end_date":"2021-07-08","start_date":"2021-07-06","location":" Virtual Event, United States"},"project":[{"name":"Coordination in constrained and natural distributed systems","call_identifier":"H2020","_id":"26A5D39A-B435-11E9-9278-68D0E5697425","grant_number":"840605"}],"quality_controlled":"1","external_id":{"arxiv":["2005.07761"]},"oa":1,"main_file_link":[{"url":"https://arxiv.org/abs/2005.07761","open_access":"1"}],"publication_identifier":{"isbn":["9781450380706"]},"month":"07"},{"article_type":"original","citation":{"short":"D.-A. Alistarh, G. Nadiradze, A. Sabour, Algorithmica (2021).","mla":"Alistarh, Dan-Adrian, et al. “Dynamic Averaging Load Balancing on Cycles.” Algorithmica, Springer Nature, 2021, doi:10.1007/s00453-021-00905-9.","chicago":"Alistarh, Dan-Adrian, Giorgi Nadiradze, and Amirmojtaba Sabour. “Dynamic Averaging Load Balancing on Cycles.” Algorithmica. Springer Nature, 2021. https://doi.org/10.1007/s00453-021-00905-9.","ama":"Alistarh D-A, Nadiradze G, Sabour A. Dynamic averaging load balancing on cycles. Algorithmica. 2021. doi:10.1007/s00453-021-00905-9","apa":"Alistarh, D.-A., Nadiradze, G., & Sabour, A. (2021). Dynamic averaging load balancing on cycles. Algorithmica. Virtual, Online; Germany: Springer Nature. https://doi.org/10.1007/s00453-021-00905-9","ieee":"D.-A. Alistarh, G. Nadiradze, and A. Sabour, “Dynamic averaging load balancing on cycles,” Algorithmica. Springer Nature, 2021.","ista":"Alistarh D-A, Nadiradze G, Sabour A. 2021. Dynamic averaging load balancing on cycles. Algorithmica."},"publication":"Algorithmica","date_published":"2021-12-24T00:00:00Z","scopus_import":"1","article_processing_charge":"Yes (via OA deal)","has_accepted_license":"1","day":"24","title":"Dynamic averaging load balancing on cycles","ddc":["000"],"status":"public","_id":"8286","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","file":[{"creator":"cchlebak","content_type":"application/pdf","file_size":525950,"access_level":"open_access","file_name":"2021_Algorithmica_Alistarh.pdf","success":1,"checksum":"21169b25b0c8e17b21e12af22bff9870","date_updated":"2021-12-27T10:36:40Z","date_created":"2021-12-27T10:36:40Z","file_id":"10577","relation":"main_file"}],"oa_version":"Published Version","type":"journal_article","abstract":[{"text":"We consider the following dynamic load-balancing process: given an underlying graph G with n nodes, in each step t≥ 0, one unit of load is created, and placed at a randomly chosen graph node. In the same step, the chosen node picks a random neighbor, and the two nodes balance their loads by averaging them. We are interested in the expected gap between the minimum and maximum loads at nodes as the process progresses, and its dependence on n and on the graph structure. Variants of the above graphical balanced allocation process have been studied previously by Peres, Talwar, and Wieder [Peres et al., 2015], and by Sauerwald and Sun [Sauerwald and Sun, 2015]. These authors left as open the question of characterizing the gap in the case of cycle graphs in the dynamic case, where weights are created during the algorithm’s execution. For this case, the only known upper bound is of 𝒪(n log n), following from a majorization argument due to [Peres et al., 2015], which analyzes a related graphical allocation process. In this paper, we provide an upper bound of 𝒪 (√n log n) on the expected gap of the above process for cycles of length n. We introduce a new potential analysis technique, which enables us to bound the difference in load between k-hop neighbors on the cycle, for any k ≤ n/2. We complement this with a \"gap covering\" argument, which bounds the maximum value of the gap by bounding its value across all possible subsets of a certain structure, and recursively bounding the gaps within each subset. We provide analytical and experimental evidence that our upper bound on the gap is tight up to a logarithmic factor. ","lang":"eng"}],"project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"},{"name":"IST Austria Open Access Fund","_id":"B67AFEDC-15C9-11EA-A837-991A96BB2854"}],"quality_controlled":"1","isi":1,"oa":1,"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"external_id":{"arxiv":["2003.09297"],"isi":["000734004600001"]},"language":[{"iso":"eng"}],"doi":"10.1007/s00453-021-00905-9","conference":{"name":"ICALP: International Colloquium on Automata, Languages, and Programming ","end_date":"2020-07-11","location":"Virtual, Online; Germany","start_date":"2020-07-08"},"publication_identifier":{"eissn":["1432-0541"],"issn":["0178-4617"]},"month":"12","department":[{"_id":"DaAl"}],"publisher":"Springer Nature","publication_status":"published","year":"2021","acknowledgement":"The authors sincerely thank Thomas Sauerwald and George Giakkoupis for insightful discussions, and Mohsen Ghaffari, Yuval Peres, and Udi Wieder for feedback on earlier versions of this draft. We also thank the ICALP anonymous reviewers for their very useful comments. Open access funding provided by Institute of Science and Technology (IST Austria). Funding was provided by European Research Council (Grant No. PR1042ERC01).","date_created":"2020-08-24T06:24:04Z","date_updated":"2024-03-05T07:35:53Z","related_material":{"link":[{"relation":"earlier_version","url":"https://doi.org/10.4230/LIPIcs.ICALP.2020.7"}],"record":[{"relation":"earlier_version","status":"public","id":"15077"}]},"author":[{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian"},{"id":"3279A00C-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0001-5634-0731","first_name":"Giorgi","last_name":"Nadiradze","full_name":"Nadiradze, Giorgi"},{"id":"bcc145fd-e77f-11ea-ae8b-80d661dbff67","first_name":"Amirmojtaba","last_name":"Sabour","full_name":"Sabour, Amirmojtaba"}],"ec_funded":1,"file_date_updated":"2021-12-27T10:36:40Z"},{"file_date_updated":"2021-06-23T07:09:41Z","author":[{"full_name":"Ramezani-Kebrya, Ali","first_name":"Ali","last_name":"Ramezani-Kebrya"},{"full_name":"Faghri, Fartash","last_name":"Faghri","first_name":"Fartash"},{"first_name":"Ilya","last_name":"Markov","full_name":"Markov, Ilya"},{"full_name":"Aksenov, Vitalii","last_name":"Aksenov","first_name":"Vitalii","id":"2980135A-F248-11E8-B48F-1D18A9856A87"},{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X"},{"full_name":"Roy, Daniel M.","first_name":"Daniel M.","last_name":"Roy"}],"volume":22,"date_updated":"2024-03-06T12:22:07Z","date_created":"2021-06-20T22:01:33Z","year":"2021","department":[{"_id":"DaAl"}],"publisher":"Journal of Machine Learning Research","publication_status":"published","publication_identifier":{"eissn":["15337928"],"issn":["15324435"]},"month":"04","language":[{"iso":"eng"}],"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"main_file_link":[{"open_access":"1","url":"https://www.jmlr.org/papers/v22/20-255.html"}],"external_id":{"arxiv":["1908.06077"]},"oa":1,"quality_controlled":"1","issue":"114","abstract":[{"lang":"eng","text":"As the size and complexity of models and datasets grow, so does the need for communication-efficient variants of stochastic gradient descent that can be deployed to perform parallel model training. One popular communication-compression method for data-parallel SGD is QSGD (Alistarh et al., 2017), which quantizes and encodes gradients to reduce communication costs. The baseline variant of QSGD provides strong theoretical guarantees, however, for practical purposes, the authors proposed a heuristic variant which we call QSGDinf, which demonstrated impressive empirical gains for distributed training of large neural networks. In this paper, we build on this work to propose a new gradient quantization scheme, and show that it has both stronger theoretical guarantees than QSGD, and matches and exceeds the empirical performance of the QSGDinf heuristic and of other compression methods."}],"type":"journal_article","oa_version":"Published Version","file":[{"access_level":"open_access","file_name":"2021_JournalOfMachineLearningResearch_Ramezani-Kebrya.pdf","file_size":11237154,"content_type":"application/pdf","creator":"asandaue","relation":"main_file","file_id":"9595","checksum":"6428aa8bcb67768b6949c99b55d5281d","success":1,"date_created":"2021-06-23T07:09:41Z","date_updated":"2021-06-23T07:09:41Z"}],"_id":"9571","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","intvolume":" 22","ddc":["000"],"title":"NUQSGD: Provably communication-efficient data-parallel SGD via nonuniform quantization","status":"public","article_processing_charge":"No","has_accepted_license":"1","day":"01","scopus_import":"1","date_published":"2021-04-01T00:00:00Z","citation":{"mla":"Ramezani-Kebrya, Ali, et al. “NUQSGD: Provably Communication-Efficient Data-Parallel SGD via Nonuniform Quantization.” Journal of Machine Learning Research, vol. 22, no. 114, Journal of Machine Learning Research, 2021, p. 1−43.","short":"A. Ramezani-Kebrya, F. Faghri, I. Markov, V. Aksenov, D.-A. Alistarh, D.M. Roy, Journal of Machine Learning Research 22 (2021) 1−43.","chicago":"Ramezani-Kebrya, Ali, Fartash Faghri, Ilya Markov, Vitalii Aksenov, Dan-Adrian Alistarh, and Daniel M. Roy. “NUQSGD: Provably Communication-Efficient Data-Parallel SGD via Nonuniform Quantization.” Journal of Machine Learning Research. Journal of Machine Learning Research, 2021.","ama":"Ramezani-Kebrya A, Faghri F, Markov I, Aksenov V, Alistarh D-A, Roy DM. NUQSGD: Provably communication-efficient data-parallel SGD via nonuniform quantization. Journal of Machine Learning Research. 2021;22(114):1−43.","ista":"Ramezani-Kebrya A, Faghri F, Markov I, Aksenov V, Alistarh D-A, Roy DM. 2021. NUQSGD: Provably communication-efficient data-parallel SGD via nonuniform quantization. Journal of Machine Learning Research. 22(114), 1−43.","ieee":"A. Ramezani-Kebrya, F. Faghri, I. Markov, V. Aksenov, D.-A. Alistarh, and D. M. Roy, “NUQSGD: Provably communication-efficient data-parallel SGD via nonuniform quantization,” Journal of Machine Learning Research, vol. 22, no. 114. Journal of Machine Learning Research, p. 1−43, 2021.","apa":"Ramezani-Kebrya, A., Faghri, F., Markov, I., Aksenov, V., Alistarh, D.-A., & Roy, D. M. (2021). NUQSGD: Provably communication-efficient data-parallel SGD via nonuniform quantization. Journal of Machine Learning Research. Journal of Machine Learning Research."},"publication":"Journal of Machine Learning Research","page":"1−43","article_type":"original"},{"oa_version":"Published Version","status":"public","title":"Fast approximate shortest paths in the congested clique","intvolume":" 34","user_id":"3E5EF7F0-F248-11E8-B48F-1D18A9856A87","_id":"7939","abstract":[{"lang":"eng","text":"We design fast deterministic algorithms for distance computation in the Congested Clique model. Our key contributions include:\r\n A (2+ϵ)-approximation for all-pairs shortest paths in O(log2n/ϵ) rounds on unweighted undirected graphs. With a small additional additive factor, this also applies for weighted graphs. This is the first sub-polynomial constant-factor approximation for APSP in this model.\r\n A (1+ϵ)-approximation for multi-source shortest paths from O(n−−√) sources in O(log2n/ϵ) rounds on weighted undirected graphs. This is the first sub-polynomial algorithm obtaining this approximation for a set of sources of polynomial size.\r\n\r\nOur main techniques are new distance tools that are obtained via improved algorithms for sparse matrix multiplication, which we leverage to construct efficient hopsets and shortest paths. Furthermore, our techniques extend to additional distance problems for which we improve upon the state-of-the-art, including diameter approximation, and an exact single-source shortest paths algorithm for weighted undirected graphs in O~(n1/6) rounds. "}],"type":"journal_article","date_published":"2021-12-01T00:00:00Z","article_type":"original","page":"463-487","publication":"Distributed Computing","citation":{"short":"K. Censor-Hillel, M. Dory, J. Korhonen, D. Leitersdorf, Distributed Computing 34 (2021) 463–487.","mla":"Censor-Hillel, Keren, et al. “Fast Approximate Shortest Paths in the Congested Clique.” Distributed Computing, vol. 34, Springer Nature, 2021, pp. 463–87, doi:10.1007/s00446-020-00380-5.","chicago":"Censor-Hillel, Keren, Michal Dory, Janne Korhonen, and Dean Leitersdorf. “Fast Approximate Shortest Paths in the Congested Clique.” Distributed Computing. Springer Nature, 2021. https://doi.org/10.1007/s00446-020-00380-5.","ama":"Censor-Hillel K, Dory M, Korhonen J, Leitersdorf D. Fast approximate shortest paths in the congested clique. Distributed Computing. 2021;34:463-487. doi:10.1007/s00446-020-00380-5","apa":"Censor-Hillel, K., Dory, M., Korhonen, J., & Leitersdorf, D. (2021). Fast approximate shortest paths in the congested clique. Distributed Computing. Springer Nature. https://doi.org/10.1007/s00446-020-00380-5","ieee":"K. Censor-Hillel, M. Dory, J. Korhonen, and D. Leitersdorf, “Fast approximate shortest paths in the congested clique,” Distributed Computing, vol. 34. Springer Nature, pp. 463–487, 2021.","ista":"Censor-Hillel K, Dory M, Korhonen J, Leitersdorf D. 2021. Fast approximate shortest paths in the congested clique. Distributed Computing. 34, 463–487."},"day":"01","article_processing_charge":"Yes (via OA deal)","scopus_import":"1","date_updated":"2024-03-07T14:43:39Z","date_created":"2020-06-07T22:00:54Z","volume":34,"author":[{"last_name":"Censor-Hillel","first_name":"Keren","full_name":"Censor-Hillel, Keren"},{"first_name":"Michal","last_name":"Dory","full_name":"Dory, Michal"},{"id":"C5402D42-15BC-11E9-A202-CA2BE6697425","last_name":"Korhonen","first_name":"Janne","full_name":"Korhonen, Janne"},{"full_name":"Leitersdorf, Dean","first_name":"Dean","last_name":"Leitersdorf"}],"related_material":{"record":[{"status":"public","relation":"earlier_version","id":"6933"}]},"publication_status":"published","publisher":"Springer Nature","department":[{"_id":"DaAl"}],"acknowledgement":"Open access funding provided by Institute of Science and Technology (IST Austria). We thank Mohsen Ghaffari, Michael Elkin and Merav Parter for fruitful discussions. This project has received funding from the European Union’s Horizon 2020 Research And Innovation Program under Grant Agreement No. 755839.","year":"2021","language":[{"iso":"eng"}],"doi":"10.1007/s00446-020-00380-5","isi":1,"quality_controlled":"1","project":[{"_id":"B67AFEDC-15C9-11EA-A837-991A96BB2854","name":"IST Austria Open Access Fund"}],"external_id":{"arxiv":["1903.05956"],"isi":["000556444600001"]},"main_file_link":[{"url":"https://doi.org/10.1007/s00446-020-00380-5","open_access":"1"}],"oa":1,"month":"12","publication_identifier":{"eissn":["1432-0452"],"issn":["0178-2770"]}},{"conference":{"name":"USENIX: Annual Technical Conference","end_date":"2018-07-13","start_date":"2018-07-11","location":"Boston, MA, United States"},"date_published":"2020-01-01T00:00:00Z","language":[{"iso":"eng"}],"publication":"Proceedings of the 2018 USENIX Annual Technical Conference","citation":{"chicago":"Arbel-Raviv, Maya, Trevor A Brown, and Adam Morrison. “Getting to the Root of Concurrent Binary Search Tree Performance.” In Proceedings of the 2018 USENIX Annual Technical Conference, 295–306. USENIX Association, 2020.","mla":"Arbel-Raviv, Maya, et al. “Getting to the Root of Concurrent Binary Search Tree Performance.” Proceedings of the 2018 USENIX Annual Technical Conference, USENIX Association, 2020, pp. 295–306.","short":"M. Arbel-Raviv, T.A. Brown, A. Morrison, in:, Proceedings of the 2018 USENIX Annual Technical Conference, USENIX Association, 2020, pp. 295–306.","ista":"Arbel-Raviv M, Brown TA, Morrison A. 2020. Getting to the root of concurrent binary search tree performance. Proceedings of the 2018 USENIX Annual Technical Conference. USENIX: Annual Technical Conference, 295–306.","apa":"Arbel-Raviv, M., Brown, T. A., & Morrison, A. (2020). Getting to the root of concurrent binary search tree performance. In Proceedings of the 2018 USENIX Annual Technical Conference (pp. 295–306). Boston, MA, United States: USENIX Association.","ieee":"M. Arbel-Raviv, T. A. Brown, and A. Morrison, “Getting to the root of concurrent binary search tree performance,” in Proceedings of the 2018 USENIX Annual Technical Conference, Boston, MA, United States, 2020, pp. 295–306.","ama":"Arbel-Raviv M, Brown TA, Morrison A. Getting to the root of concurrent binary search tree performance. In: Proceedings of the 2018 USENIX Annual Technical Conference. USENIX Association; 2020:295-306."},"main_file_link":[{"open_access":"1","url":"https://www.usenix.org/system/files/conference/atc18/atc18-arbel-raviv.pdf"}],"oa":1,"quality_controlled":"1","page":"295-306","project":[{"_id":"26450934-B435-11E9-9278-68D0E5697425","name":"NSERC Postdoctoral fellowship"}],"day":"01","month":"01","article_processing_charge":"No","publication_identifier":{"isbn":["9781939133021"]},"scopus_import":"1","author":[{"first_name":"Maya","last_name":"Arbel-Raviv","full_name":"Arbel-Raviv, Maya"},{"full_name":"Brown, Trevor A","id":"3569F0A0-F248-11E8-B48F-1D18A9856A87","last_name":"Brown","first_name":"Trevor A"},{"first_name":"Adam","last_name":"Morrison","full_name":"Morrison, Adam"}],"date_updated":"2021-01-11T15:25:48Z","date_created":"2020-01-14T07:27:08Z","oa_version":"Published Version","year":"2020","_id":"7272","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","publication_status":"published","ddc":["000"],"status":"public","title":"Getting to the root of concurrent binary search tree performance","department":[{"_id":"DaAl"}],"publisher":"USENIX Association","abstract":[{"lang":"eng","text":"Many systems rely on optimistic concurrent search trees for multi-core scalability. In principle, optimistic trees have a simple performance story: searches are read-only and so run in parallel, with writes to shared memory occurring only when modifying the data structure. However, this paper shows that in practice, obtaining the full performance benefits of optimistic search trees is not so simple.\r\n\r\nWe focus on optimistic binary search trees (BSTs) and perform a detailed performance analysis of 10 state-of-the-art BSTs on large scale x86-64 hardware, using both microbenchmarks and an in-memory database system. We find and explain significant unexpected performance differences between BSTs with similar tree structure and search implementations, which we trace to subtle performance-degrading interactions of BSTs with systems software and hardware subsystems. We further derive a prescriptive approach to avoid this performance degradation, as well as algorithmic insights on optimistic BST design. Our work underlines the gap between the theory and practice of multi-core performance, and calls for further research to help bridge this gap."}],"type":"conference"},{"day":"01","has_accepted_license":"1","article_processing_charge":"No","scopus_import":"1","date_published":"2020-02-01T00:00:00Z","publication":"23rd International Conference on Principles of Distributed Systems","citation":{"ama":"Alistarh D-A, Fedorov A, Koval N. In search of the fastest concurrent union-find algorithm. In: 23rd International Conference on Principles of Distributed Systems. Vol 153. Schloss Dagstuhl - Leibniz-Zentrum für Informatik; 2020:15:1-15:16. doi:10.4230/LIPIcs.OPODIS.2019.15","apa":"Alistarh, D.-A., Fedorov, A., & Koval, N. (2020). In search of the fastest concurrent union-find algorithm. In 23rd International Conference on Principles of Distributed Systems (Vol. 153, p. 15:1-15:16). Neuchatal, Switzerland: Schloss Dagstuhl - Leibniz-Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.OPODIS.2019.15","ieee":"D.-A. Alistarh, A. Fedorov, and N. Koval, “In search of the fastest concurrent union-find algorithm,” in 23rd International Conference on Principles of Distributed Systems, Neuchatal, Switzerland, 2020, vol. 153, p. 15:1-15:16.","ista":"Alistarh D-A, Fedorov A, Koval N. 2020. In search of the fastest concurrent union-find algorithm. 23rd International Conference on Principles of Distributed Systems. OPODIS: International Conference on Principles of Distributed Systems, LIPIcs, vol. 153, 15:1-15:16.","short":"D.-A. Alistarh, A. Fedorov, N. Koval, in:, 23rd International Conference on Principles of Distributed Systems, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020, p. 15:1-15:16.","mla":"Alistarh, Dan-Adrian, et al. “In Search of the Fastest Concurrent Union-Find Algorithm.” 23rd International Conference on Principles of Distributed Systems, vol. 153, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020, p. 15:1-15:16, doi:10.4230/LIPIcs.OPODIS.2019.15.","chicago":"Alistarh, Dan-Adrian, Alexander Fedorov, and Nikita Koval. “In Search of the Fastest Concurrent Union-Find Algorithm.” In 23rd International Conference on Principles of Distributed Systems, 153:15:1-15:16. Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020. https://doi.org/10.4230/LIPIcs.OPODIS.2019.15."},"page":"15:1-15:16","abstract":[{"lang":"eng","text":"Union-Find (or Disjoint-Set Union) is one of the fundamental problems in computer science; it has been well-studied from both theoretical and practical perspectives in the sequential case. Recently, there has been mounting interest in analyzing this problem in the concurrent scenario, and several asymptotically-efficient algorithms have been proposed. Yet, to date, there is very little known about the practical performance of concurrent Union-Find. This work addresses this gap. We evaluate and analyze the performance of several concurrent Union-Find algorithms and optimization strategies across a wide range of platforms (Intel, AMD, and ARM) and workloads (social, random, and road networks, as well as integrations into more complex algorithms). We first observe that, due to the limited computational cost, the number of induced cache misses is the critical determining factor for the performance of existing algorithms. We introduce new techniques to reduce this cost by storing node priorities implicitly and by using plain reads and writes in a way that does not affect the correctness of the algorithms. Finally, we show that Union-Find implementations are an interesting application for Transactional Memory (TM): one of the fastest algorithm variants we discovered is a sequential one that uses coarse-grained locking with the lock elision optimization to reduce synchronization cost and increase scalability. "}],"type":"conference","alternative_title":["LIPIcs"],"file":[{"file_name":"2019_LIPIcs_Alistarh.pdf","access_level":"open_access","file_size":13074131,"content_type":"application/pdf","creator":"dernst","relation":"main_file","file_id":"7609","date_updated":"2020-07-14T12:48:01Z","date_created":"2020-03-23T09:22:48Z","checksum":"d66f07ecb609d9f02433e39f80a447e9"}],"oa_version":"Published Version","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"7605","title":"In search of the fastest concurrent union-find algorithm","status":"public","ddc":["000"],"intvolume":" 153","month":"02","publication_identifier":{"isbn":["9783959771337"],"issn":["18688969"]},"conference":{"location":"Neuchatal, Switzerland","start_date":"2019-12-17","end_date":"2019-12-19","name":"OPODIS: International Conference on Principles of Distributed Systems"},"doi":"10.4230/LIPIcs.OPODIS.2019.15","language":[{"iso":"eng"}],"external_id":{"arxiv":["1911.06347"]},"tmp":{"legal_code_url":"https://creativecommons.org/licenses/by/3.0/legalcode","name":"Creative Commons Attribution 3.0 Unported (CC BY 3.0)","short":"CC BY (3.0)","image":"/images/cc_by.png"},"oa":1,"quality_controlled":"1","file_date_updated":"2020-07-14T12:48:01Z","license":"https://creativecommons.org/licenses/by/3.0/","author":[{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"},{"first_name":"Alexander","last_name":"Fedorov","full_name":"Fedorov, Alexander"},{"id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87","first_name":"Nikita","last_name":"Koval","full_name":"Koval, Nikita"}],"date_updated":"2023-02-23T13:12:12Z","date_created":"2020-03-22T23:00:46Z","volume":153,"year":"2020","publication_status":"published","publisher":"Schloss Dagstuhl - Leibniz-Zentrum für Informatik","department":[{"_id":"DaAl"}]},{"day":"01","has_accepted_license":"1","article_processing_charge":"No","date_published":"2020-07-01T00:00:00Z","publication":"Proceedings of the 2020 ACM Symposium on Principles of Distributed Computing","citation":{"ista":"Czumaj A, Davies P, Parter M. 2020. Simple, deterministic, constant-round coloring in the congested clique. Proceedings of the 2020 ACM Symposium on Principles of Distributed Computing. PODC: Symposium on Principles of Distributed Computing, 309–318.","ieee":"A. Czumaj, P. Davies, and M. Parter, “Simple, deterministic, constant-round coloring in the congested clique,” in Proceedings of the 2020 ACM Symposium on Principles of Distributed Computing, Salerno, Italy, 2020, pp. 309–318.","apa":"Czumaj, A., Davies, P., & Parter, M. (2020). Simple, deterministic, constant-round coloring in the congested clique. In Proceedings of the 2020 ACM Symposium on Principles of Distributed Computing (pp. 309–318). Salerno, Italy: Association for Computing Machinery. https://doi.org/10.1145/3382734.3405751","ama":"Czumaj A, Davies P, Parter M. Simple, deterministic, constant-round coloring in the congested clique. In: Proceedings of the 2020 ACM Symposium on Principles of Distributed Computing. Association for Computing Machinery; 2020:309-318. doi:10.1145/3382734.3405751","chicago":"Czumaj, Artur, Peter Davies, and Merav Parter. “Simple, Deterministic, Constant-Round Coloring in the Congested Clique.” In Proceedings of the 2020 ACM Symposium on Principles of Distributed Computing, 309–18. Association for Computing Machinery, 2020. https://doi.org/10.1145/3382734.3405751.","mla":"Czumaj, Artur, et al. “Simple, Deterministic, Constant-Round Coloring in the Congested Clique.” Proceedings of the 2020 ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2020, pp. 309–18, doi:10.1145/3382734.3405751.","short":"A. Czumaj, P. Davies, M. Parter, in:, Proceedings of the 2020 ACM Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2020, pp. 309–318."},"page":"309-318","abstract":[{"text":"We settle the complexity of the (Δ+1)-coloring and (Δ+1)-list coloring problems in the CONGESTED CLIQUE model by presenting a simple deterministic algorithm for both problems running in a constant number of rounds. This matches the complexity of the recent breakthrough randomized constant-round (Δ+1)-list coloring algorithm due to Chang et al. (PODC'19), and significantly improves upon the state-of-the-art O(logΔ)-round deterministic (Δ+1)-coloring bound of Parter (ICALP'18).\r\nA remarkable property of our algorithm is its simplicity. Whereas the state-of-the-art randomized algorithms for this problem are based on the quite involved local coloring algorithm of Chang et al. (STOC'18), our algorithm can be described in just a few lines. At a high level, it applies a careful derandomization of a recursive procedure which partitions the nodes and their respective palettes into separate bins. We show that after O(1) recursion steps, the remaining uncolored subgraph within each bin has linear size, and thus can be solved locally by collecting it to a single node. This algorithm can also be implemented in the Massively Parallel Computation (MPC) model provided that each machine has linear (in n, the number of nodes in the input graph) space.\r\nWe also show an extension of our algorithm to the MPC regime in which machines have sublinear space: we present the first deterministic (Δ+1)-list coloring algorithm designed for sublinear-space MPC, which runs in O(logΔ+loglogn) rounds.","lang":"eng"}],"type":"conference","file":[{"checksum":"46fe4fc58a64eb04068115573f631d4c","success":1,"date_created":"2020-10-08T08:17:36Z","date_updated":"2020-10-08T08:17:36Z","relation":"main_file","file_id":"8624","content_type":"application/pdf","file_size":520051,"creator":"pdavies","access_level":"open_access","file_name":"ColoringArxiv.pdf"}],"oa_version":"Submitted Version","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"7803","ddc":["000"],"status":"public","title":"Simple, deterministic, constant-round coloring in the congested clique","month":"07","conference":{"name":"PODC: Symposium on Principles of Distributed Computing","location":"Salerno, Italy","start_date":"2020-08-03","end_date":"2020-08-07"},"doi":"10.1145/3382734.3405751","language":[{"iso":"eng"}],"oa":1,"external_id":{"arxiv":["2009.06043"]},"quality_controlled":"1","project":[{"grant_number":"754411","_id":"260C2330-B435-11E9-9278-68D0E5697425","call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships"}],"file_date_updated":"2020-10-08T08:17:36Z","ec_funded":1,"author":[{"full_name":"Czumaj, Artur","last_name":"Czumaj","first_name":"Artur","orcid":"0000-0002-5646-9524"},{"full_name":"Davies, Peter","orcid":"0000-0002-5646-9524","id":"11396234-BB50-11E9-B24C-90FCE5697425","last_name":"Davies","first_name":"Peter"},{"last_name":"Parter","first_name":"Merav","full_name":"Parter, Merav"}],"date_created":"2020-05-06T09:02:14Z","date_updated":"2021-01-12T08:15:37Z","year":"2020","publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"Association for Computing Machinery"},{"publication_identifier":{"isbn":["9783959771689"],"issn":["1868-8969"]},"month":"08","oa":1,"tmp":{"legal_code_url":"https://creativecommons.org/licenses/by/3.0/legalcode","name":"Creative Commons Attribution 3.0 Unported (CC BY 3.0)","short":"CC BY (3.0)","image":"/images/cc_by.png"},"external_id":{"arxiv":["2008.01009"]},"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"quality_controlled":"1","doi":"10.4230/LIPIcs.DISC.2020.3","conference":{"start_date":"2020-10-12","location":"Freiburg, Germany","end_date":"2020-10-16","name":"DISC: Symposium on Distributed Computing"},"language":[{"iso":"eng"}],"ec_funded":1,"file_date_updated":"2021-03-11T12:33:35Z","year":"2020","acknowledgement":"Vitaly Aksenov: Government of Russian Federation (Grant 08-08).\r\nDan Alistarh: ERC Starting Grant 805223 ScaleML.","publisher":"Schloss Dagstuhl - Leibniz-Zentrum für Informatik","department":[{"_id":"DaAl"}],"publication_status":"published","author":[{"last_name":"Aksenov","first_name":"Vitaly","full_name":"Aksenov, Vitaly"},{"orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian","full_name":"Alistarh, Dan-Adrian"},{"first_name":"Alexandra","last_name":"Drozdova","full_name":"Drozdova, Alexandra"},{"last_name":"Mohtashami","first_name":"Amirkeivan","full_name":"Mohtashami, Amirkeivan"}],"volume":179,"date_created":"2020-11-05T15:26:17Z","date_updated":"2023-02-23T13:41:40Z","series_title":"LIPIcs","article_processing_charge":"No","has_accepted_license":"1","day":"03","citation":{"chicago":"Aksenov, Vitaly, Dan-Adrian Alistarh, Alexandra Drozdova, and Amirkeivan Mohtashami. “The Splay-List: A Distribution-Adaptive Concurrent Skip-List.” In 34th International Symposium on Distributed Computing, 179:3:1-3:18. LIPIcs. Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020. https://doi.org/10.4230/LIPIcs.DISC.2020.3.","mla":"Aksenov, Vitaly, et al. “The Splay-List: A Distribution-Adaptive Concurrent Skip-List.” 34th International Symposium on Distributed Computing, vol. 179, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020, p. 3:1-3:18, doi:10.4230/LIPIcs.DISC.2020.3.","short":"V. Aksenov, D.-A. Alistarh, A. Drozdova, A. Mohtashami, in:, 34th International Symposium on Distributed Computing, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020, p. 3:1-3:18.","ista":"Aksenov V, Alistarh D-A, Drozdova A, Mohtashami A. 2020. The splay-list: A distribution-adaptive concurrent skip-list. 34th International Symposium on Distributed Computing. DISC: Symposium on Distributed ComputingLIPIcs vol. 179, 3:1-3:18.","apa":"Aksenov, V., Alistarh, D.-A., Drozdova, A., & Mohtashami, A. (2020). The splay-list: A distribution-adaptive concurrent skip-list. In 34th International Symposium on Distributed Computing (Vol. 179, p. 3:1-3:18). Freiburg, Germany: Schloss Dagstuhl - Leibniz-Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.DISC.2020.3","ieee":"V. Aksenov, D.-A. Alistarh, A. Drozdova, and A. Mohtashami, “The splay-list: A distribution-adaptive concurrent skip-list,” in 34th International Symposium on Distributed Computing, Freiburg, Germany, 2020, vol. 179, p. 3:1-3:18.","ama":"Aksenov V, Alistarh D-A, Drozdova A, Mohtashami A. The splay-list: A distribution-adaptive concurrent skip-list. In: 34th International Symposium on Distributed Computing. Vol 179. LIPIcs. Schloss Dagstuhl - Leibniz-Zentrum für Informatik; 2020:3:1-3:18. doi:10.4230/LIPIcs.DISC.2020.3"},"publication":"34th International Symposium on Distributed Computing","page":"3:1-3:18","date_published":"2020-08-03T00:00:00Z","type":"conference","abstract":[{"text":"The design and implementation of efficient concurrent data structures have\r\nseen significant attention. However, most of this work has focused on\r\nconcurrent data structures providing good \\emph{worst-case} guarantees. In real\r\nworkloads, objects are often accessed at different rates, since access\r\ndistributions may be non-uniform. Efficient distribution-adaptive data\r\nstructures are known in the sequential case, e.g. the splay-trees; however,\r\nthey often are hard to translate efficiently in the concurrent case.\r\n In this paper, we investigate distribution-adaptive concurrent data\r\nstructures and propose a new design called the splay-list. At a high level, the\r\nsplay-list is similar to a standard skip-list, with the key distinction that\r\nthe height of each element adapts dynamically to its access rate: popular\r\nelements ``move up,'' whereas rarely-accessed elements decrease in height. We\r\nshow that the splay-list provides order-optimal amortized complexity bounds for\r\na subset of operations while being amenable to efficient concurrent\r\nimplementation. Experimental results show that the splay-list can leverage\r\ndistribution-adaptivity to improve on the performance of classic concurrent\r\ndesigns, and can outperform the only previously-known distribution-adaptive\r\ndesign in certain settings.","lang":"eng"}],"_id":"8725","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","intvolume":" 179","status":"public","ddc":["000"],"title":"The splay-list: A distribution-adaptive concurrent skip-list","oa_version":"Published Version","file":[{"checksum":"a626a9c47df52b6f6d97edd910dae4ba","success":1,"date_created":"2021-03-11T12:33:35Z","date_updated":"2021-03-11T12:33:35Z","relation":"main_file","file_id":"9237","file_size":740358,"content_type":"application/pdf","creator":"dernst","access_level":"open_access","file_name":"2020_LIPIcs_Aksenov.pdf"}]},{"month":"12","publication_identifier":{"isbn":["9781713829546"],"issn":["10495258"]},"language":[{"iso":"eng"}],"conference":{"end_date":"2020-12-12","start_date":"2020-12-06","location":"Vancouver, Canada","name":"NeurIPS: Conference on Neural Information Processing Systems"},"quality_controlled":"1","project":[{"_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning"}],"main_file_link":[{"open_access":"1","url":"https://proceedings.neurips.cc/paper/2020/hash/d1ff1ec86b62cd5f3903ff19c3a326b2-Abstract.html"}],"external_id":{"arxiv":["2004.14340"]},"oa":1,"ec_funded":1,"date_created":"2021-07-04T22:01:26Z","date_updated":"2023-02-23T14:03:06Z","volume":33,"author":[{"first_name":"Sidak Pal","last_name":"Singh","id":"DD138E24-D89D-11E9-9DC0-DEF6E5697425","full_name":"Singh, Sidak Pal"},{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"}],"publication_status":"published","publisher":"Curran Associates","department":[{"_id":"DaAl"},{"_id":"ToHe"}],"acknowledgement":"This project has received funding from the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). Also, we would like to thank Alexander Shevchenko, Alexandra Peste, and other members of the group for fruitful discussions.","year":"2020","day":"06","article_processing_charge":"No","scopus_import":"1","date_published":"2020-12-06T00:00:00Z","page":"18098-18109","publication":"Advances in Neural Information Processing Systems","citation":{"chicago":"Singh, Sidak Pal, and Dan-Adrian Alistarh. “WoodFisher: Efficient Second-Order Approximation for Neural Network Compression.” In Advances in Neural Information Processing Systems, 33:18098–109. Curran Associates, 2020.","mla":"Singh, Sidak Pal, and Dan-Adrian Alistarh. “WoodFisher: Efficient Second-Order Approximation for Neural Network Compression.” Advances in Neural Information Processing Systems, vol. 33, Curran Associates, 2020, pp. 18098–109.","short":"S.P. Singh, D.-A. Alistarh, in:, Advances in Neural Information Processing Systems, Curran Associates, 2020, pp. 18098–18109.","ista":"Singh SP, Alistarh D-A. 2020. WoodFisher: Efficient second-order approximation for neural network compression. Advances in Neural Information Processing Systems. NeurIPS: Conference on Neural Information Processing Systems vol. 33, 18098–18109.","ieee":"S. P. Singh and D.-A. Alistarh, “WoodFisher: Efficient second-order approximation for neural network compression,” in Advances in Neural Information Processing Systems, Vancouver, Canada, 2020, vol. 33, pp. 18098–18109.","apa":"Singh, S. P., & Alistarh, D.-A. (2020). WoodFisher: Efficient second-order approximation for neural network compression. In Advances in Neural Information Processing Systems (Vol. 33, pp. 18098–18109). Vancouver, Canada: Curran Associates.","ama":"Singh SP, Alistarh D-A. WoodFisher: Efficient second-order approximation for neural network compression. In: Advances in Neural Information Processing Systems. Vol 33. Curran Associates; 2020:18098-18109."},"abstract":[{"text":"Second-order information, in the form of Hessian- or Inverse-Hessian-vector products, is a fundamental tool for solving optimization problems. Recently, there has been significant interest in utilizing this information in the context of deep\r\nneural networks; however, relatively little is known about the quality of existing approximations in this context. Our work examines this question, identifies issues with existing approaches, and proposes a method called WoodFisher to compute a faithful and efficient estimate of the inverse Hessian. Our main application is to neural network compression, where we build on the classic Optimal Brain Damage/Surgeon framework. We demonstrate that WoodFisher significantly outperforms popular state-of-the-art methods for oneshot pruning. Further, even when iterative, gradual pruning is allowed, our method results in a gain in test accuracy over the state-of-the-art approaches, for standard image classification datasets such as ImageNet ILSVRC. We examine how our method can be extended to take into account first-order information, as well as\r\nillustrate its ability to automatically set layer-wise pruning thresholds and perform compression in the limited-data regime. The code is available at the following link, https://github.com/IST-DASLab/WoodFisher.","lang":"eng"}],"type":"conference","oa_version":"Published Version","title":"WoodFisher: Efficient second-order approximation for neural network compression","status":"public","intvolume":" 33","user_id":"6785fbc1-c503-11eb-8a32-93094b40e1cf","_id":"9632"},{"date_published":"2020-12-06T00:00:00Z","citation":{"short":"V. Aksenov, D.-A. Alistarh, J. Korhonen, in:, Advances in Neural Information Processing Systems, Curran Associates, 2020, pp. 22361–22372.","mla":"Aksenov, Vitaly, et al. “Scalable Belief Propagation via Relaxed Scheduling.” Advances in Neural Information Processing Systems, vol. 33, Curran Associates, 2020, pp. 22361–72.","chicago":"Aksenov, Vitaly, Dan-Adrian Alistarh, and Janne Korhonen. “Scalable Belief Propagation via Relaxed Scheduling.” In Advances in Neural Information Processing Systems, 33:22361–72. Curran Associates, 2020.","ama":"Aksenov V, Alistarh D-A, Korhonen J. Scalable belief propagation via relaxed scheduling. In: Advances in Neural Information Processing Systems. Vol 33. Curran Associates; 2020:22361-22372.","ieee":"V. Aksenov, D.-A. Alistarh, and J. Korhonen, “Scalable belief propagation via relaxed scheduling,” in Advances in Neural Information Processing Systems, Vancouver, Canada, 2020, vol. 33, pp. 22361–22372.","apa":"Aksenov, V., Alistarh, D.-A., & Korhonen, J. (2020). Scalable belief propagation via relaxed scheduling. In Advances in Neural Information Processing Systems (Vol. 33, pp. 22361–22372). Vancouver, Canada: Curran Associates.","ista":"Aksenov V, Alistarh D-A, Korhonen J. 2020. Scalable belief propagation via relaxed scheduling. Advances in Neural Information Processing Systems. NeurIPS: Conference on Neural Information Processing Systems vol. 33, 22361–22372."},"publication":"Advances in Neural Information Processing Systems","page":"22361-22372","article_processing_charge":"No","day":"06","scopus_import":"1","oa_version":"Published Version","_id":"9631","user_id":"6785fbc1-c503-11eb-8a32-93094b40e1cf","intvolume":" 33","title":"Scalable belief propagation via relaxed scheduling","status":"public","abstract":[{"text":"The ability to leverage large-scale hardware parallelism has been one of the key enablers of the accelerated recent progress in machine learning. Consequently, there has been considerable effort invested into developing efficient parallel variants of classic machine learning algorithms. However, despite the wealth of knowledge on parallelization, some classic machine learning algorithms often prove hard to parallelize efficiently while maintaining convergence. In this paper, we focus on efficient parallel algorithms for the key machine learning task of inference on graphical models, in particular on the fundamental belief propagation algorithm. We address the challenge of efficiently parallelizing this classic paradigm by showing how to leverage scalable relaxed schedulers in this context. We present an extensive empirical study, showing that our approach outperforms previous parallel belief propagation implementations both in terms of scalability and in terms of wall-clock convergence time, on a range of practical applications.","lang":"eng"}],"type":"conference","conference":{"name":"NeurIPS: Conference on Neural Information Processing Systems","end_date":"2020-12-12","start_date":"2020-12-06","location":"Vancouver, Canada"},"language":[{"iso":"eng"}],"oa":1,"main_file_link":[{"url":"https://proceedings.neurips.cc/paper/2020/hash/fdb2c3bab9d0701c4a050a4d8d782c7f-Abstract.html","open_access":"1"}],"external_id":{"arxiv":["2002.11505"]},"project":[{"_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223","call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning"}],"quality_controlled":"1","publication_identifier":{"issn":["10495258"],"isbn":["9781713829546"]},"month":"12","author":[{"full_name":"Aksenov, Vitaly","last_name":"Aksenov","first_name":"Vitaly"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"},{"first_name":"Janne","last_name":"Korhonen","id":"C5402D42-15BC-11E9-A202-CA2BE6697425","full_name":"Korhonen, Janne"}],"volume":33,"date_created":"2021-07-04T22:01:26Z","date_updated":"2023-02-23T14:03:03Z","year":"2020","acknowledgement":"We thank Marco Mondelli for discussions related to LDPC decoding, and Giorgi Nadiradze for discussions on analysis of relaxed schedulers. This project has received funding from the European Research Council (ERC) under the European\r\nUnion’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML).","publisher":"Curran Associates","department":[{"_id":"DaAl"}],"publication_status":"published","ec_funded":1},{"_id":"9415","user_id":"3E5EF7F0-F248-11E8-B48F-1D18A9856A87","ddc":["000"],"status":"public","title":"Inducing and exploiting activation sparsity for fast neural network inference","intvolume":" 119","oa_version":"Published Version","file":[{"relation":"main_file","file_id":"9421","checksum":"2aaaa7d7226e49161311d91627cf783b","success":1,"date_created":"2021-05-25T09:51:36Z","date_updated":"2021-05-25T09:51:36Z","access_level":"open_access","file_name":"2020_PMLR_Kurtz.pdf","content_type":"application/pdf","file_size":741899,"creator":"kschuh"}],"type":"conference","abstract":[{"text":"Optimizing convolutional neural networks for fast inference has recently become an extremely active area of research. One of the go-to solutions in this context is weight pruning, which aims to reduce computational and memory footprint by removing large subsets of the connections in a neural network. Surprisingly, much less attention has been given to exploiting sparsity in the activation maps, which tend to be naturally sparse in many settings thanks to the structure of rectified linear (ReLU) activation functions. In this paper, we present an in-depth analysis of methods for maximizing the sparsity of the activations in a trained neural network, and show that, when coupled with an efficient sparse-input convolution algorithm, we can leverage this sparsity for significant performance gains. To induce highly sparse activation maps without accuracy loss, we introduce a new regularization technique, coupled with a new threshold-based sparsification method based on a parameterized activation function called Forced-Activation-Threshold Rectified Linear Unit (FATReLU). We examine the impact of our methods on popular image classification models, showing that most architectures can adapt to significantly sparser activation maps without any accuracy loss. Our second contribution is showing that these these compression gains can be translated into inference speedups: we provide a new algorithm to enable fast convolution operations over networks with sparse activations, and show that it can enable significant speedups for end-to-end inference on a range of popular models on the large-scale ImageNet image classification task on modern Intel CPUs, with little or no retraining cost. ","lang":"eng"}],"publication":"37th International Conference on Machine Learning, ICML 2020","citation":{"chicago":"Kurtz, Mark, Justin Kopinsky, Rati Gelashvili, Alexander Matveev, John Carr, Michael Goin, William Leiserson, et al. “Inducing and Exploiting Activation Sparsity for Fast Neural Network Inference.” In 37th International Conference on Machine Learning, ICML 2020, 119:5533–43, 2020.","mla":"Kurtz, Mark, et al. “Inducing and Exploiting Activation Sparsity for Fast Neural Network Inference.” 37th International Conference on Machine Learning, ICML 2020, vol. 119, 2020, pp. 5533–43.","short":"M. Kurtz, J. Kopinsky, R. Gelashvili, A. Matveev, J. Carr, M. Goin, W. Leiserson, S. Moore, B. Nell, N. Shavit, D.-A. Alistarh, in:, 37th International Conference on Machine Learning, ICML 2020, 2020, pp. 5533–5543.","ista":"Kurtz M, Kopinsky J, Gelashvili R, Matveev A, Carr J, Goin M, Leiserson W, Moore S, Nell B, Shavit N, Alistarh D-A. 2020. Inducing and exploiting activation sparsity for fast neural network inference. 37th International Conference on Machine Learning, ICML 2020. ICML: International Conference on Machine Learning vol. 119, 5533–5543.","ieee":"M. Kurtz et al., “Inducing and exploiting activation sparsity for fast neural network inference,” in 37th International Conference on Machine Learning, ICML 2020, Online, 2020, vol. 119, pp. 5533–5543.","apa":"Kurtz, M., Kopinsky, J., Gelashvili, R., Matveev, A., Carr, J., Goin, M., … Alistarh, D.-A. (2020). Inducing and exploiting activation sparsity for fast neural network inference. In 37th International Conference on Machine Learning, ICML 2020 (Vol. 119, pp. 5533–5543). Online.","ama":"Kurtz M, Kopinsky J, Gelashvili R, et al. Inducing and exploiting activation sparsity for fast neural network inference. In: 37th International Conference on Machine Learning, ICML 2020. Vol 119. ; 2020:5533-5543."},"page":"5533-5543","date_published":"2020-07-12T00:00:00Z","scopus_import":"1","day":"12","has_accepted_license":"1","article_processing_charge":"No","year":"2020","department":[{"_id":"DaAl"}],"author":[{"first_name":"Mark","last_name":"Kurtz","full_name":"Kurtz, Mark"},{"full_name":"Kopinsky, Justin","last_name":"Kopinsky","first_name":"Justin"},{"full_name":"Gelashvili, Rati","last_name":"Gelashvili","first_name":"Rati"},{"full_name":"Matveev, Alexander","first_name":"Alexander","last_name":"Matveev"},{"last_name":"Carr","first_name":"John","full_name":"Carr, John"},{"first_name":"Michael","last_name":"Goin","full_name":"Goin, Michael"},{"full_name":"Leiserson, William","first_name":"William","last_name":"Leiserson"},{"full_name":"Moore, Sage","first_name":"Sage","last_name":"Moore"},{"full_name":"Nell, Bill","last_name":"Nell","first_name":"Bill"},{"full_name":"Shavit, Nir","first_name":"Nir","last_name":"Shavit"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"date_updated":"2023-02-23T13:57:24Z","date_created":"2021-05-23T22:01:45Z","volume":119,"file_date_updated":"2021-05-25T09:51:36Z","oa":1,"quality_controlled":"1","conference":{"end_date":"2020-07-18","location":"Online","start_date":"2020-07-12","name":"ICML: International Conference on Machine Learning"},"language":[{"iso":"eng"}],"month":"07","publication_identifier":{"issn":["2640-3498"]}},{"type":"journal_article","abstract":[{"text":"Modern scientific instruments produce vast amounts of data, which can overwhelm the processing ability of computer systems. Lossy compression of data is an intriguing solution, but comes with its own drawbacks, such as potential signal loss, and the need for careful optimization of the compression ratio. In this work, we focus on a setting where this problem is especially acute: compressive sensing frameworks for interferometry and medical imaging. We ask the following question: can the precision of the data representation be lowered for all inputs, with recovery guarantees and practical performance Our first contribution is a theoretical analysis of the normalized Iterative Hard Thresholding (IHT) algorithm when all input data, meaning both the measurement matrix and the observation vector are quantized aggressively. We present a variant of low precision normalized IHT that, under mild conditions, can still provide recovery guarantees. The second contribution is the application of our quantization framework to radio astronomy and magnetic resonance imaging. We show that lowering the precision of the data can significantly accelerate image recovery. We evaluate our approach on telescope data and samples of brain images using CPU and FPGA implementations achieving up to a 9x speedup with negligible loss of recovery quality.","lang":"eng"}],"status":"public","title":"Compressive sensing using iterative hard thresholding with low precision data representation: Theory and applications","intvolume":" 68","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","_id":"8268","oa_version":"Preprint","scopus_import":"1","day":"20","article_processing_charge":"No","article_type":"original","page":"4268-4282","publication":"IEEE Transactions on Signal Processing","citation":{"mla":"Gurel, Nezihe Merve, et al. “Compressive Sensing Using Iterative Hard Thresholding with Low Precision Data Representation: Theory and Applications.” IEEE Transactions on Signal Processing, vol. 68, IEEE, 2020, pp. 4268–82, doi:10.1109/TSP.2020.3010355.","short":"N.M. Gurel, K. Kara, A. Stojanov, T. Smith, T. Lemmin, D.-A. Alistarh, M. Puschel, C. Zhang, IEEE Transactions on Signal Processing 68 (2020) 4268–4282.","chicago":"Gurel, Nezihe Merve, Kaan Kara, Alen Stojanov, Tyler Smith, Thomas Lemmin, Dan-Adrian Alistarh, Markus Puschel, and Ce Zhang. “Compressive Sensing Using Iterative Hard Thresholding with Low Precision Data Representation: Theory and Applications.” IEEE Transactions on Signal Processing. IEEE, 2020. https://doi.org/10.1109/TSP.2020.3010355.","ama":"Gurel NM, Kara K, Stojanov A, et al. Compressive sensing using iterative hard thresholding with low precision data representation: Theory and applications. IEEE Transactions on Signal Processing. 2020;68:4268-4282. doi:10.1109/TSP.2020.3010355","ista":"Gurel NM, Kara K, Stojanov A, Smith T, Lemmin T, Alistarh D-A, Puschel M, Zhang C. 2020. Compressive sensing using iterative hard thresholding with low precision data representation: Theory and applications. IEEE Transactions on Signal Processing. 68, 4268–4282.","ieee":"N. M. Gurel et al., “Compressive sensing using iterative hard thresholding with low precision data representation: Theory and applications,” IEEE Transactions on Signal Processing, vol. 68. IEEE, pp. 4268–4282, 2020.","apa":"Gurel, N. M., Kara, K., Stojanov, A., Smith, T., Lemmin, T., Alistarh, D.-A., … Zhang, C. (2020). Compressive sensing using iterative hard thresholding with low precision data representation: Theory and applications. IEEE Transactions on Signal Processing. IEEE. https://doi.org/10.1109/TSP.2020.3010355"},"date_published":"2020-07-20T00:00:00Z","publication_status":"published","publisher":"IEEE","department":[{"_id":"DaAl"}],"year":"2020","acknowledgement":"The authors would like to thank Dr. Michiel Brentjens at the Netherlands Institute for Radio Astronomy (ASTRON) for providing radio interferometer data and Dr. Josip Marjanovic and Dr. Franciszek Hennel at the Magnetic Resonance Technology of ETH Zurich for providing their insights on the experiments. CZ and the DS3Lab gratefully acknowledge the support from the Swiss Data Science Center, Alibaba, Google Focused Research Awards, Huawei, MeteoSwiss, Oracle Labs, Swisscom, Zurich Insurance, Chinese Scholarship Council, and the Department of Computer Science at ETH Zurich.","date_updated":"2023-08-22T08:40:08Z","date_created":"2020-08-16T22:00:56Z","volume":68,"author":[{"full_name":"Gurel, Nezihe Merve","last_name":"Gurel","first_name":"Nezihe Merve"},{"full_name":"Kara, Kaan","last_name":"Kara","first_name":"Kaan"},{"full_name":"Stojanov, Alen","first_name":"Alen","last_name":"Stojanov"},{"last_name":"Smith","first_name":"Tyler","full_name":"Smith, Tyler"},{"full_name":"Lemmin, Thomas","first_name":"Thomas","last_name":"Lemmin"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"},{"first_name":"Markus","last_name":"Puschel","full_name":"Puschel, Markus"},{"last_name":"Zhang","first_name":"Ce","full_name":"Zhang, Ce"}],"month":"07","publication_identifier":{"eissn":["19410476"],"issn":["1053587X"]},"isi":1,"quality_controlled":"1","oa":1,"main_file_link":[{"url":"https://arxiv.org/abs/1802.04907","open_access":"1"}],"external_id":{"isi":["000562044500001"],"arxiv":["1802.04907"]},"language":[{"iso":"eng"}],"doi":"10.1109/TSP.2020.3010355"},{"abstract":[{"lang":"eng","text":"Load imbalance pervasively exists in distributed deep learning training systems, either caused by the inherent imbalance in learned tasks or by the system itself. Traditional synchronous Stochastic Gradient Descent (SGD)\r\nachieves good accuracy for a wide variety of tasks, but relies on global synchronization to accumulate the gradients at every training step. In this paper, we propose eager-SGD, which relaxes the global synchronization for\r\ndecentralized accumulation. To implement eager-SGD, we propose to use two partial collectives: solo and majority. With solo allreduce, the faster processes contribute their gradients eagerly without waiting for the slower processes, whereas with majority allreduce, at least half of the participants must contribute gradients before continuing, all without using a central parameter server. We theoretically prove the convergence of the algorithms and describe the partial collectives in detail. Experimental results on load-imbalanced environments (CIFAR-10, ImageNet, and UCF101 datasets) show\r\nthat eager-SGD achieves 1.27x speedup over the state-of-the-art synchronous SGD, without losing accuracy."}],"type":"conference","oa_version":"Preprint","_id":"8722","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","status":"public","title":"Taming unbalanced training workloads in deep learning with partial collective operations","article_processing_charge":"No","day":"01","date_published":"2020-02-01T00:00:00Z","citation":{"ista":"Li S, Tal Ben-Nun TB-N, Girolamo SD, Alistarh D-A, Hoefler T. 2020. Taming unbalanced training workloads in deep learning with partial collective operations. Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. PPoPP: Sympopsium on Principles and Practice of Parallel Programming, 45–61.","apa":"Li, S., Tal Ben-Nun, T. B.-N., Girolamo, S. D., Alistarh, D.-A., & Hoefler, T. (2020). Taming unbalanced training workloads in deep learning with partial collective operations. In Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (pp. 45–61). San Diego, CA, United States: Association for Computing Machinery. https://doi.org/10.1145/3332466.3374528","ieee":"S. Li, T. B.-N. Tal Ben-Nun, S. D. Girolamo, D.-A. Alistarh, and T. Hoefler, “Taming unbalanced training workloads in deep learning with partial collective operations,” in Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, San Diego, CA, United States, 2020, pp. 45–61.","ama":"Li S, Tal Ben-Nun TB-N, Girolamo SD, Alistarh D-A, Hoefler T. Taming unbalanced training workloads in deep learning with partial collective operations. In: Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. Association for Computing Machinery; 2020:45-61. doi:10.1145/3332466.3374528","chicago":"Li, Shigang, Tal Ben-Nun Tal Ben-Nun, Salvatore Di Girolamo, Dan-Adrian Alistarh, and Torsten Hoefler. “Taming Unbalanced Training Workloads in Deep Learning with Partial Collective Operations.” In Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, 45–61. Association for Computing Machinery, 2020. https://doi.org/10.1145/3332466.3374528.","mla":"Li, Shigang, et al. “Taming Unbalanced Training Workloads in Deep Learning with Partial Collective Operations.” Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2020, pp. 45–61, doi:10.1145/3332466.3374528.","short":"S. Li, T.B.-N. Tal Ben-Nun, S.D. Girolamo, D.-A. Alistarh, T. Hoefler, in:, Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2020, pp. 45–61."},"publication":"Proceedings of the 25th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming","page":"45-61","ec_funded":1,"author":[{"full_name":"Li, Shigang","first_name":"Shigang","last_name":"Li"},{"first_name":"Tal Ben-Nun","last_name":"Tal Ben-Nun","full_name":"Tal Ben-Nun, Tal Ben-Nun"},{"first_name":"Salvatore Di","last_name":"Girolamo","full_name":"Girolamo, Salvatore Di"},{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X"},{"full_name":"Hoefler, Torsten","last_name":"Hoefler","first_name":"Torsten"}],"date_created":"2020-11-05T15:25:30Z","date_updated":"2023-08-22T12:13:48Z","year":"2020","publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"publication_status":"published","month":"02","doi":"10.1145/3332466.3374528","conference":{"name":"PPoPP: Sympopsium on Principles and Practice of Parallel Programming","location":"San Diego, CA, United States","start_date":"2020-02-22","end_date":"2020-02-26"},"language":[{"iso":"eng"}],"oa":1,"main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/1908.04207"}],"external_id":{"arxiv":["1908.04207"],"isi":["000564476500004"]},"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"isi":1,"quality_controlled":"1"},{"date_published":"2020-03-01T00:00:00Z","article_type":"original","page":"506-517","publication":"Ecology Letters","citation":{"chicago":"Rybicki, Joel, Nerea Abrego, and Otso Ovaskainen. “Habitat Fragmentation and Species Diversity in Competitive Communities.” Ecology Letters. Wiley, 2020. https://doi.org/10.1111/ele.13450.","short":"J. Rybicki, N. Abrego, O. Ovaskainen, Ecology Letters 23 (2020) 506–517.","mla":"Rybicki, Joel, et al. “Habitat Fragmentation and Species Diversity in Competitive Communities.” Ecology Letters, vol. 23, no. 3, Wiley, 2020, pp. 506–17, doi:10.1111/ele.13450.","ieee":"J. Rybicki, N. Abrego, and O. Ovaskainen, “Habitat fragmentation and species diversity in competitive communities,” Ecology Letters, vol. 23, no. 3. Wiley, pp. 506–517, 2020.","apa":"Rybicki, J., Abrego, N., & Ovaskainen, O. (2020). Habitat fragmentation and species diversity in competitive communities. Ecology Letters. Wiley. https://doi.org/10.1111/ele.13450","ista":"Rybicki J, Abrego N, Ovaskainen O. 2020. Habitat fragmentation and species diversity in competitive communities. Ecology Letters. 23(3), 506–517.","ama":"Rybicki J, Abrego N, Ovaskainen O. Habitat fragmentation and species diversity in competitive communities. Ecology Letters. 2020;23(3):506-517. doi:10.1111/ele.13450"},"day":"01","has_accepted_license":"1","article_processing_charge":"Yes (via OA deal)","scopus_import":"1","file":[{"checksum":"372f67f2744f4b6049e9778364766c22","date_created":"2020-02-14T12:02:50Z","date_updated":"2020-07-14T12:47:54Z","file_id":"7486","relation":"main_file","creator":"dernst","content_type":"application/pdf","file_size":3005474,"access_level":"open_access","file_name":"2020_EcologyLetters_Rybicki.pdf"}],"oa_version":"Published Version","ddc":["000"],"status":"public","title":"Habitat fragmentation and species diversity in competitive communities","intvolume":" 23","user_id":"c635000d-4b10-11ee-a964-aac5a93f6ac1","_id":"7224","abstract":[{"lang":"eng","text":"Habitat loss is one of the key drivers of the ongoing decline of biodiversity. However, ecologists still argue about how fragmentation of habitat (independent of habitat loss) affects species richness. The recently proposed habitat amount hypothesis posits that species richness only depends on the total amount of habitat in a local landscape. In contrast, empirical studies report contrasting patterns: some find positive and others negative effects of fragmentation per se on species richness. To explain this apparent disparity, we devise a stochastic, spatially explicit model of competitive species communities in heterogeneous habitats. The model shows that habitat loss and fragmentation have complex effects on species diversity in competitive communities. When the total amount of habitat is large, fragmentation per se tends to increase species diversity, but if the total amount of habitat is small, the situation is reversed: fragmentation per se decreases species diversity."}],"issue":"3","type":"journal_article","language":[{"iso":"eng"}],"doi":"10.1111/ele.13450","isi":1,"quality_controlled":"1","project":[{"call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships","_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411"},{"grant_number":"840605","_id":"26A5D39A-B435-11E9-9278-68D0E5697425","name":"Coordination in constrained and natural distributed systems","call_identifier":"H2020"}],"external_id":{"isi":["000503625200001"]},"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"month":"03","publication_identifier":{"eissn":["1461-0248"],"issn":["1461-023X"]},"date_created":"2020-01-04T11:04:30Z","date_updated":"2023-09-05T16:04:30Z","volume":23,"author":[{"orcid":"0000-0002-6432-6646","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","last_name":"Rybicki","first_name":"Joel","full_name":"Rybicki, Joel"},{"full_name":"Abrego, Nerea","first_name":"Nerea","last_name":"Abrego"},{"first_name":"Otso","last_name":"Ovaskainen","full_name":"Ovaskainen, Otso"}],"publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"Wiley","year":"2020","file_date_updated":"2020-07-14T12:47:54Z","ec_funded":1},{"abstract":[{"lang":"eng","text":"We study the problem of learning from multiple untrusted data sources, a scenario of increasing practical relevance given the recent emergence of crowdsourcing and collaborative learning paradigms. Specifically, we analyze the situation in which a learning system obtains datasets from multiple sources, some of which might be biased or even adversarially perturbed. It is\r\nknown that in the single-source case, an adversary with the power to corrupt a fixed fraction of the training data can prevent PAC-learnability, that is, even in the limit of infinitely much training data, no learning system can approach the optimal test error. In this work we show that, surprisingly, the same is not true in the multi-source setting, where the adversary can arbitrarily\r\ncorrupt a fixed fraction of the data sources. Our main results are a generalization bound that provides finite-sample guarantees for this learning setting, as well as corresponding lower bounds. Besides establishing PAC-learnability our results also show that in a cooperative learning setting sharing data with other parties has provable benefits, even if some\r\nparticipants are malicious. "}],"type":"conference","oa_version":"Published Version","file":[{"access_level":"open_access","file_name":"2020_PMLR_Konstantinov.pdf","content_type":"application/pdf","file_size":281286,"creator":"dernst","relation":"main_file","file_id":"9120","checksum":"cc755d0054bc4b2be778ea7aa7884d2f","success":1,"date_created":"2021-02-15T09:00:01Z","date_updated":"2021-02-15T09:00:01Z"}],"intvolume":" 119","title":"On the sample complexity of adversarial multi-source PAC learning","status":"public","ddc":["000"],"user_id":"3E5EF7F0-F248-11E8-B48F-1D18A9856A87","_id":"8724","has_accepted_license":"1","article_processing_charge":"No","day":"12","scopus_import":"1","date_published":"2020-07-12T00:00:00Z","page":"5416-5425","citation":{"mla":"Konstantinov, Nikola H., et al. “On the Sample Complexity of Adversarial Multi-Source PAC Learning.” Proceedings of the 37th International Conference on Machine Learning, vol. 119, ML Research Press, 2020, pp. 5416–25.","short":"N.H. Konstantinov, E. Frantar, D.-A. Alistarh, C. Lampert, in:, Proceedings of the 37th International Conference on Machine Learning, ML Research Press, 2020, pp. 5416–5425.","chicago":"Konstantinov, Nikola H, Elias Frantar, Dan-Adrian Alistarh, and Christoph Lampert. “On the Sample Complexity of Adversarial Multi-Source PAC Learning.” In Proceedings of the 37th International Conference on Machine Learning, 119:5416–25. ML Research Press, 2020.","ama":"Konstantinov NH, Frantar E, Alistarh D-A, Lampert C. On the sample complexity of adversarial multi-source PAC learning. In: Proceedings of the 37th International Conference on Machine Learning. Vol 119. ML Research Press; 2020:5416-5425.","ista":"Konstantinov NH, Frantar E, Alistarh D-A, Lampert C. 2020. On the sample complexity of adversarial multi-source PAC learning. Proceedings of the 37th International Conference on Machine Learning. ICML: International Conference on Machine Learning vol. 119, 5416–5425.","ieee":"N. H. Konstantinov, E. Frantar, D.-A. Alistarh, and C. Lampert, “On the sample complexity of adversarial multi-source PAC learning,” in Proceedings of the 37th International Conference on Machine Learning, Online, 2020, vol. 119, pp. 5416–5425.","apa":"Konstantinov, N. H., Frantar, E., Alistarh, D.-A., & Lampert, C. (2020). On the sample complexity of adversarial multi-source PAC learning. In Proceedings of the 37th International Conference on Machine Learning (Vol. 119, pp. 5416–5425). Online: ML Research Press."},"publication":"Proceedings of the 37th International Conference on Machine Learning","ec_funded":1,"file_date_updated":"2021-02-15T09:00:01Z","volume":119,"date_updated":"2023-09-07T13:42:08Z","date_created":"2020-11-05T15:25:58Z","related_material":{"record":[{"status":"public","relation":"dissertation_contains","id":"10799"}],"link":[{"url":"http://proceedings.mlr.press/v119/konstantinov20a/konstantinov20a-supp.pdf","relation":"supplementary_material"}]},"author":[{"full_name":"Konstantinov, Nikola H","id":"4B9D76E4-F248-11E8-B48F-1D18A9856A87","last_name":"Konstantinov","first_name":"Nikola H"},{"full_name":"Frantar, Elias","id":"09a8f98d-ec99-11ea-ae11-c063a7b7fe5f","last_name":"Frantar","first_name":"Elias"},{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X"},{"first_name":"Christoph","last_name":"Lampert","id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0001-8622-7887","full_name":"Lampert, Christoph"}],"publisher":"ML Research Press","department":[{"_id":"DaAl"},{"_id":"ChLa"}],"publication_status":"published","year":"2020","acknowledgement":"Dan Alistarh is supported in part by the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 805223 ScaleML). This research was supported by the Scientific Service Units (SSU) of IST Austria through resources provided by Scientific Computing (SciComp).","publication_identifier":{"issn":["2640-3498"]},"month":"07","language":[{"iso":"eng"}],"acknowledged_ssus":[{"_id":"ScienComp"}],"conference":{"end_date":"2020-07-18","location":"Online","start_date":"2020-07-12","name":"ICML: International Conference on Machine Learning"},"project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"quality_controlled":"1","oa":1,"external_id":{"arxiv":["2002.10384"]}},{"file_date_updated":"2020-10-08T08:16:48Z","ec_funded":1,"year":"2020","publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"Springer Nature","author":[{"last_name":"Bhatia","first_name":"Sumit","full_name":"Bhatia, Sumit"},{"last_name":"Chatterjee","first_name":"Bapi","orcid":"0000-0002-2742-4028","id":"3C41A08A-F248-11E8-B48F-1D18A9856A87","full_name":"Chatterjee, Bapi"},{"full_name":"Nathani, Deepak","last_name":"Nathani","first_name":"Deepak"},{"full_name":"Kaul, Manohar","first_name":"Manohar","last_name":"Kaul"}],"date_updated":"2024-02-22T13:16:06Z","date_created":"2019-12-29T23:00:45Z","volume":881,"month":"01","publication_identifier":{"isbn":["9783030366865"],"eissn":["18609503"],"issn":["1860949X"]},"external_id":{"isi":["000843927300003"]},"oa":1,"quality_controlled":"1","isi":1,"project":[{"call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships","_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411"}],"conference":{"start_date":"2019-12-10","location":"Lisbon, Portugal","end_date":"2019-12-12","name":"COMPLEX: International Conference on Complex Networks and their Applications"},"doi":"10.1007/978-3-030-36687-2_3","language":[{"iso":"eng"}],"type":"conference","alternative_title":["SCI"],"abstract":[{"lang":"eng","text":"Persistent homology is a powerful tool in Topological Data Analysis (TDA) to capture the topological properties of data succinctly at different spatial resolutions. For graphical data, the shape, and structure of the neighborhood of individual data items (nodes) are an essential means of characterizing their properties. We propose the use of persistent homology methods to capture structural and topological properties of graphs and use it to address the problem of link prediction. We achieve encouraging results on nine different real-world datasets that attest to the potential of persistent homology-based methods for network analysis."}],"user_id":"3E5EF7F0-F248-11E8-B48F-1D18A9856A87","_id":"7213","title":"A persistent homology perspective to the link prediction problem","ddc":["004"],"status":"public","intvolume":" 881","file":[{"creator":"bchatter","file_size":310598,"content_type":"application/pdf","file_name":"main.pdf","access_level":"open_access","date_updated":"2020-10-08T08:16:48Z","date_created":"2020-10-08T08:16:48Z","success":1,"checksum":"8951f094c8c7dae9ff8db885199bc296","file_id":"8625","relation":"main_file"}],"oa_version":"Submitted Version","scopus_import":"1","day":"01","has_accepted_license":"1","article_processing_charge":"No","publication":"Complex Networks and their applications VIII","citation":{"short":"S. Bhatia, B. Chatterjee, D. Nathani, M. Kaul, in:, Complex Networks and Their Applications VIII, Springer Nature, 2020, pp. 27–39.","mla":"Bhatia, Sumit, et al. “A Persistent Homology Perspective to the Link Prediction Problem.” Complex Networks and Their Applications VIII, vol. 881, Springer Nature, 2020, pp. 27–39, doi:10.1007/978-3-030-36687-2_3.","chicago":"Bhatia, Sumit, Bapi Chatterjee, Deepak Nathani, and Manohar Kaul. “A Persistent Homology Perspective to the Link Prediction Problem.” In Complex Networks and Their Applications VIII, 881:27–39. Springer Nature, 2020. https://doi.org/10.1007/978-3-030-36687-2_3.","ama":"Bhatia S, Chatterjee B, Nathani D, Kaul M. A persistent homology perspective to the link prediction problem. In: Complex Networks and Their Applications VIII. Vol 881. Springer Nature; 2020:27-39. doi:10.1007/978-3-030-36687-2_3","ieee":"S. Bhatia, B. Chatterjee, D. Nathani, and M. Kaul, “A persistent homology perspective to the link prediction problem,” in Complex Networks and their applications VIII, Lisbon, Portugal, 2020, vol. 881, pp. 27–39.","apa":"Bhatia, S., Chatterjee, B., Nathani, D., & Kaul, M. (2020). A persistent homology perspective to the link prediction problem. In Complex Networks and their applications VIII (Vol. 881, pp. 27–39). Lisbon, Portugal: Springer Nature. https://doi.org/10.1007/978-3-030-36687-2_3","ista":"Bhatia S, Chatterjee B, Nathani D, Kaul M. 2020. A persistent homology perspective to the link prediction problem. Complex Networks and their applications VIII. COMPLEX: International Conference on Complex Networks and their Applications, SCI, vol. 881, 27–39."},"page":"27-39","date_published":"2020-01-01T00:00:00Z"},{"abstract":[{"lang":"eng","text":"The Massively Parallel Computation (MPC) model is an emerging model which distills core aspects of distributed and parallel computation. It has been developed as a tool to solve (typically graph) problems in systems where the input is distributed over many machines with limited space.\r\n\t\r\nRecent work has focused on the regime in which machines have sublinear (in $n$, the number of nodes in the input graph) space, with randomized algorithms presented for fundamental graph problems of Maximal Matching and Maximal Independent Set. However, there have been no prior corresponding deterministic algorithms.\r\n\t\r\n\tA major challenge underlying the sublinear space setting is that the local space of each machine might be too small to store all the edges incident to a single node. This poses a considerable obstacle compared to the classical models in which each node is assumed to know and have easy access to its incident edges. To overcome this barrier we introduce a new graph sparsification technique that deterministically computes a low-degree subgraph with additional desired properties. The degree of the nodes in this subgraph is small in the sense that the edges of each node can be now stored on a single machine. This low-degree subgraph also has the property that solving the problem on this subgraph provides \\emph{significant} global progress, i.e., progress towards solving the problem for the original input graph.\r\n\t\r\nUsing this framework to derandomize the well-known randomized algorithm of Luby [SICOMP'86], we obtain $O(\\log \\Delta+\\log\\log n)$-round deterministic MPC algorithms for solving the fundamental problems of Maximal Matching and Maximal Independent Set with $O(n^{\\epsilon})$ space on each machine for any constant $\\epsilon > 0$. Based on the recent work of Ghaffari et al. [FOCS'18], this additive $O(\\log\\log n)$ factor is conditionally essential. These algorithms can also be shown to run in $O(\\log \\Delta)$ rounds in the closely related model of CONGESTED CLIQUE, improving upon the state-of-the-art bound of $O(\\log^2 \\Delta)$ rounds by Censor-Hillel et al. [DISC'17]."}],"issue":"7","type":"conference","oa_version":"Preprint","status":"public","title":"Graph sparsification for derandomizing massively parallel computation with low space","_id":"7802","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","day":"01","article_processing_charge":"No","scopus_import":"1","date_published":"2020-07-01T00:00:00Z","page":"175-185","publication":"Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2020)","citation":{"mla":"Czumaj, Artur, et al. “Graph Sparsification for Derandomizing Massively Parallel Computation with Low Space.” Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2020), no. 7, Association for Computing Machinery, 2020, pp. 175–85, doi:10.1145/3350755.3400282.","short":"A. Czumaj, P. Davies, M. Parter, in:, Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2020), Association for Computing Machinery, 2020, pp. 175–185.","chicago":"Czumaj, Artur, Peter Davies, and Merav Parter. “Graph Sparsification for Derandomizing Massively Parallel Computation with Low Space.” In Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2020), 175–85. Association for Computing Machinery, 2020. https://doi.org/10.1145/3350755.3400282.","ama":"Czumaj A, Davies P, Parter M. Graph sparsification for derandomizing massively parallel computation with low space. In: Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2020). Association for Computing Machinery; 2020:175-185. doi:10.1145/3350755.3400282","ista":"Czumaj A, Davies P, Parter M. 2020. Graph sparsification for derandomizing massively parallel computation with low space. Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2020). SPAA: Symposium on Parallelism in Algorithms and Architectures, 175–185.","apa":"Czumaj, A., Davies, P., & Parter, M. (2020). Graph sparsification for derandomizing massively parallel computation with low space. In Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2020) (pp. 175–185). Virtual Event, United States: Association for Computing Machinery. https://doi.org/10.1145/3350755.3400282","ieee":"A. Czumaj, P. Davies, and M. Parter, “Graph sparsification for derandomizing massively parallel computation with low space,” in Proceedings of the 32nd ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2020), Virtual Event, United States, 2020, no. 7, pp. 175–185."},"ec_funded":1,"date_updated":"2024-02-28T12:53:09Z","date_created":"2020-05-06T08:53:34Z","author":[{"orcid":"0000-0002-5646-9524","first_name":"Artur","last_name":"Czumaj","full_name":"Czumaj, Artur"},{"first_name":"Peter","last_name":"Davies","id":"11396234-BB50-11E9-B24C-90FCE5697425","orcid":"0000-0002-5646-9524","full_name":"Davies, Peter"},{"full_name":"Parter, Merav","last_name":"Parter","first_name":"Merav"}],"related_material":{"record":[{"relation":"later_version","status":"public","id":"9541"}]},"publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"Association for Computing Machinery","year":"2020","month":"07","language":[{"iso":"eng"}],"conference":{"name":"SPAA: Symposium on Parallelism in Algorithms and Architectures","location":"Virtual Event, United States","start_date":"2020-07-15","end_date":"2020-07-17"},"doi":"10.1145/3350755.3400282","quality_controlled":"1","isi":1,"project":[{"call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships","grant_number":"754411","_id":"260C2330-B435-11E9-9278-68D0E5697425"}],"external_id":{"arxiv":["1912.05390"],"isi":["000744436200015"]},"main_file_link":[{"url":"https://arxiv.org/abs/1912.05390","open_access":"1"}],"oa":1},{"publication_identifier":{"isbn":["9781450368186"]},"month":"02","language":[{"iso":"eng"}],"doi":"10.1145/3332466.3374542","conference":{"name":"PPOPP: Principles and Practice of Parallel Programming","location":"San Diego, CA, United States","start_date":"2020-02-22","end_date":"2020-02-26"},"project":[{"name":"Elastic Coordination for Scalable Machine Learning","call_identifier":"H2020","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"quality_controlled":"1","isi":1,"external_id":{"isi":["000564476500020"]},"oa":1,"main_file_link":[{"url":"https://doi.org/10.1145/3332466.3374542","open_access":"1"}],"ec_funded":1,"date_created":"2020-04-05T22:00:49Z","date_updated":"2024-02-28T12:55:14Z","author":[{"last_name":"Brown","first_name":"Trevor A","id":"3569F0A0-F248-11E8-B48F-1D18A9856A87","full_name":"Brown, Trevor A"},{"full_name":"Prokopec, Aleksandar","first_name":"Aleksandar","last_name":"Prokopec"},{"full_name":"Alistarh, Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh"}],"publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"publication_status":"published","acknowledgement":"This project has received funding from the European Research Council (ERC) under the European Union Horizon 2020 research and innovation program, grant agreement No 805223, ERC Starting Grant ScaleML. We acknowledge the support of the Natural Sciences and\r\nEngineering Research Council of Canada (NSERC). ","year":"2020","article_processing_charge":"No","day":"19","scopus_import":"1","date_published":"2020-02-19T00:00:00Z","page":"276-291","citation":{"chicago":"Brown, Trevor A, Aleksandar Prokopec, and Dan-Adrian Alistarh. “Non-Blocking Interpolation Search Trees with Doubly-Logarithmic Running Time.” In Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, 276–91. Association for Computing Machinery, 2020. https://doi.org/10.1145/3332466.3374542.","mla":"Brown, Trevor A., et al. “Non-Blocking Interpolation Search Trees with Doubly-Logarithmic Running Time.” Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2020, pp. 276–91, doi:10.1145/3332466.3374542.","short":"T.A. Brown, A. Prokopec, D.-A. Alistarh, in:, Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, Association for Computing Machinery, 2020, pp. 276–291.","ista":"Brown TA, Prokopec A, Alistarh D-A. 2020. Non-blocking interpolation search trees with doubly-logarithmic running time. Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. PPOPP: Principles and Practice of Parallel Programming, 276–291.","apa":"Brown, T. A., Prokopec, A., & Alistarh, D.-A. (2020). Non-blocking interpolation search trees with doubly-logarithmic running time. In Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (pp. 276–291). San Diego, CA, United States: Association for Computing Machinery. https://doi.org/10.1145/3332466.3374542","ieee":"T. A. Brown, A. Prokopec, and D.-A. Alistarh, “Non-blocking interpolation search trees with doubly-logarithmic running time,” in Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, San Diego, CA, United States, 2020, pp. 276–291.","ama":"Brown TA, Prokopec A, Alistarh D-A. Non-blocking interpolation search trees with doubly-logarithmic running time. In: Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming. Association for Computing Machinery; 2020:276-291. doi:10.1145/3332466.3374542"},"publication":"Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming","abstract":[{"text":"Balanced search trees typically use key comparisons to guide their operations, and achieve logarithmic running time. By relying on numerical properties of the keys, interpolation search achieves lower search complexity and better performance. Although interpolation-based data structures were investigated in the past, their non-blocking concurrent variants have received very little attention so far.\r\nIn this paper, we propose the first non-blocking implementation of the classic interpolation search tree (IST) data structure. For arbitrary key distributions, the data structure ensures worst-case O(log n + p) amortized time for search, insertion and deletion traversals. When the input key distributions are smooth, lookups run in expected O(log log n + p) time, and insertion and deletion run in expected amortized O(log log n + p) time, where p is a bound on the number of threads. To improve the scalability of concurrent insertion and deletion, we propose a novel parallel rebuilding technique, which should be of independent interest.\r\nWe evaluate whether the theoretical improvements translate to practice by implementing the concurrent interpolation search tree, and benchmarking it on uniform and nonuniform key distributions, for dataset sizes in the millions to billions of keys. Relative to the state-of-the-art concurrent data structures, the concurrent interpolation search tree achieves performance improvements of up to 15% under high update rates, and of up to 50% under moderate update rates. Further, ISTs exhibit up to 2X less cache-misses, and consume 1.2 -- 2.6X less memory compared to the next best alternative on typical dataset sizes. We find that the results are surprisingly robust to distributional skew, which suggests that our data structure can be a promising alternative to classic concurrent search structures.","lang":"eng"}],"type":"conference","oa_version":"Published Version","title":"Non-blocking interpolation search trees with doubly-logarithmic running time","status":"public","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"7636"},{"scopus_import":"1","day":"06","month":"07","publication_identifier":{"isbn":["9781450369350"]},"article_processing_charge":"No","publication":"Annual ACM Symposium on Parallelism in Algorithms and Architectures","external_id":{"isi":["000744436200004"]},"citation":{"ama":"Alistarh D-A, Brown TA, Singhal N. Memory tagging: Minimalist synchronization for scalable concurrent data structures. In: Annual ACM Symposium on Parallelism in Algorithms and Architectures. Association for Computing Machinery; 2020:37-49. doi:10.1145/3350755.3400213","ieee":"D.-A. Alistarh, T. A. Brown, and N. Singhal, “Memory tagging: Minimalist synchronization for scalable concurrent data structures,” in Annual ACM Symposium on Parallelism in Algorithms and Architectures, Virtual Event, United States, 2020, no. 7, pp. 37–49.","apa":"Alistarh, D.-A., Brown, T. A., & Singhal, N. (2020). Memory tagging: Minimalist synchronization for scalable concurrent data structures. In Annual ACM Symposium on Parallelism in Algorithms and Architectures (pp. 37–49). Virtual Event, United States: Association for Computing Machinery. https://doi.org/10.1145/3350755.3400213","ista":"Alistarh D-A, Brown TA, Singhal N. 2020. Memory tagging: Minimalist synchronization for scalable concurrent data structures. Annual ACM Symposium on Parallelism in Algorithms and Architectures. SPAA: Symposium on Parallelism in Algorithms and Architectures, 37–49.","short":"D.-A. Alistarh, T.A. Brown, N. Singhal, in:, Annual ACM Symposium on Parallelism in Algorithms and Architectures, Association for Computing Machinery, 2020, pp. 37–49.","mla":"Alistarh, Dan-Adrian, et al. “Memory Tagging: Minimalist Synchronization for Scalable Concurrent Data Structures.” Annual ACM Symposium on Parallelism in Algorithms and Architectures, no. 7, Association for Computing Machinery, 2020, pp. 37–49, doi:10.1145/3350755.3400213.","chicago":"Alistarh, Dan-Adrian, Trevor A Brown, and Nandini Singhal. “Memory Tagging: Minimalist Synchronization for Scalable Concurrent Data Structures.” In Annual ACM Symposium on Parallelism in Algorithms and Architectures, 37–49. Association for Computing Machinery, 2020. https://doi.org/10.1145/3350755.3400213."},"isi":1,"quality_controlled":"1","page":"37-49","conference":{"name":"SPAA: Symposium on Parallelism in Algorithms and Architectures","end_date":"2020-07-17","start_date":"2020-07-15","location":"Virtual Event, United States"},"date_published":"2020-07-06T00:00:00Z","doi":"10.1145/3350755.3400213","language":[{"iso":"eng"}],"type":"conference","abstract":[{"lang":"eng","text":"There has been a significant amount of research on hardware and software support for efficient concurrent data structures; yet, the question of how to build correct, simple, and scalable data structures has not yet been definitively settled. In this paper, we revisit this question from a minimalist perspective, and ask: what is the smallest amount of synchronization required for correct and efficient concurrent search data structures, and how could this minimal synchronization support be provided in hardware?\r\n\r\nTo address these questions, we introduce memory tagging, a simple hardware mechanism which enables the programmer to \"tag\" a dynamic set of memory locations, at cache-line granularity, and later validate whether the memory has been concurrently modified, with the possibility of updating one of the underlying locations atomically if validation succeeds. We provide several examples showing that this mechanism can enable fast and arguably simple concurrent data structure designs, such as lists, binary search trees, balanced search trees, range queries, and Software Transactional Memory (STM) implementations. We provide an implementation of memory tags in the Graphite multi-core simulator, showing that the mechanism can be implemented entirely at the level of L1 cache, and that it can enable non-trivial speedups versus existing implementations of the above data structures."}],"issue":"7","_id":"8191","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","year":"2020","publication_status":"published","status":"public","title":"Memory tagging: Minimalist synchronization for scalable concurrent data structures","department":[{"_id":"DaAl"}],"publisher":"Association for Computing Machinery","author":[{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"},{"full_name":"Brown, Trevor A","first_name":"Trevor A","last_name":"Brown","id":"3569F0A0-F248-11E8-B48F-1D18A9856A87"},{"full_name":"Singhal, Nandini","last_name":"Singhal","first_name":"Nandini"}],"date_created":"2020-08-02T22:00:58Z","date_updated":"2024-02-28T12:56:32Z","oa_version":"None"},{"day":"19","month":"02","article_processing_charge":"No","publication_identifier":{"isbn":["9781450368186"]},"scopus_import":"1","conference":{"end_date":"2020-02-26","start_date":"2020-02-22","location":"San Diego, CA, United States","name":"PPOPP: Principles and Practice of Parallel Programming"},"doi":"10.1145/3332466.3374503","date_published":"2020-02-19T00:00:00Z","language":[{"iso":"eng"}],"publication":"Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP","citation":{"mla":"Koval, Nikita, et al. “Testing Concurrency on the JVM with Lincheck.” Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP, Association for Computing Machinery, 2020, pp. 423–24, doi:10.1145/3332466.3374503.","short":"N. Koval, M. Sokolova, A. Fedorov, D.-A. Alistarh, D. Tsitelov, in:, Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP, Association for Computing Machinery, 2020, pp. 423–424.","chicago":"Koval, Nikita, Mariia Sokolova, Alexander Fedorov, Dan-Adrian Alistarh, and Dmitry Tsitelov. “Testing Concurrency on the JVM with Lincheck.” In Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP, 423–24. Association for Computing Machinery, 2020. https://doi.org/10.1145/3332466.3374503.","ama":"Koval N, Sokolova M, Fedorov A, Alistarh D-A, Tsitelov D. Testing concurrency on the JVM with Lincheck. In: Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP. Association for Computing Machinery; 2020:423-424. doi:10.1145/3332466.3374503","ista":"Koval N, Sokolova M, Fedorov A, Alistarh D-A, Tsitelov D. 2020. Testing concurrency on the JVM with Lincheck. Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP. PPOPP: Principles and Practice of Parallel Programming, 423–424.","apa":"Koval, N., Sokolova, M., Fedorov, A., Alistarh, D.-A., & Tsitelov, D. (2020). Testing concurrency on the JVM with Lincheck. In Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP (pp. 423–424). San Diego, CA, United States: Association for Computing Machinery. https://doi.org/10.1145/3332466.3374503","ieee":"N. Koval, M. Sokolova, A. Fedorov, D.-A. Alistarh, and D. Tsitelov, “Testing concurrency on the JVM with Lincheck,” in Proceedings of the ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP, San Diego, CA, United States, 2020, pp. 423–424."},"quality_controlled":"1","page":"423-424","abstract":[{"text":"Concurrent programming can be notoriously complex and error-prone. Programming bugs can arise from a variety of sources, such as operation re-reordering, or incomplete understanding of the memory model. A variety of formal and model checking methods have been developed to address this fundamental difficulty. While technically interesting, existing academic methods are still hard to apply to the large codebases typical of industrial deployments, which limits their practical impact.","lang":"eng"}],"type":"conference","author":[{"id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87","last_name":"Koval","first_name":"Nikita","full_name":"Koval, Nikita"},{"full_name":"Sokolova, Mariia","id":"26217AE4-77FF-11EA-8101-AD24D49E41F4","first_name":"Mariia","last_name":"Sokolova"},{"first_name":"Alexander","last_name":"Fedorov","full_name":"Fedorov, Alexander"},{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"},{"full_name":"Tsitelov, Dmitry","last_name":"Tsitelov","first_name":"Dmitry"}],"date_updated":"2024-02-28T12:53:46Z","date_created":"2020-04-05T22:00:48Z","oa_version":"None","year":"2020","_id":"7635","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","publication_status":"published","status":"public","title":"Testing concurrency on the JVM with Lincheck","department":[{"_id":"DaAl"}],"publisher":"Association for Computing Machinery"},{"scopus_import":"1","publication_identifier":{"isbn":["9781450375825"]},"article_processing_charge":"No","month":"07","day":"31","page":"54-56","quality_controlled":"1","citation":{"ista":"Alistarh D-A, Aspnes J, Ellen F, Gelashvili R, Zhu L. 2020. Brief Announcement: Why Extension-Based Proofs Fail. Proceedings of the 39th Symposium on Principles of Distributed Computing. PODC: Principles of Distributed Computing, 54–56.","ieee":"D.-A. Alistarh, J. Aspnes, F. Ellen, R. Gelashvili, and L. Zhu, “Brief Announcement: Why Extension-Based Proofs Fail,” in Proceedings of the 39th Symposium on Principles of Distributed Computing, Virtual, Italy, 2020, pp. 54–56.","apa":"Alistarh, D.-A., Aspnes, J., Ellen, F., Gelashvili, R., & Zhu, L. (2020). Brief Announcement: Why Extension-Based Proofs Fail. In Proceedings of the 39th Symposium on Principles of Distributed Computing (pp. 54–56). Virtual, Italy: Association for Computing Machinery. https://doi.org/10.1145/3382734.3405743","ama":"Alistarh D-A, Aspnes J, Ellen F, Gelashvili R, Zhu L. Brief Announcement: Why Extension-Based Proofs Fail. In: Proceedings of the 39th Symposium on Principles of Distributed Computing. Association for Computing Machinery; 2020:54-56. doi:10.1145/3382734.3405743","chicago":"Alistarh, Dan-Adrian, James Aspnes, Faith Ellen, Rati Gelashvili, and Leqi Zhu. “Brief Announcement: Why Extension-Based Proofs Fail.” In Proceedings of the 39th Symposium on Principles of Distributed Computing, 54–56. Association for Computing Machinery, 2020. https://doi.org/10.1145/3382734.3405743.","mla":"Alistarh, Dan-Adrian, et al. “Brief Announcement: Why Extension-Based Proofs Fail.” Proceedings of the 39th Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2020, pp. 54–56, doi:10.1145/3382734.3405743.","short":"D.-A. Alistarh, J. Aspnes, F. Ellen, R. Gelashvili, L. Zhu, in:, Proceedings of the 39th Symposium on Principles of Distributed Computing, Association for Computing Machinery, 2020, pp. 54–56."},"publication":"Proceedings of the 39th Symposium on Principles of Distributed Computing","language":[{"iso":"eng"}],"doi":"10.1145/3382734.3405743","date_published":"2020-07-31T00:00:00Z","conference":{"name":"PODC: Principles of Distributed Computing","location":"Virtual, Italy","start_date":"2020-08-03","end_date":"2020-08-07"},"type":"conference","abstract":[{"text":"We introduce extension-based proofs, a class of impossibility proofs that includes valency arguments. They are modelled as an interaction between a prover and a protocol. Using proofs based on combinatorial topology, it has been shown that it is impossible to deterministically solve k-set agreement among n > k ≥ 2 processes in a wait-free manner. However, it was unknown whether proofs based on simpler techniques were possible. We explain why this impossibility result cannot be obtained by an extension-based proof and, hence, extension-based proofs are limited in power.","lang":"eng"}],"publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"publication_status":"published","title":"Brief Announcement: Why Extension-Based Proofs Fail","status":"public","year":"2020","_id":"8383","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","oa_version":"None","date_updated":"2024-02-28T12:54:19Z","date_created":"2020-09-13T22:01:18Z","author":[{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian"},{"full_name":"Aspnes, James","first_name":"James","last_name":"Aspnes"},{"first_name":"Faith","last_name":"Ellen","full_name":"Ellen, Faith"},{"first_name":"Rati","last_name":"Gelashvili","full_name":"Gelashvili, Rati"},{"full_name":"Zhu, Leqi","first_name":"Leqi","last_name":"Zhu"}]},{"intvolume":" 179","status":"public","title":"Brief announcement: Efficient load-balancing through distributed token dropping","ddc":["000"],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"15074","file":[{"relation":"main_file","file_id":"15075","date_created":"2024-03-05T07:08:27Z","date_updated":"2024-03-05T07:08:27Z","checksum":"23e2d9321aef53092dc1e24a8ab82d72","success":1,"file_name":"2020_LIPIcs_Brandt.pdf","access_level":"open_access","file_size":303529,"content_type":"application/pdf","creator":"dernst"}],"oa_version":"Published Version","alternative_title":["LIPIcs"],"type":"conference","abstract":[{"text":"We introduce a new graph problem, the token dropping game, and we show how to solve it efficiently in a distributed setting. We use the token dropping game as a tool to design an efficient distributed algorithm for the stable orientation problem, which is a special case of the more general locally optimal semi-matching problem. The prior work by Czygrinow et al. (DISC 2012) finds a locally optimal semi-matching in O(Δ⁵) rounds in graphs of maximum degree Δ, which directly implies an algorithm with the same runtime for stable orientations. We improve the runtime to O(Δ⁴) for stable orientations and prove a lower bound of Ω(Δ) rounds.","lang":"eng"}],"citation":{"ama":"Brandt S, Keller B, Rybicki J, Suomela J, Uitto J. Brief announcement: Efficient load-balancing through distributed token dropping. In: 34th International Symposium on Distributed Computing. Vol 179. Schloss Dagstuhl - Leibniz-Zentrum für Informatik; 2020. doi:10.4230/LIPIcs.DISC.2020.40","ieee":"S. Brandt, B. Keller, J. Rybicki, J. Suomela, and J. Uitto, “Brief announcement: Efficient load-balancing through distributed token dropping,” in 34th International Symposium on Distributed Computing, Virtual, 2020, vol. 179.","apa":"Brandt, S., Keller, B., Rybicki, J., Suomela, J., & Uitto, J. (2020). Brief announcement: Efficient load-balancing through distributed token dropping. In 34th International Symposium on Distributed Computing (Vol. 179). Virtual: Schloss Dagstuhl - Leibniz-Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.DISC.2020.40","ista":"Brandt S, Keller B, Rybicki J, Suomela J, Uitto J. 2020. Brief announcement: Efficient load-balancing through distributed token dropping. 34th International Symposium on Distributed Computing. DISC: Symposium on Distributed Computing, LIPIcs, vol. 179, 40.","short":"S. Brandt, B. Keller, J. Rybicki, J. Suomela, J. Uitto, in:, 34th International Symposium on Distributed Computing, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020.","mla":"Brandt, Sebastian, et al. “Brief Announcement: Efficient Load-Balancing through Distributed Token Dropping.” 34th International Symposium on Distributed Computing, vol. 179, 40, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020, doi:10.4230/LIPIcs.DISC.2020.40.","chicago":"Brandt, Sebastian, Barbara Keller, Joel Rybicki, Jukka Suomela, and Jara Uitto. “Brief Announcement: Efficient Load-Balancing through Distributed Token Dropping.” In 34th International Symposium on Distributed Computing, Vol. 179. Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020. https://doi.org/10.4230/LIPIcs.DISC.2020.40."},"publication":"34th International Symposium on Distributed Computing","date_published":"2020-10-07T00:00:00Z","scopus_import":"1","article_processing_charge":"No","has_accepted_license":"1","day":"07","department":[{"_id":"DaAl"}],"publisher":"Schloss Dagstuhl - Leibniz-Zentrum für Informatik","publication_status":"published","year":"2020","volume":179,"date_updated":"2024-03-05T07:13:13Z","date_created":"2024-03-05T07:09:12Z","related_material":{"record":[{"status":"public","relation":"later_version","id":"9678"}]},"author":[{"first_name":"Sebastian","last_name":"Brandt","full_name":"Brandt, Sebastian"},{"full_name":"Keller, Barbara","last_name":"Keller","first_name":"Barbara"},{"full_name":"Rybicki, Joel","first_name":"Joel","last_name":"Rybicki","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0002-6432-6646"},{"full_name":"Suomela, Jukka","last_name":"Suomela","first_name":"Jukka"},{"first_name":"Jara","last_name":"Uitto","full_name":"Uitto, Jara"}],"article_number":"40","file_date_updated":"2024-03-05T07:08:27Z","quality_controlled":"1","external_id":{"arxiv":["2005.07761"]},"tmp":{"legal_code_url":"https://creativecommons.org/licenses/by/3.0/legalcode","name":"Creative Commons Attribution 3.0 Unported (CC BY 3.0)","short":"CC BY (3.0)","image":"/images/cc_by.png"},"oa":1,"language":[{"iso":"eng"}],"doi":"10.4230/LIPIcs.DISC.2020.40","conference":{"name":"DISC: Symposium on Distributed Computing","start_date":"2020-10-12","location":"Virtual","end_date":"2020-10-16"},"month":"10"},{"abstract":[{"text":"We consider the following dynamic load-balancing process: given an underlying graph G with n nodes, in each step t≥ 0, one unit of load is created, and placed at a randomly chosen graph node. In the same step, the chosen node picks a random neighbor, and the two nodes balance their loads by averaging them. We are interested in the expected gap between the minimum and maximum loads at nodes as the process progresses, and its dependence on n and on the graph structure. Variants of the above graphical balanced allocation process have been studied previously by Peres, Talwar, and Wieder [Peres et al., 2015], and by Sauerwald and Sun [Sauerwald and Sun, 2015]. These authors left as open the question of characterizing the gap in the case of cycle graphs in the dynamic case, where weights are created during the algorithm’s execution. For this case, the only known upper bound is of 𝒪(n log n), following from a majorization argument due to [Peres et al., 2015], which analyzes a related graphical allocation process. In this paper, we provide an upper bound of 𝒪 (√n log n) on the expected gap of the above process for cycles of length n. We introduce a new potential analysis technique, which enables us to bound the difference in load between k-hop neighbors on the cycle, for any k ≤ n/2. We complement this with a \"gap covering\" argument, which bounds the maximum value of the gap by bounding its value across all possible subsets of a certain structure, and recursively bounding the gaps within each subset. We provide analytical and experimental evidence that our upper bound on the gap is tight up to a logarithmic factor.","lang":"eng"}],"type":"conference","alternative_title":["LIPIcs"],"oa_version":"Published Version","file":[{"date_updated":"2024-03-05T07:25:15Z","date_created":"2024-03-05T07:25:15Z","checksum":"e5eb16199f4ccfd77a321977eb3f026f","success":1,"relation":"main_file","file_id":"15078","content_type":"application/pdf","file_size":782987,"creator":"dernst","file_name":"2020_LIPIcs_Alistarh.pdf","access_level":"open_access"}],"_id":"15077","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","intvolume":" 168","ddc":["000"],"title":"Dynamic averaging load balancing on cycles","status":"public","has_accepted_license":"1","article_processing_charge":"No","day":"29","scopus_import":"1","date_published":"2020-06-29T00:00:00Z","citation":{"chicago":"Alistarh, Dan-Adrian, Giorgi Nadiradze, and Amirmojtaba Sabour. “Dynamic Averaging Load Balancing on Cycles.” In 47th International Colloquium on Automata, Languages, and Programming, Vol. 168. Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020. https://doi.org/10.4230/LIPIcs.ICALP.2020.7.","mla":"Alistarh, Dan-Adrian, et al. “Dynamic Averaging Load Balancing on Cycles.” 47th International Colloquium on Automata, Languages, and Programming, vol. 168, 7, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020, doi:10.4230/LIPIcs.ICALP.2020.7.","short":"D.-A. Alistarh, G. Nadiradze, A. Sabour, in:, 47th International Colloquium on Automata, Languages, and Programming, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2020.","ista":"Alistarh D-A, Nadiradze G, Sabour A. 2020. Dynamic averaging load balancing on cycles. 47th International Colloquium on Automata, Languages, and Programming. ICALP: International Colloquium on Automata, Languages, and Programming, LIPIcs, vol. 168, 7.","ieee":"D.-A. Alistarh, G. Nadiradze, and A. Sabour, “Dynamic averaging load balancing on cycles,” in 47th International Colloquium on Automata, Languages, and Programming, Saarbrücken, Germany, Virtual, 2020, vol. 168.","apa":"Alistarh, D.-A., Nadiradze, G., & Sabour, A. (2020). Dynamic averaging load balancing on cycles. In 47th International Colloquium on Automata, Languages, and Programming (Vol. 168). Saarbrücken, Germany, Virtual: Schloss Dagstuhl - Leibniz-Zentrum für Informatik. https://doi.org/10.4230/LIPIcs.ICALP.2020.7","ama":"Alistarh D-A, Nadiradze G, Sabour A. Dynamic averaging load balancing on cycles. In: 47th International Colloquium on Automata, Languages, and Programming. Vol 168. Schloss Dagstuhl - Leibniz-Zentrum für Informatik; 2020. doi:10.4230/LIPIcs.ICALP.2020.7"},"publication":"47th International Colloquium on Automata, Languages, and Programming","ec_funded":1,"file_date_updated":"2024-03-05T07:25:15Z","article_number":"7","related_material":{"record":[{"id":"8286","status":"public","relation":"later_version"}]},"author":[{"full_name":"Alistarh, Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh"},{"last_name":"Nadiradze","first_name":"Giorgi","orcid":"0000-0001-5634-0731","id":"3279A00C-F248-11E8-B48F-1D18A9856A87","full_name":"Nadiradze, Giorgi"},{"full_name":"Sabour, Amirmojtaba","first_name":"Amirmojtaba","last_name":"Sabour","id":"bcc145fd-e77f-11ea-ae8b-80d661dbff67"}],"volume":168,"date_updated":"2024-03-05T07:35:53Z","date_created":"2024-03-05T07:25:37Z","acknowledgement":"The authors sincerely thank Thomas Sauerwald and George Giakkoupis for insightful discussions, and Mohsen Ghaffari, Yuval Peres, and Udi Wieder for feedback on earlier\r\nversions of this draft. We also thank the ICALP anonymous reviewers for their very useful comments.\r\nFunding: European Research Council funding award PR1042ERC01","year":"2020","department":[{"_id":"DaAl"}],"publisher":"Schloss Dagstuhl - Leibniz-Zentrum für Informatik","publication_status":"published","month":"06","doi":"10.4230/LIPIcs.ICALP.2020.7","conference":{"end_date":"2020-07-11","location":"Saarbrücken, Germany, Virtual","start_date":"2020-07-08","name":"ICALP: International Colloquium on Automata, Languages, and Programming"},"language":[{"iso":"eng"}],"tmp":{"legal_code_url":"https://creativecommons.org/licenses/by/3.0/legalcode","name":"Creative Commons Attribution 3.0 Unported (CC BY 3.0)","short":"CC BY (3.0)","image":"/images/cc_by.png"},"oa":1,"external_id":{"arxiv":["2003.09297"]},"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"quality_controlled":"1"},{"article_type":"original","citation":{"ama":"Jelínek V, Töpfer M. On grounded L-graphs and their relatives. Electronic Journal of Combinatorics. 2019;26(3). doi:10.37236/8096","ieee":"V. Jelínek and M. Töpfer, “On grounded L-graphs and their relatives,” Electronic Journal of Combinatorics, vol. 26, no. 3. Electronic Journal of Combinatorics, 2019.","apa":"Jelínek, V., & Töpfer, M. (2019). On grounded L-graphs and their relatives. Electronic Journal of Combinatorics. Electronic Journal of Combinatorics. https://doi.org/10.37236/8096","ista":"Jelínek V, Töpfer M. 2019. On grounded L-graphs and their relatives. Electronic Journal of Combinatorics. 26(3), P3.17.","short":"V. Jelínek, M. Töpfer, Electronic Journal of Combinatorics 26 (2019).","mla":"Jelínek, Vít, and Martin Töpfer. “On Grounded L-Graphs and Their Relatives.” Electronic Journal of Combinatorics, vol. 26, no. 3, P3.17, Electronic Journal of Combinatorics, 2019, doi:10.37236/8096.","chicago":"Jelínek, Vít, and Martin Töpfer. “On Grounded L-Graphs and Their Relatives.” Electronic Journal of Combinatorics. Electronic Journal of Combinatorics, 2019. https://doi.org/10.37236/8096."},"publication":"Electronic Journal of Combinatorics","date_published":"2019-07-19T00:00:00Z","scopus_import":"1","article_processing_charge":"No","has_accepted_license":"1","day":"19","intvolume":" 26","ddc":["510"],"status":"public","title":"On grounded L-graphs and their relatives","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"6759","file":[{"file_id":"6764","relation":"main_file","date_updated":"2020-07-14T12:47:39Z","date_created":"2019-08-05T06:46:55Z","checksum":"20fc366fc6683ef0b074a019b73a663a","file_name":"2019_eJourCombinatorics_Jelinek.pdf","access_level":"open_access","creator":"dernst","file_size":533697,"content_type":"application/pdf"}],"oa_version":"Published Version","type":"journal_article","issue":"3","abstract":[{"text":"We consider the graph class Grounded-L corresponding to graphs that admit an intersection representation by L-shaped curves, where additionally the topmost points of each curve are assumed to belong to a common horizontal line. We prove that Grounded-L graphs admit an equivalent characterisation in terms of vertex ordering with forbidden patterns. \r\nWe also compare this class to related intersection classes, such as the grounded segment graphs, the monotone L-graphs (a.k.a. max point-tolerance graphs), or the outer-1-string graphs. We give constructions showing that these classes are all distinct and satisfy only trivial or previously known inclusions.","lang":"eng"}],"project":[{"grant_number":"665385","_id":"2564DBCA-B435-11E9-9278-68D0E5697425","name":"International IST Doctoral Program","call_identifier":"H2020"}],"quality_controlled":"1","tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"external_id":{"arxiv":["1808.04148"]},"oa":1,"language":[{"iso":"eng"}],"doi":"10.37236/8096","publication_identifier":{"eissn":["10778926"]},"month":"07","publisher":"Electronic Journal of Combinatorics","department":[{"_id":"DaAl"}],"publication_status":"published","year":"2019","volume":26,"date_updated":"2022-03-18T12:32:02Z","date_created":"2019-08-04T21:59:20Z","author":[{"first_name":"Vít","last_name":"Jelínek","full_name":"Jelínek, Vít"},{"full_name":"Töpfer, Martin","first_name":"Martin","last_name":"Töpfer","id":"4B865388-F248-11E8-B48F-1D18A9856A87"}],"article_number":"P3.17","ec_funded":1,"file_date_updated":"2020-07-14T12:47:39Z"},{"publication_identifier":{"eisbn":["978-3-95977-126-9"]},"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"external_id":{"arxiv":["1908.02743"]},"project":[{"call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships","_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411"}],"quality_controlled":"1","doi":"10.4230/LIPICS.DISC.2019.29","conference":{"name":"DISC: International Symposium on Distributed Computing","end_date":"2019-10-18","start_date":"2019-10-14","location":"Budapest, Hungary"},"language":[{"iso":"eng"}],"ec_funded":1,"file_date_updated":"2020-07-14T12:47:44Z","year":"2019","publisher":"Schloss Dagstuhl - Leibniz-Zentrum für Informatik","department":[{"_id":"DaAl"}],"publication_status":"published","author":[{"first_name":"Thomas","last_name":"Nowak","full_name":"Nowak, Thomas"},{"full_name":"Rybicki, Joel","first_name":"Joel","last_name":"Rybicki","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0002-6432-6646"}],"volume":146,"date_updated":"2021-01-12T08:09:38Z","date_created":"2019-10-08T12:41:38Z","scopus_import":1,"keyword":["consensus","approximate agreement","Byzantine faults","chordal graphs","lattice agreement"],"has_accepted_license":"1","article_processing_charge":"No","citation":{"ama":"Nowak T, Rybicki J. Byzantine approximate agreement on graphs. In: 33rd International Symposium on Distributed Computing. Vol 146. Schloss Dagstuhl - Leibniz-Zentrum für Informatik; 2019:29:1--29:17. doi:10.4230/LIPICS.DISC.2019.29","ista":"Nowak T, Rybicki J. 2019. Byzantine approximate agreement on graphs. 33rd International Symposium on Distributed Computing. DISC: International Symposium on Distributed Computing, LIPIcs, vol. 146, 29:1--29:17.","apa":"Nowak, T., & Rybicki, J. (2019). Byzantine approximate agreement on graphs. In 33rd International Symposium on Distributed Computing (Vol. 146, p. 29:1--29:17). Budapest, Hungary: Schloss Dagstuhl - Leibniz-Zentrum für Informatik. https://doi.org/10.4230/LIPICS.DISC.2019.29","ieee":"T. Nowak and J. Rybicki, “Byzantine approximate agreement on graphs,” in 33rd International Symposium on Distributed Computing, Budapest, Hungary, 2019, vol. 146, p. 29:1--29:17.","mla":"Nowak, Thomas, and Joel Rybicki. “Byzantine Approximate Agreement on Graphs.” 33rd International Symposium on Distributed Computing, vol. 146, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2019, p. 29:1--29:17, doi:10.4230/LIPICS.DISC.2019.29.","short":"T. Nowak, J. Rybicki, in:, 33rd International Symposium on Distributed Computing, Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2019, p. 29:1--29:17.","chicago":"Nowak, Thomas, and Joel Rybicki. “Byzantine Approximate Agreement on Graphs.” In 33rd International Symposium on Distributed Computing, 146:29:1--29:17. Schloss Dagstuhl - Leibniz-Zentrum für Informatik, 2019. https://doi.org/10.4230/LIPICS.DISC.2019.29."},"publication":"33rd International Symposium on Distributed Computing","page":"29:1--29:17","date_published":"2019-01-01T00:00:00Z","type":"conference","alternative_title":["LIPIcs"],"abstract":[{"lang":"eng","text":"Consider a distributed system with n processors out of which f can be Byzantine faulty. In the\r\napproximate agreement task, each processor i receives an input value xi and has to decide on an\r\noutput value yi such that\r\n1. the output values are in the convex hull of the non-faulty processors’ input values,\r\n2. the output values are within distance d of each other.\r\n\r\n\r\nClassically, the values are assumed to be from an m-dimensional Euclidean space, where m ≥ 1.\r\nIn this work, we study the task in a discrete setting, where input values with some structure\r\nexpressible as a graph. Namely, the input values are vertices of a finite graph G and the goal is to\r\noutput vertices that are within distance d of each other in G, but still remain in the graph-induced\r\nconvex hull of the input values. For d = 0, the task reduces to consensus and cannot be solved with\r\na deterministic algorithm in an asynchronous system even with a single crash fault. For any d ≥ 1,\r\nwe show that the task is solvable in asynchronous systems when G is chordal and n > (ω + 1)f,\r\nwhere ω is the clique number of G. In addition, we give the first Byzantine-tolerant algorithm for a\r\nvariant of lattice agreement. For synchronous systems, we show tight resilience bounds for the exact\r\nvariants of these and related tasks over a large class of combinatorial structures."}],"_id":"6931","user_id":"3E5EF7F0-F248-11E8-B48F-1D18A9856A87","intvolume":" 146","status":"public","title":"Byzantine approximate agreement on graphs","ddc":["004"],"file":[{"relation":"main_file","file_id":"6934","checksum":"2d2202f90c6ac991e50876451627c4b5","date_updated":"2020-07-14T12:47:44Z","date_created":"2019-10-08T12:47:19Z","access_level":"open_access","file_name":"LIPIcs-DISC-2019-29.pdf","file_size":639378,"content_type":"application/pdf","creator":"jrybicki"}],"oa_version":"Published Version"},{"type":"conference","abstract":[{"text":"Graph algorithms applied in many applications, including social networks, communication networks, VLSI design, graphics, and several others, require dynamic modifications - addition and removal of vertices and/or edges - in the graph. This paper presents a novel concurrent non-blocking algorithm to implement a dynamic unbounded directed graph in a shared-memory machine. The addition and removal operations of vertices and edges are lock-free. For a finite sized graph, the lookup operations are wait-free. Most significant component of the presented algorithm is the reachability query in a concurrent graph. The reachability queries in our algorithm are obstruction-free and thus impose minimal additional synchronization cost over other operations. We prove that each of the data structure operations are linearizable. We extensively evaluate a sample C/C++ implementation of the algorithm through a number of micro-benchmarks. The experimental results show that the proposed algorithm scales well with the number of threads and on an average provides 5 to 7x performance improvement over a concurrent graph implementation using coarse-grained locking.","lang":"eng"}],"title":"A simple and practical concurrent non-blocking unbounded graph with linearizable reachability queries","status":"public","_id":"5947","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","oa_version":"Preprint","scopus_import":"1","day":"04","article_processing_charge":"No","page":"168-177","publication":"ACM International Conference Proceeding Series","citation":{"ieee":"B. Chatterjee, S. Peri, M. Sa, and N. Singhal, “A simple and practical concurrent non-blocking unbounded graph with linearizable reachability queries,” in ACM International Conference Proceeding Series, Bangalore, India, 2019, pp. 168–177.","apa":"Chatterjee, B., Peri, S., Sa, M., & Singhal, N. (2019). A simple and practical concurrent non-blocking unbounded graph with linearizable reachability queries. In ACM International Conference Proceeding Series (pp. 168–177). Bangalore, India: ACM. https://doi.org/10.1145/3288599.3288617","ista":"Chatterjee B, Peri S, Sa M, Singhal N. 2019. A simple and practical concurrent non-blocking unbounded graph with linearizable reachability queries. ACM International Conference Proceeding Series. ICDCN: Conference on Distributed Computing and Networking, 168–177.","ama":"Chatterjee B, Peri S, Sa M, Singhal N. A simple and practical concurrent non-blocking unbounded graph with linearizable reachability queries. In: ACM International Conference Proceeding Series. ACM; 2019:168-177. doi:10.1145/3288599.3288617","chicago":"Chatterjee, Bapi, Sathya Peri, Muktikanta Sa, and Nandini Singhal. “A Simple and Practical Concurrent Non-Blocking Unbounded Graph with Linearizable Reachability Queries.” In ACM International Conference Proceeding Series, 168–77. ACM, 2019. https://doi.org/10.1145/3288599.3288617.","short":"B. Chatterjee, S. Peri, M. Sa, N. Singhal, in:, ACM International Conference Proceeding Series, ACM, 2019, pp. 168–177.","mla":"Chatterjee, Bapi, et al. “A Simple and Practical Concurrent Non-Blocking Unbounded Graph with Linearizable Reachability Queries.” ACM International Conference Proceeding Series, ACM, 2019, pp. 168–77, doi:10.1145/3288599.3288617."},"date_published":"2019-01-04T00:00:00Z","publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"ACM","year":"2019","date_updated":"2023-08-24T14:41:53Z","date_created":"2019-02-10T22:59:17Z","author":[{"full_name":"Chatterjee, Bapi","first_name":"Bapi","last_name":"Chatterjee","id":"3C41A08A-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0002-2742-4028"},{"full_name":"Peri, Sathya","first_name":"Sathya","last_name":"Peri"},{"last_name":"Sa","first_name":"Muktikanta","full_name":"Sa, Muktikanta"},{"full_name":"Singhal, Nandini","first_name":"Nandini","last_name":"Singhal"}],"month":"01","publication_identifier":{"isbn":["978-1-4503-6094-4 "]},"quality_controlled":"1","isi":1,"external_id":{"isi":["000484491600019"],"arxiv":["1809.00896"]},"oa":1,"main_file_link":[{"url":"https://arxiv.org/abs/1809.00896","open_access":"1"}],"language":[{"iso":"eng"}],"conference":{"end_date":"2019-01-07","start_date":"2019-01-04","location":"Bangalore, India","name":"ICDCN: Conference on Distributed Computing and Networking"},"doi":"10.1145/3288599.3288617"},{"month":"02","day":"01","publication_identifier":{"isbn":["9781450362252"]},"article_processing_charge":"No","quality_controlled":"1","isi":1,"page":"417-418","publication":"Proceedings of the 24th Symposium on Principles and Practice of Parallel Programming","external_id":{"isi":["000587604600044"]},"citation":{"ama":"Koval N, Alistarh D-A, Elizarov R. Lock-Free Channels for Programming via Communicating Sequential Processes. ACM Press; 2019:417-418. doi:10.1145/3293883.3297000","ista":"Koval N, Alistarh D-A, Elizarov R. 2019. Lock-free channels for programming via communicating sequential processes, ACM Press,p.","apa":"Koval, N., Alistarh, D.-A., & Elizarov, R. (2019). Lock-free channels for programming via communicating sequential processes. Proceedings of the 24th Symposium on Principles and Practice of Parallel Programming (pp. 417–418). Washington, NY, United States: ACM Press. https://doi.org/10.1145/3293883.3297000","ieee":"N. Koval, D.-A. Alistarh, and R. Elizarov, Lock-free channels for programming via communicating sequential processes. ACM Press, 2019, pp. 417–418.","mla":"Koval, Nikita, et al. “Lock-Free Channels for Programming via Communicating Sequential Processes.” Proceedings of the 24th Symposium on Principles and Practice of Parallel Programming, ACM Press, 2019, pp. 417–18, doi:10.1145/3293883.3297000.","short":"N. Koval, D.-A. Alistarh, R. Elizarov, Lock-Free Channels for Programming via Communicating Sequential Processes, ACM Press, 2019.","chicago":"Koval, Nikita, Dan-Adrian Alistarh, and Roman Elizarov. Lock-Free Channels for Programming via Communicating Sequential Processes. Proceedings of the 24th Symposium on Principles and Practice of Parallel Programming. ACM Press, 2019. https://doi.org/10.1145/3293883.3297000."},"language":[{"iso":"eng"}],"conference":{"name":"PPoPP: Principles and Practice of Parallel Programming","end_date":"2019-02-20","start_date":"2019-02-16","location":"Washington, NY, United States"},"doi":"10.1145/3293883.3297000","date_published":"2019-02-01T00:00:00Z","type":"conference_poster","abstract":[{"text":"Traditional concurrent programming involves manipulating shared mutable state. Alternatives to this programming style are communicating sequential processes (CSP) [1] and actor [2] models, which share data via explicit communication. Rendezvous channelis the common abstraction for communication between several processes, where senders and receivers perform a rendezvous handshake as a part of their protocol (senders wait for receivers and vice versa). Additionally to this, channels support the select expression. In this work, we present the first efficient lock-free channel algorithm, and compare it against Go [3] and Kotlin [4] baseline implementations.","lang":"eng"}],"status":"public","publication_status":"published","title":"Lock-free channels for programming via communicating sequential processes","publisher":"ACM Press","department":[{"_id":"DaAl"}],"_id":"6485","year":"2019","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","date_updated":"2023-08-25T10:41:20Z","date_created":"2019-05-24T10:09:12Z","oa_version":"None","author":[{"full_name":"Koval, Nikita","id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87","last_name":"Koval","first_name":"Nikita"},{"first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","full_name":"Alistarh, Dan-Adrian"},{"full_name":"Elizarov, Roman","last_name":"Elizarov","first_name":"Roman"}]},{"oa":1,"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"external_id":{"isi":["000486348700001"]},"project":[{"grant_number":"754411","_id":"260C2330-B435-11E9-9278-68D0E5697425","name":"ISTplus - Postdoctoral Fellowships","call_identifier":"H2020"}],"isi":1,"quality_controlled":"1","doi":"10.1111/ecog.04444","language":[{"iso":"eng"}],"publication_identifier":{"eissn":["1600-0587"],"issn":["0906-7590"]},"month":"11","year":"2019","publisher":"Wiley","department":[{"_id":"DaAl"}],"publication_status":"published","author":[{"first_name":"Otso","last_name":"Ovaskainen","full_name":"Ovaskainen, Otso"},{"last_name":"Rybicki","first_name":"Joel","orcid":"0000-0002-6432-6646","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","full_name":"Rybicki, Joel"},{"full_name":"Abrego, Nerea","first_name":"Nerea","last_name":"Abrego"}],"volume":42,"date_created":"2019-10-08T13:01:24Z","date_updated":"2023-08-30T06:57:25Z","ec_funded":1,"file_date_updated":"2020-07-14T12:47:45Z","citation":{"apa":"Ovaskainen, O., Rybicki, J., & Abrego, N. (2019). What can observational data reveal about metacommunity processes? Ecography. Wiley. https://doi.org/10.1111/ecog.04444","ieee":"O. Ovaskainen, J. Rybicki, and N. Abrego, “What can observational data reveal about metacommunity processes?,” Ecography, vol. 42, no. 11. Wiley, pp. 1877–1886, 2019.","ista":"Ovaskainen O, Rybicki J, Abrego N. 2019. What can observational data reveal about metacommunity processes? Ecography. 42(11), 1877–1886.","ama":"Ovaskainen O, Rybicki J, Abrego N. What can observational data reveal about metacommunity processes? Ecography. 2019;42(11):1877-1886. doi:10.1111/ecog.04444","chicago":"Ovaskainen, Otso, Joel Rybicki, and Nerea Abrego. “What Can Observational Data Reveal about Metacommunity Processes?” Ecography. Wiley, 2019. https://doi.org/10.1111/ecog.04444.","short":"O. Ovaskainen, J. Rybicki, N. Abrego, Ecography 42 (2019) 1877–1886.","mla":"Ovaskainen, Otso, et al. “What Can Observational Data Reveal about Metacommunity Processes?” Ecography, vol. 42, no. 11, Wiley, 2019, pp. 1877–86, doi:10.1111/ecog.04444."},"publication":"Ecography","page":"1877-1886","article_type":"original","date_published":"2019-11-01T00:00:00Z","scopus_import":"1","article_processing_charge":"No","has_accepted_license":"1","day":"01","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","_id":"6936","intvolume":" 42","status":"public","title":"What can observational data reveal about metacommunity processes?","ddc":["577"],"oa_version":"Published Version","file":[{"access_level":"open_access","file_name":"ecog.04444.pdf","content_type":"application/pdf","file_size":1682718,"creator":"jrybicki","relation":"main_file","file_id":"6937","checksum":"6c9fbbd5ea8ce10ae93e55ad560a7bf9","date_updated":"2020-07-14T12:47:45Z","date_created":"2019-10-08T13:07:44Z"}],"type":"journal_article","issue":"11","abstract":[{"text":"A key challenge for community ecology is to understand to what extent observational data can be used to infer the underlying community assembly processes. As different processes can lead to similar or even identical patterns, statistical analyses of non‐manipulative observational data never yield undisputable causal inference on the underlying processes. Still, most empirical studies in community ecology are based on observational data, and hence understanding under which circumstances such data can shed light on assembly processes is a central concern for community ecologists. We simulated a spatial agent‐based model that generates variation in metacommunity dynamics across multiple axes, including the four classic metacommunity paradigms as special cases. We further simulated a virtual ecologist who analysed snapshot data sampled from the simulations using eighteen output metrics derived from beta‐diversity and habitat variation indices, variation partitioning and joint species distribution modelling. Our results indicated two main axes of variation in the output metrics. The first axis of variation described whether the landscape has patchy or continuous variation, and thus was essentially independent of the properties of the species community. The second axis of variation related to the level of predictability of the metacommunity. The most predictable communities were niche‐based metacommunities inhabiting static landscapes with marked environmental heterogeneity, such as metacommunities following the species sorting paradigm or the mass effects paradigm. The most unpredictable communities were neutral‐based metacommunities inhabiting dynamics landscapes with little spatial heterogeneity, such as metacommunities following the neutral or patch sorting paradigms. The output metrics from joint species distribution modelling yielded generally the highest resolution to disentangle among the simulated scenarios. Yet, the different types of statistical approaches utilized in this study carried complementary information, and thus our results suggest that the most comprehensive evaluation of metacommunity structure can be obtained by combining them.\r\n","lang":"eng"}]},{"article_type":"original","publication":"Journal of the ACM","citation":{"ama":"Lenzen C, Rybicki J. Self-stabilising Byzantine clock synchronisation is almost as easy as consensus. Journal of the ACM. 2019;66(5). doi:10.1145/3339471","ista":"Lenzen C, Rybicki J. 2019. Self-stabilising Byzantine clock synchronisation is almost as easy as consensus. Journal of the ACM. 66(5), 32.","ieee":"C. Lenzen and J. Rybicki, “Self-stabilising Byzantine clock synchronisation is almost as easy as consensus,” Journal of the ACM, vol. 66, no. 5. ACM, 2019.","apa":"Lenzen, C., & Rybicki, J. (2019). Self-stabilising Byzantine clock synchronisation is almost as easy as consensus. Journal of the ACM. ACM. https://doi.org/10.1145/3339471","mla":"Lenzen, Christoph, and Joel Rybicki. “Self-Stabilising Byzantine Clock Synchronisation Is Almost as Easy as Consensus.” Journal of the ACM, vol. 66, no. 5, 32, ACM, 2019, doi:10.1145/3339471.","short":"C. Lenzen, J. Rybicki, Journal of the ACM 66 (2019).","chicago":"Lenzen, Christoph, and Joel Rybicki. “Self-Stabilising Byzantine Clock Synchronisation Is Almost as Easy as Consensus.” Journal of the ACM. ACM, 2019. https://doi.org/10.1145/3339471."},"date_published":"2019-09-01T00:00:00Z","scopus_import":"1","day":"01","article_processing_charge":"Yes","has_accepted_license":"1","ddc":["000"],"status":"public","title":"Self-stabilising Byzantine clock synchronisation is almost as easy as consensus","intvolume":" 66","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","_id":"6972","oa_version":"Published Version","file":[{"file_id":"6975","relation":"main_file","date_updated":"2020-07-14T12:47:46Z","date_created":"2019-10-25T12:58:38Z","checksum":"7e5d95c478e0e393f4927fcf7e48194e","file_name":"2019_JACM_Lenzen.pdf","access_level":"open_access","creator":"dernst","file_size":2183085,"content_type":"application/pdf"}],"type":"journal_article","abstract":[{"lang":"eng","text":"We give fault-tolerant algorithms for establishing synchrony in distributed systems in which each of thennodes has its own clock. Our algorithms operate in a very strong fault model: we require self-stabilisation, i.e.,the initial state of the system may be arbitrary, and there can be up to f2018 IEEE Conference on Decision and Control. IEEE, 2019. https://doi.org/10.1109/cdc.2018.8619625.","mla":"Khirirat, Sarit, et al. “Gradient Compression for Communication-Limited Convex Optimization.” 2018 IEEE Conference on Decision and Control, 8619625, IEEE, 2019, doi:10.1109/cdc.2018.8619625.","short":"S. Khirirat, M. Johansson, D.-A. Alistarh, in:, 2018 IEEE Conference on Decision and Control, IEEE, 2019.","ista":"Khirirat S, Johansson M, Alistarh D-A. 2019. Gradient compression for communication-limited convex optimization. 2018 IEEE Conference on Decision and Control. CDC: Conference on Decision and Control, 8619625.","ieee":"S. Khirirat, M. Johansson, and D.-A. Alistarh, “Gradient compression for communication-limited convex optimization,” in 2018 IEEE Conference on Decision and Control, Miami Beach, FL, United States, 2019.","apa":"Khirirat, S., Johansson, M., & Alistarh, D.-A. (2019). Gradient compression for communication-limited convex optimization. In 2018 IEEE Conference on Decision and Control. Miami Beach, FL, United States: IEEE. https://doi.org/10.1109/cdc.2018.8619625","ama":"Khirirat S, Johansson M, Alistarh D-A. Gradient compression for communication-limited convex optimization. In: 2018 IEEE Conference on Decision and Control. IEEE; 2019. doi:10.1109/cdc.2018.8619625"},"external_id":{"isi":["000458114800023"]},"day":"21","month":"01","publication_identifier":{"isbn":["9781538613955"],"issn":["0743-1546"]},"article_processing_charge":"No","scopus_import":"1"},{"author":[{"full_name":"Renggli, Cedric","last_name":"Renggli","first_name":"Cedric"},{"last_name":"Ashkboos","first_name":"Saleh","id":"0D0A9058-257B-11EA-A937-9341C3D8BC8A","full_name":"Ashkboos, Saleh"},{"full_name":"Aghagolzadeh, Mehdi","first_name":"Mehdi","last_name":"Aghagolzadeh"},{"full_name":"Alistarh, Dan-Adrian","last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87"},{"full_name":"Hoefler, Torsten","first_name":"Torsten","last_name":"Hoefler"}],"date_created":"2019-12-22T23:00:42Z","date_updated":"2023-09-06T14:37:55Z","year":"2019","publisher":"ACM","department":[{"_id":"DaAl"}],"publication_status":"published","ec_funded":1,"article_number":"a11","doi":"10.1145/3295500.3356222","conference":{"end_date":"2019-11-19","location":"Denver, CO, Unites States","start_date":"2019-11-17","name":"SC: Conference for High Performance Computing, Networking, Storage and Analysis"},"language":[{"iso":"eng"}],"oa":1,"external_id":{"arxiv":["1802.08021"],"isi":["000545976800011"]},"main_file_link":[{"url":"https://arxiv.org/abs/1802.08021","open_access":"1"}],"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"quality_controlled":"1","isi":1,"publication_identifier":{"eissn":["21674337"],"isbn":["9781450362290"],"issn":["21674329"]},"month":"11","oa_version":"Preprint","user_id":"c635000d-4b10-11ee-a964-aac5a93f6ac1","_id":"7201","title":"SparCML: High-performance sparse communication for machine learning","status":"public","abstract":[{"lang":"eng","text":"Applying machine learning techniques to the quickly growing data in science and industry requires highly-scalable algorithms. Large datasets are most commonly processed \"data parallel\" distributed across many nodes. Each node's contribution to the overall gradient is summed using a global allreduce. This allreduce is the single communication and thus scalability bottleneck for most machine learning workloads. We observe that frequently, many gradient values are (close to) zero, leading to sparse of sparsifyable communications. To exploit this insight, we analyze, design, and implement a set of communication-efficient protocols for sparse input data, in conjunction with efficient machine learning algorithms which can leverage these primitives. Our communication protocols generalize standard collective operations, by allowing processes to contribute arbitrary sparse input data vectors. Our generic communication library, SparCML1, extends MPI to support additional features, such as non-blocking (asynchronous) operations and low-precision data representations. As such, SparCML and its techniques will form the basis of future highly-scalable machine learning frameworks."}],"type":"conference","date_published":"2019-11-17T00:00:00Z","citation":{"ama":"Renggli C, Ashkboos S, Aghagolzadeh M, Alistarh D-A, Hoefler T. SparCML: High-performance sparse communication for machine learning. In: International Conference for High Performance Computing, Networking, Storage and Analysis, SC. ACM; 2019. doi:10.1145/3295500.3356222","ieee":"C. Renggli, S. Ashkboos, M. Aghagolzadeh, D.-A. Alistarh, and T. Hoefler, “SparCML: High-performance sparse communication for machine learning,” in International Conference for High Performance Computing, Networking, Storage and Analysis, SC, Denver, CO, Unites States, 2019.","apa":"Renggli, C., Ashkboos, S., Aghagolzadeh, M., Alistarh, D.-A., & Hoefler, T. (2019). SparCML: High-performance sparse communication for machine learning. In International Conference for High Performance Computing, Networking, Storage and Analysis, SC. Denver, CO, Unites States: ACM. https://doi.org/10.1145/3295500.3356222","ista":"Renggli C, Ashkboos S, Aghagolzadeh M, Alistarh D-A, Hoefler T. 2019. SparCML: High-performance sparse communication for machine learning. International Conference for High Performance Computing, Networking, Storage and Analysis, SC. SC: Conference for High Performance Computing, Networking, Storage and Analysis, a11.","short":"C. Renggli, S. Ashkboos, M. Aghagolzadeh, D.-A. Alistarh, T. Hoefler, in:, International Conference for High Performance Computing, Networking, Storage and Analysis, SC, ACM, 2019.","mla":"Renggli, Cedric, et al. “SparCML: High-Performance Sparse Communication for Machine Learning.” International Conference for High Performance Computing, Networking, Storage and Analysis, SC, a11, ACM, 2019, doi:10.1145/3295500.3356222.","chicago":"Renggli, Cedric, Saleh Ashkboos, Mehdi Aghagolzadeh, Dan-Adrian Alistarh, and Torsten Hoefler. “SparCML: High-Performance Sparse Communication for Machine Learning.” In International Conference for High Performance Computing, Networking, Storage and Analysis, SC. ACM, 2019. https://doi.org/10.1145/3295500.3356222."},"publication":"International Conference for High Performance Computing, Networking, Storage and Analysis, SC","article_processing_charge":"No","day":"17","scopus_import":"1"},{"file":[{"creator":"dernst","file_size":1917374,"content_type":"application/pdf","file_name":"2019_BMCBioinfo_Aganezov.pdf","access_level":"open_access","date_created":"2020-01-02T16:10:58Z","date_updated":"2020-07-14T12:47:54Z","checksum":"7a30357efdcf8f66587ed495c0927724","file_id":"7221","relation":"main_file"}],"oa_version":"Published Version","user_id":"c635000d-4b10-11ee-a964-aac5a93f6ac1","_id":"7214","intvolume":" 20","ddc":["570"],"status":"public","title":"Recovering rearranged cancer chromosomes from karyotype graphs","abstract":[{"lang":"eng","text":"Background: Many cancer genomes are extensively rearranged with highly aberrant chromosomal karyotypes. Structural and copy number variations in cancer genomes can be determined via abnormal mapping of sequenced reads to the reference genome. Recently it became possible to reconcile both of these types of large-scale variations into a karyotype graph representation of the rearranged cancer genomes. Such a representation, however, does not directly describe the linear and/or circular structure of the underlying rearranged cancer chromosomes, thus limiting possible analysis of cancer genomes somatic evolutionary process as well as functional genomic changes brought by the large-scale genome rearrangements.\r\n\r\nResults: Here we address the aforementioned limitation by introducing a novel methodological framework for recovering rearranged cancer chromosomes from karyotype graphs. For a cancer karyotype graph we formulate an Eulerian Decomposition Problem (EDP) of finding a collection of linear and/or circular rearranged cancer chromosomes that are determined by the graph. We derive and prove computational complexities for several variations of the EDP. We then demonstrate that Eulerian decomposition of the cancer karyotype graphs is not always unique and present the Consistent Contig Covering Problem (CCCP) of recovering unambiguous cancer contigs from the cancer karyotype graph, and describe a novel algorithm CCR capable of solving CCCP in polynomial time. We apply CCR on a prostate cancer dataset and demonstrate that it is capable of consistently recovering large cancer contigs even when underlying cancer genomes are highly rearranged.\r\n\r\nConclusions: CCR can recover rearranged cancer contigs from karyotype graphs thereby addressing existing limitation in inferring chromosomal structures of rearranged cancer genomes and advancing our understanding of both patient/cancer-specific as well as the overall genetic instability in cancer."}],"type":"journal_article","date_published":"2019-12-17T00:00:00Z","citation":{"short":"S. Aganezov, I. Zban, V. Aksenov, N. Alexeev, M.C. Schatz, BMC Bioinformatics 20 (2019).","mla":"Aganezov, Sergey, et al. “Recovering Rearranged Cancer Chromosomes from Karyotype Graphs.” BMC Bioinformatics, vol. 20, 641, BMC, 2019, doi:10.1186/s12859-019-3208-4.","chicago":"Aganezov, Sergey, Ilya Zban, Vitalii Aksenov, Nikita Alexeev, and Michael C. Schatz. “Recovering Rearranged Cancer Chromosomes from Karyotype Graphs.” BMC Bioinformatics. BMC, 2019. https://doi.org/10.1186/s12859-019-3208-4.","ama":"Aganezov S, Zban I, Aksenov V, Alexeev N, Schatz MC. Recovering rearranged cancer chromosomes from karyotype graphs. BMC Bioinformatics. 2019;20. doi:10.1186/s12859-019-3208-4","ieee":"S. Aganezov, I. Zban, V. Aksenov, N. Alexeev, and M. C. Schatz, “Recovering rearranged cancer chromosomes from karyotype graphs,” BMC Bioinformatics, vol. 20. BMC, 2019.","apa":"Aganezov, S., Zban, I., Aksenov, V., Alexeev, N., & Schatz, M. C. (2019). Recovering rearranged cancer chromosomes from karyotype graphs. BMC Bioinformatics. BMC. https://doi.org/10.1186/s12859-019-3208-4","ista":"Aganezov S, Zban I, Aksenov V, Alexeev N, Schatz MC. 2019. Recovering rearranged cancer chromosomes from karyotype graphs. BMC Bioinformatics. 20, 641."},"publication":"BMC Bioinformatics","article_type":"original","has_accepted_license":"1","article_processing_charge":"No","day":"17","scopus_import":"1","author":[{"first_name":"Sergey","last_name":"Aganezov","full_name":"Aganezov, Sergey"},{"full_name":"Zban, Ilya","last_name":"Zban","first_name":"Ilya"},{"full_name":"Aksenov, Vitalii","first_name":"Vitalii","last_name":"Aksenov","id":"2980135A-F248-11E8-B48F-1D18A9856A87"},{"full_name":"Alexeev, Nikita","first_name":"Nikita","last_name":"Alexeev"},{"full_name":"Schatz, Michael C.","last_name":"Schatz","first_name":"Michael C."}],"volume":20,"date_updated":"2023-09-06T14:51:06Z","date_created":"2019-12-29T23:00:46Z","year":"2019","publisher":"BMC","department":[{"_id":"DaAl"}],"publication_status":"published","file_date_updated":"2020-07-14T12:47:54Z","article_number":"641","doi":"10.1186/s12859-019-3208-4","language":[{"iso":"eng"}],"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"external_id":{"isi":["000511618800007"]},"isi":1,"quality_controlled":"1","publication_identifier":{"eissn":["14712105"]},"month":"12"},{"abstract":[{"text":"Traditional concurrent programming involves manipulating shared mutable state. Alternatives to this programming style are communicating sequential processes (CSP) and actor models, which share data via explicit communication. These models have been known for almost half a century, and have recently had started to gain significant traction among modern programming languages. The common abstraction for communication between several processes is the channel. Although channels are similar to producer-consumer data structures, they have different semantics and support additional operations, such as the select expression. Despite their growing popularity, most known implementations of channels use lock-based data structures and can be rather inefficient.\r\n\r\nIn this paper, we present the first efficient lock-free algorithm for implementing a communication channel for CSP programming. We provide implementations and experimental results in the Kotlin and Go programming languages. Our new algorithm outperforms existing implementations on many workloads, while providing non-blocking progress guarantee. Our design can serve as an example of how to construct general communication data structures for CSP and actor models. ","lang":"eng"}],"type":"conference","alternative_title":["LNCS"],"oa_version":"None","user_id":"c635000d-4b10-11ee-a964-aac5a93f6ac1","_id":"7228","title":"Scalable FIFO channels for programming via communicating sequential processes","status":"public","intvolume":" 11725","day":"13","article_processing_charge":"No","scopus_import":"1","date_published":"2019-08-13T00:00:00Z","publication":"25th Anniversary of Euro-Par","citation":{"short":"N. Koval, D.-A. Alistarh, R. Elizarov, in:, 25th Anniversary of Euro-Par, Springer Nature, 2019, pp. 317–333.","mla":"Koval, Nikita, et al. “Scalable FIFO Channels for Programming via Communicating Sequential Processes.” 25th Anniversary of Euro-Par, vol. 11725, Springer Nature, 2019, pp. 317–33, doi:10.1007/978-3-030-29400-7_23.","chicago":"Koval, Nikita, Dan-Adrian Alistarh, and Roman Elizarov. “Scalable FIFO Channels for Programming via Communicating Sequential Processes.” In 25th Anniversary of Euro-Par, 11725:317–33. Springer Nature, 2019. https://doi.org/10.1007/978-3-030-29400-7_23.","ama":"Koval N, Alistarh D-A, Elizarov R. Scalable FIFO channels for programming via communicating sequential processes. In: 25th Anniversary of Euro-Par. Vol 11725. Springer Nature; 2019:317-333. doi:10.1007/978-3-030-29400-7_23","apa":"Koval, N., Alistarh, D.-A., & Elizarov, R. (2019). Scalable FIFO channels for programming via communicating sequential processes. In 25th Anniversary of Euro-Par (Vol. 11725, pp. 317–333). Göttingen, Germany: Springer Nature. https://doi.org/10.1007/978-3-030-29400-7_23","ieee":"N. Koval, D.-A. Alistarh, and R. Elizarov, “Scalable FIFO channels for programming via communicating sequential processes,” in 25th Anniversary of Euro-Par, Göttingen, Germany, 2019, vol. 11725, pp. 317–333.","ista":"Koval N, Alistarh D-A, Elizarov R. 2019. Scalable FIFO channels for programming via communicating sequential processes. 25th Anniversary of Euro-Par. Euro-Par: European Conference on Parallel Processing, LNCS, vol. 11725, 317–333."},"page":"317-333","author":[{"last_name":"Koval","first_name":"Nikita","id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87","full_name":"Koval, Nikita"},{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X"},{"full_name":"Elizarov, Roman","first_name":"Roman","last_name":"Elizarov"}],"date_updated":"2023-09-06T14:53:59Z","date_created":"2020-01-05T23:00:46Z","volume":11725,"year":"2019","publication_status":"published","publisher":"Springer Nature","department":[{"_id":"DaAl"}],"month":"08","publication_identifier":{"eissn":["1611-3349"],"isbn":["978-3-0302-9399-4"],"issn":["0302-9743"]},"conference":{"name":"Euro-Par: European Conference on Parallel Processing","start_date":"2019-08-26","location":"Göttingen, Germany","end_date":"2019-08-30"},"doi":"10.1007/978-3-030-29400-7_23","language":[{"iso":"eng"}],"external_id":{"isi":["000851061400023"]},"isi":1,"quality_controlled":"1"},{"article_processing_charge":"No","day":"01","scopus_import":"1","date_published":"2019-06-01T00:00:00Z","page":"12481-12512","citation":{"ama":"Yu C, Tang H, Renggli C, et al. Distributed learning over unreliable networks. In: 36th International Conference on Machine Learning, ICML 2019. Vol 2019-June. IMLS; 2019:12481-12512.","ista":"Yu C, Tang H, Renggli C, Kassing S, Singla A, Alistarh D-A, Zhang C, Liu J. 2019. Distributed learning over unreliable networks. 36th International Conference on Machine Learning, ICML 2019. ICML: International Conference on Machine Learning vol. 2019–June, 12481–12512.","apa":"Yu, C., Tang, H., Renggli, C., Kassing, S., Singla, A., Alistarh, D.-A., … Liu, J. (2019). Distributed learning over unreliable networks. In 36th International Conference on Machine Learning, ICML 2019 (Vol. 2019–June, pp. 12481–12512). Long Beach, CA, United States: IMLS.","ieee":"C. Yu et al., “Distributed learning over unreliable networks,” in 36th International Conference on Machine Learning, ICML 2019, Long Beach, CA, United States, 2019, vol. 2019–June, pp. 12481–12512.","mla":"Yu, Chen, et al. “Distributed Learning over Unreliable Networks.” 36th International Conference on Machine Learning, ICML 2019, vol. 2019–June, IMLS, 2019, pp. 12481–512.","short":"C. Yu, H. Tang, C. Renggli, S. Kassing, A. Singla, D.-A. Alistarh, C. Zhang, J. Liu, in:, 36th International Conference on Machine Learning, ICML 2019, IMLS, 2019, pp. 12481–12512.","chicago":"Yu, Chen, Hanlin Tang, Cedric Renggli, Simon Kassing, Ankit Singla, Dan-Adrian Alistarh, Ce Zhang, and Ji Liu. “Distributed Learning over Unreliable Networks.” In 36th International Conference on Machine Learning, ICML 2019, 2019–June:12481–512. IMLS, 2019."},"publication":"36th International Conference on Machine Learning, ICML 2019","abstract":[{"text":"Most of today's distributed machine learning systems assume reliable networks: whenever two machines exchange information (e.g., gradients or models), the network should guarantee the delivery of the message. At the same time, recent work exhibits the impressive tolerance of machine learning algorithms to errors or noise arising from relaxed communication or synchronization. In this paper, we connect these two trends, and consider the following question: Can we design machine learning systems that are tolerant to network unreliability during training? With this motivation, we focus on a theoretical problem of independent interest-given a standard distributed parameter server architecture, if every communication between the worker and the server has a non-zero probability p of being dropped, does there exist an algorithm that still converges, and at what speed? The technical contribution of this paper is a novel theoretical analysis proving that distributed learning over unreliable network can achieve comparable convergence rate to centralized or distributed learning over reliable networks. Further, we prove that the influence of the packet drop rate diminishes with the growth of the number of parameter servers. We map this theoretical result onto a real-world scenario, training deep neural networks over an unreliable network layer, and conduct network simulation to validate the system improvement by allowing the networks to be unreliable.","lang":"eng"}],"type":"conference","oa_version":"Preprint","status":"public","title":"Distributed learning over unreliable networks","user_id":"c635000d-4b10-11ee-a964-aac5a93f6ac1","_id":"7437","publication_identifier":{"isbn":["9781510886988"]},"month":"06","language":[{"iso":"eng"}],"conference":{"start_date":"2019-06-10","location":"Long Beach, CA, United States","end_date":"2019-06-15","name":"ICML: International Conference on Machine Learning"},"quality_controlled":"1","isi":1,"main_file_link":[{"url":"https://arxiv.org/abs/1810.07766","open_access":"1"}],"oa":1,"external_id":{"isi":["000684034307036"],"arxiv":["1810.07766"]},"volume":"2019-June","date_updated":"2023-09-06T15:21:48Z","date_created":"2020-02-02T23:01:06Z","author":[{"full_name":"Yu, Chen","first_name":"Chen","last_name":"Yu"},{"first_name":"Hanlin","last_name":"Tang","full_name":"Tang, Hanlin"},{"full_name":"Renggli, Cedric","last_name":"Renggli","first_name":"Cedric"},{"full_name":"Kassing, Simon","first_name":"Simon","last_name":"Kassing"},{"full_name":"Singla, Ankit","last_name":"Singla","first_name":"Ankit"},{"full_name":"Alistarh, Dan-Adrian","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh"},{"full_name":"Zhang, Ce","first_name":"Ce","last_name":"Zhang"},{"last_name":"Liu","first_name":"Ji","full_name":"Liu, Ji"}],"department":[{"_id":"DaAl"}],"publisher":"IMLS","publication_status":"published","year":"2019"},{"year":"2019","publisher":"ACM Press","department":[{"_id":"DaAl"}],"publication_status":"published","related_material":{"record":[{"status":"public","relation":"dissertation_contains","id":"10429"}]},"author":[{"full_name":"Alistarh, Dan-Adrian","first_name":"Dan-Adrian","last_name":"Alistarh","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X"},{"orcid":"0000-0001-5634-0731","id":"3279A00C-F248-11E8-B48F-1D18A9856A87","last_name":"Nadiradze","first_name":"Giorgi","full_name":"Nadiradze, Giorgi"},{"id":"2F4DB10C-F248-11E8-B48F-1D18A9856A87","first_name":"Nikita","last_name":"Koval","full_name":"Koval, Nikita"}],"date_updated":"2023-09-07T13:31:39Z","date_created":"2019-07-24T08:59:36Z","ec_funded":1,"oa":1,"external_id":{"arxiv":["2003.09363"],"isi":["000507618500018"]},"main_file_link":[{"url":"https://arxiv.org/abs/2003.09363","open_access":"1"}],"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","grant_number":"805223","_id":"268A44D6-B435-11E9-9278-68D0E5697425"}],"quality_controlled":"1","isi":1,"doi":"10.1145/3323165.3323201","conference":{"name":"SPAA: Symposium on Parallelism in Algorithms and Architectures","start_date":"2019-06-22","location":"Phoenix, AZ, United States","end_date":"2019-06-24"},"language":[{"iso":"eng"}],"publication_identifier":{"isbn":["9781450361842"]},"month":"06","_id":"6673","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","status":"public","title":"Efficiency guarantees for parallel incremental algorithms under relaxed schedulers","oa_version":"Preprint","type":"conference","abstract":[{"text":"Several classic problems in graph processing and computational geometry are solved via incremental algorithms, which split computation into a series of small tasks acting on shared state, which gets updated progressively. While the sequential variant of such algorithms usually specifies a fixed (but sometimes random) order in which the tasks should be performed, a standard approach to parallelizing such algorithms is to relax this constraint to allow for out-of-order parallel execution. This is the case for parallel implementations of Dijkstra's single-source shortest-paths (SSSP) algorithm, and for parallel Delaunay mesh triangulation. While many software frameworks parallelize incremental computation in this way, it is still not well understood whether this relaxed ordering approach can still provide any complexity guarantees. In this paper, we address this problem, and analyze the efficiency guarantees provided by a range of incremental algorithms when parallelized via relaxed schedulers. We show that, for algorithms such as Delaunay mesh triangulation and sorting by insertion, schedulers with a maximum relaxation factor of k in terms of the maximum priority inversion allowed will introduce a maximum amount of wasted work of O(łog n poly(k)), where n is the number of tasks to be executed. For SSSP, we show that the additional work is O(poly(k), dmax / wmin), where dmax is the maximum distance between two nodes, and wmin is the minimum such distance. In practical settings where n >> k, this suggests that the overheads of relaxation will be outweighed by the improved scalability of the relaxed scheduler. On the negative side, we provide lower bounds showing that certain algorithms will inherently incur a non-trivial amount of wasted work due to scheduler relaxation, even for relatively benign relaxed schedulers.","lang":"eng"}],"citation":{"ama":"Alistarh D-A, Nadiradze G, Koval N. Efficiency guarantees for parallel incremental algorithms under relaxed schedulers. In: 31st ACM Symposium on Parallelism in Algorithms and Architectures. ACM Press; 2019:145-154. doi:10.1145/3323165.3323201","ista":"Alistarh D-A, Nadiradze G, Koval N. 2019. Efficiency guarantees for parallel incremental algorithms under relaxed schedulers. 31st ACM Symposium on Parallelism in Algorithms and Architectures. SPAA: Symposium on Parallelism in Algorithms and Architectures, 145–154.","ieee":"D.-A. Alistarh, G. Nadiradze, and N. Koval, “Efficiency guarantees for parallel incremental algorithms under relaxed schedulers,” in 31st ACM Symposium on Parallelism in Algorithms and Architectures, Phoenix, AZ, United States, 2019, pp. 145–154.","apa":"Alistarh, D.-A., Nadiradze, G., & Koval, N. (2019). Efficiency guarantees for parallel incremental algorithms under relaxed schedulers. In 31st ACM Symposium on Parallelism in Algorithms and Architectures (pp. 145–154). Phoenix, AZ, United States: ACM Press. https://doi.org/10.1145/3323165.3323201","mla":"Alistarh, Dan-Adrian, et al. “Efficiency Guarantees for Parallel Incremental Algorithms under Relaxed Schedulers.” 31st ACM Symposium on Parallelism in Algorithms and Architectures, ACM Press, 2019, pp. 145–54, doi:10.1145/3323165.3323201.","short":"D.-A. Alistarh, G. Nadiradze, N. Koval, in:, 31st ACM Symposium on Parallelism in Algorithms and Architectures, ACM Press, 2019, pp. 145–154.","chicago":"Alistarh, Dan-Adrian, Giorgi Nadiradze, and Nikita Koval. “Efficiency Guarantees for Parallel Incremental Algorithms under Relaxed Schedulers.” In 31st ACM Symposium on Parallelism in Algorithms and Architectures, 145–54. ACM Press, 2019. https://doi.org/10.1145/3323165.3323201."},"publication":"31st ACM Symposium on Parallelism in Algorithms and Architectures","page":"145-154","date_published":"2019-06-01T00:00:00Z","scopus_import":"1","article_processing_charge":"No","day":"01"},{"page":"927-938","citation":{"apa":"Wendler, C., Alistarh, D.-A., & Püschel, M. (2019). Powerset convolutional neural networks (Vol. 32, pp. 927–938). Presented at the NIPS: Conference on Neural Information Processing Systems, Vancouver, Canada: Neural Information Processing Systems Foundation.","ieee":"C. Wendler, D.-A. Alistarh, and M. Püschel, “Powerset convolutional neural networks,” presented at the NIPS: Conference on Neural Information Processing Systems, Vancouver, Canada, 2019, vol. 32, pp. 927–938.","ista":"Wendler C, Alistarh D-A, Püschel M. 2019. Powerset convolutional neural networks. NIPS: Conference on Neural Information Processing Systems vol. 32, 927–938.","ama":"Wendler C, Alistarh D-A, Püschel M. Powerset convolutional neural networks. In: Vol 32. Neural Information Processing Systems Foundation; 2019:927-938.","chicago":"Wendler, Chris, Dan-Adrian Alistarh, and Markus Püschel. “Powerset Convolutional Neural Networks,” 32:927–38. Neural Information Processing Systems Foundation, 2019.","short":"C. Wendler, D.-A. Alistarh, M. Püschel, in:, Neural Information Processing Systems Foundation, 2019, pp. 927–938.","mla":"Wendler, Chris, et al. Powerset Convolutional Neural Networks. Vol. 32, Neural Information Processing Systems Foundation, 2019, pp. 927–38."},"date_published":"2019-12-01T00:00:00Z","article_processing_charge":"No","day":"01","intvolume":" 32","title":"Powerset convolutional neural networks","status":"public","user_id":"c635000d-4b10-11ee-a964-aac5a93f6ac1","_id":"7542","oa_version":"Published Version","type":"conference","abstract":[{"text":"We present a novel class of convolutional neural networks (CNNs) for set functions,i.e., data indexed with the powerset of a finite set. The convolutions are derivedas linear, shift-equivariant functions for various notions of shifts on set functions.The framework is fundamentally different from graph convolutions based on theLaplacian, as it provides not one but several basic shifts, one for each element inthe ground set. Prototypical experiments with several set function classificationtasks on synthetic datasets and on datasets derived from real-world hypergraphsdemonstrate the potential of our new powerset CNNs.","lang":"eng"}],"project":[{"call_identifier":"H2020","name":"Elastic Coordination for Scalable Machine Learning","_id":"268A44D6-B435-11E9-9278-68D0E5697425","grant_number":"805223"}],"quality_controlled":"1","isi":1,"external_id":{"arxiv":["1909.02253"],"isi":["000534424300084"]},"main_file_link":[{"url":"http://papers.nips.cc/paper/8379-powerset-convolutional-neural-networks","open_access":"1"}],"oa":1,"language":[{"iso":"eng"}],"conference":{"name":"NIPS: Conference on Neural Information Processing Systems","start_date":"2019-12-08","location":"Vancouver, Canada","end_date":"2019-12-14"},"publication_identifier":{"issn":["1049-5258"]},"month":"12","department":[{"_id":"DaAl"}],"publisher":"Neural Information Processing Systems Foundation","publication_status":"published","year":"2019","volume":32,"date_created":"2020-02-28T10:03:24Z","date_updated":"2023-09-08T11:13:52Z","author":[{"first_name":"Chris","last_name":"Wendler","full_name":"Wendler, Chris"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"},{"full_name":"Püschel, Markus","last_name":"Püschel","first_name":"Markus"}],"ec_funded":1},{"date_published":"2019-08-01T00:00:00Z","publication":"Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing","citation":{"ista":"Foerster K-T, Korhonen J, Rybicki J, Schmid S. 2019. Does preprocessing help under congestion? Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing. PODC: Symposium on Principles of Distributed Computing, 259–261.","apa":"Foerster, K.-T., Korhonen, J., Rybicki, J., & Schmid, S. (2019). Does preprocessing help under congestion? In Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing (pp. 259–261). Toronto, ON, Canada: ACM. https://doi.org/10.1145/3293611.3331581","ieee":"K.-T. Foerster, J. Korhonen, J. Rybicki, and S. Schmid, “Does preprocessing help under congestion?,” in Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing, Toronto, ON, Canada, 2019, pp. 259–261.","ama":"Foerster K-T, Korhonen J, Rybicki J, Schmid S. Does preprocessing help under congestion? In: Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing. ACM; 2019:259-261. doi:10.1145/3293611.3331581","chicago":"Foerster, Klaus-Tycho, Janne Korhonen, Joel Rybicki, and Stefan Schmid. “Does Preprocessing Help under Congestion?” In Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing, 259–61. ACM, 2019. https://doi.org/10.1145/3293611.3331581.","mla":"Foerster, Klaus-Tycho, et al. “Does Preprocessing Help under Congestion?” Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing, ACM, 2019, pp. 259–61, doi:10.1145/3293611.3331581.","short":"K.-T. Foerster, J. Korhonen, J. Rybicki, S. Schmid, in:, Proceedings of the 2019 ACM Symposium on Principles of Distributed Computing, ACM, 2019, pp. 259–261."},"page":"259-261","day":"01","article_processing_charge":"No","scopus_import":"1","oa_version":"Preprint","_id":"6935","user_id":"c635000d-4b10-11ee-a964-aac5a93f6ac1","title":"Does preprocessing help under congestion?","status":"public","abstract":[{"lang":"eng","text":"This paper investigates the power of preprocessing in the CONGEST model. Schmid and Suomela (ACM HotSDN 2013) introduced the SUPPORTED CONGEST model to study the application of distributed algorithms in Software-Defined Networks (SDNs). In this paper, we show that a large class of lower bounds in the CONGEST model still hold in the SUPPORTED model, highlighting the robustness of these bounds. This also raises the question how much does\r\npreprocessing help in the CONGEST model."}],"type":"conference","conference":{"name":"PODC: Symposium on Principles of Distributed Computing","end_date":"2019-08-02","start_date":"2019-07-29","location":"Toronto, ON, Canada"},"doi":"10.1145/3293611.3331581","language":[{"iso":"eng"}],"external_id":{"arxiv":["1905.03012"],"isi":["000570442000037"]},"main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/1905.03012"}],"oa":1,"isi":1,"quality_controlled":"1","project":[{"_id":"260C2330-B435-11E9-9278-68D0E5697425","grant_number":"754411","name":"ISTplus - Postdoctoral Fellowships","call_identifier":"H2020"}],"month":"08","publication_identifier":{"isbn":["9781450362177"]},"author":[{"last_name":"Foerster","first_name":"Klaus-Tycho","full_name":"Foerster, Klaus-Tycho"},{"full_name":"Korhonen, Janne","last_name":"Korhonen","first_name":"Janne","id":"C5402D42-15BC-11E9-A202-CA2BE6697425"},{"first_name":"Joel","last_name":"Rybicki","id":"334EFD2E-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0002-6432-6646","full_name":"Rybicki, Joel"},{"first_name":"Stefan","last_name":"Schmid","full_name":"Schmid, Stefan"}],"date_updated":"2023-09-08T11:37:22Z","date_created":"2019-10-08T12:57:14Z","year":"2019","publication_status":"published","publisher":"ACM","department":[{"_id":"DaAl"}],"ec_funded":1},{"scopus_import":"1","day":"01","article_processing_charge":"No","publication":"Proceedings of the 51st Annual ACM SIGACT Symposium on Theory of Computing","citation":{"ieee":"D.-A. Alistarh, J. Aspnes, F. Ellen, R. Gelashvili, and L. Zhu, “Why extension-based proofs fail,” in Proceedings of the 51st Annual ACM SIGACT Symposium on Theory of Computing, Phoenix, AZ, United States, 2019, pp. 986–996.","apa":"Alistarh, D.-A., Aspnes, J., Ellen, F., Gelashvili, R., & Zhu, L. (2019). Why extension-based proofs fail. In Proceedings of the 51st Annual ACM SIGACT Symposium on Theory of Computing (pp. 986–996). Phoenix, AZ, United States: ACM Press. https://doi.org/10.1145/3313276.3316407","ista":"Alistarh D-A, Aspnes J, Ellen F, Gelashvili R, Zhu L. 2019. Why extension-based proofs fail. Proceedings of the 51st Annual ACM SIGACT Symposium on Theory of Computing. STOC: Symposium on Theory of Computing, 986–996.","ama":"Alistarh D-A, Aspnes J, Ellen F, Gelashvili R, Zhu L. Why extension-based proofs fail. In: Proceedings of the 51st Annual ACM SIGACT Symposium on Theory of Computing. ACM Press; 2019:986-996. doi:10.1145/3313276.3316407","chicago":"Alistarh, Dan-Adrian, James Aspnes, Faith Ellen, Rati Gelashvili, and Leqi Zhu. “Why Extension-Based Proofs Fail.” In Proceedings of the 51st Annual ACM SIGACT Symposium on Theory of Computing, 986–96. ACM Press, 2019. https://doi.org/10.1145/3313276.3316407.","short":"D.-A. Alistarh, J. Aspnes, F. Ellen, R. Gelashvili, L. Zhu, in:, Proceedings of the 51st Annual ACM SIGACT Symposium on Theory of Computing, ACM Press, 2019, pp. 986–996.","mla":"Alistarh, Dan-Adrian, et al. “Why Extension-Based Proofs Fail.” Proceedings of the 51st Annual ACM SIGACT Symposium on Theory of Computing, ACM Press, 2019, pp. 986–96, doi:10.1145/3313276.3316407."},"page":"986-996","date_published":"2019-06-01T00:00:00Z","type":"conference","abstract":[{"lang":"eng","text":"It is impossible to deterministically solve wait-free consensus in an asynchronous system. The classic proof uses a valency argument, which constructs an infinite execution by repeatedly extending a finite execution. We introduce extension-based proofs, a class of impossibility proofs that are modelled as an interaction between a prover and a protocol and that include valency arguments.\r\n\r\nUsing proofs based on combinatorial topology, it has been shown that it is impossible to deterministically solve k-set agreement among n > k ≥ 2 processes in a wait-free manner. However, it was unknown whether proofs based on simpler techniques were possible. We show that this impossibility result cannot be obtained by an extension-based proof and, hence, extension-based proofs are limited in power."}],"_id":"6676","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","status":"public","title":"Why extension-based proofs fail","oa_version":"Preprint","month":"06","publication_identifier":{"isbn":["9781450367059"]},"oa":1,"main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/1811.01421"}],"external_id":{"isi":["000523199100089"],"arxiv":["1811.01421"]},"quality_controlled":"1","isi":1,"conference":{"name":"STOC: Symposium on Theory of Computing","end_date":"2019-06-26","location":"Phoenix, AZ, United States","start_date":"2019-06-23"},"doi":"10.1145/3313276.3316407","language":[{"iso":"eng"}],"year":"2019","publication_status":"published","publisher":"ACM Press","department":[{"_id":"DaAl"}],"author":[{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"},{"full_name":"Aspnes, James","first_name":"James","last_name":"Aspnes"},{"full_name":"Ellen, Faith","last_name":"Ellen","first_name":"Faith"},{"full_name":"Gelashvili, Rati","last_name":"Gelashvili","first_name":"Rati"},{"full_name":"Zhu, Leqi","first_name":"Leqi","last_name":"Zhu"}],"related_material":{"record":[{"status":"public","relation":"later_version","id":"14364"}]},"date_created":"2019-07-24T09:13:05Z","date_updated":"2023-12-13T12:28:28Z"},{"scopus_import":"1","day":"01","article_processing_charge":"No","page":"74-83","publication":"Proceedings of the 2019 ACM Symposium on Principles of Distributed Computin","citation":{"ieee":"K. Censor-Hillel, M. Dory, J. Korhonen, and D. Leitersdorf, “Fast approximate shortest paths in the congested clique,” in Proceedings of the 2019 ACM Symposium on Principles of Distributed Computin, Toronto, ON, Canada, 2019, pp. 74–83.","apa":"Censor-Hillel, K., Dory, M., Korhonen, J., & Leitersdorf, D. (2019). Fast approximate shortest paths in the congested clique. In Proceedings of the 2019 ACM Symposium on Principles of Distributed Computin (pp. 74–83). Toronto, ON, Canada: ACM. https://doi.org/10.1145/3293611.3331633","ista":"Censor-Hillel K, Dory M, Korhonen J, Leitersdorf D. 2019. Fast approximate shortest paths in the congested clique. Proceedings of the 2019 ACM Symposium on Principles of Distributed Computin. PODC: Symposium on Principles of Distributed Computing, 74–83.","ama":"Censor-Hillel K, Dory M, Korhonen J, Leitersdorf D. Fast approximate shortest paths in the congested clique. In: Proceedings of the 2019 ACM Symposium on Principles of Distributed Computin. ACM; 2019:74-83. doi:10.1145/3293611.3331633","chicago":"Censor-Hillel, Keren, Michal Dory, Janne Korhonen, and Dean Leitersdorf. “Fast Approximate Shortest Paths in the Congested Clique.” In Proceedings of the 2019 ACM Symposium on Principles of Distributed Computin, 74–83. ACM, 2019. https://doi.org/10.1145/3293611.3331633.","short":"K. Censor-Hillel, M. Dory, J. Korhonen, D. Leitersdorf, in:, Proceedings of the 2019 ACM Symposium on Principles of Distributed Computin, ACM, 2019, pp. 74–83.","mla":"Censor-Hillel, Keren, et al. “Fast Approximate Shortest Paths in the Congested Clique.” Proceedings of the 2019 ACM Symposium on Principles of Distributed Computin, ACM, 2019, pp. 74–83, doi:10.1145/3293611.3331633."},"date_published":"2019-08-01T00:00:00Z","type":"conference","abstract":[{"lang":"eng","text":"We design fast deterministic algorithms for distance computation in the CONGESTED CLIQUE model. Our key contributions include:\r\n\r\n - A (2+ε)-approximation for all-pairs shortest paths problem in O(log²n / ε) rounds on unweighted undirected graphs. With a small additional additive factor, this also applies for weighted graphs. This is the first sub-polynomial constant-factor approximation for APSP in this model.\r\n - A (1+ε)-approximation for multi-source shortest paths problem from O(√n) sources in O(log² n / ε) rounds on weighted undirected graphs. This is the first sub-polynomial algorithm obtaining this approximation for a set of sources of polynomial size.\r\n\r\nOur main techniques are new distance tools that are obtained via improved algorithms for sparse matrix multiplication, which we leverage to construct efficient hopsets and shortest paths. Furthermore, our techniques extend to additional distance problems for which we improve upon the state-of-the-art, including diameter approximation, and an exact single-source shortest paths algorithm for weighted undirected graphs in Õ(n^{1/6}) rounds."}],"status":"public","title":"Fast approximate shortest paths in the congested clique","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","_id":"6933","oa_version":"Preprint","month":"08","publication_identifier":{"isbn":["9781450362177"]},"isi":1,"quality_controlled":"1","main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/1903.05956"}],"external_id":{"isi":["000570442000011"],"arxiv":["1903.05956"]},"oa":1,"language":[{"iso":"eng"}],"conference":{"name":"PODC: Symposium on Principles of Distributed Computing","end_date":"2019-08-02","location":"Toronto, ON, Canada","start_date":"2019-07-29"},"doi":"10.1145/3293611.3331633","publication_status":"published","department":[{"_id":"DaAl"}],"publisher":"ACM","year":"2019","date_created":"2019-10-08T12:48:42Z","date_updated":"2024-03-07T14:43:38Z","author":[{"first_name":"Keren","last_name":"Censor-Hillel","full_name":"Censor-Hillel, Keren"},{"first_name":"Michal","last_name":"Dory","full_name":"Dory, Michal"},{"full_name":"Korhonen, Janne","id":"C5402D42-15BC-11E9-A202-CA2BE6697425","last_name":"Korhonen","first_name":"Janne"},{"full_name":"Leitersdorf, Dean","last_name":"Leitersdorf","first_name":"Dean"}],"related_material":{"record":[{"relation":"later_version","status":"public","id":"7939"}]}},{"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","short":"CC BY (4.0)","image":"/images/cc_by.png"},"oa":1,"project":[{"name":"IST Austria Open Access Fund","_id":"B67AFEDC-15C9-11EA-A837-991A96BB2854"}],"quality_controlled":"1","doi":"10.1007/s00446-017-0315-1","language":[{"iso":"eng"}],"publication_identifier":{"issn":["01782770"]},"month":"11","year":"2018","department":[{"_id":"DaAl"}],"publisher":"Springer","publication_status":"published","author":[{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"},{"full_name":"Aspnes, James","last_name":"Aspnes","first_name":"James"},{"full_name":"King, Valerie","first_name":"Valerie","last_name":"King"},{"full_name":"Saia, Jared","last_name":"Saia","first_name":"Jared"}],"volume":31,"date_updated":"2023-02-23T12:23:25Z","date_created":"2018-12-11T11:47:01Z","publist_id":"7281","file_date_updated":"2020-07-14T12:46:38Z","citation":{"ista":"Alistarh D-A, Aspnes J, King V, Saia J. 2018. Communication-efficient randomized consensus. Distributed Computing. 31(6), 489–501.","ieee":"D.-A. Alistarh, J. Aspnes, V. King, and J. Saia, “Communication-efficient randomized consensus,” Distributed Computing, vol. 31, no. 6. Springer, pp. 489–501, 2018.","apa":"Alistarh, D.-A., Aspnes, J., King, V., & Saia, J. (2018). Communication-efficient randomized consensus. Distributed Computing. Springer. https://doi.org/10.1007/s00446-017-0315-1","ama":"Alistarh D-A, Aspnes J, King V, Saia J. Communication-efficient randomized consensus. Distributed Computing. 2018;31(6):489-501. doi:10.1007/s00446-017-0315-1","chicago":"Alistarh, Dan-Adrian, James Aspnes, Valerie King, and Jared Saia. “Communication-Efficient Randomized Consensus.” Distributed Computing. Springer, 2018. https://doi.org/10.1007/s00446-017-0315-1.","mla":"Alistarh, Dan-Adrian, et al. “Communication-Efficient Randomized Consensus.” Distributed Computing, vol. 31, no. 6, Springer, 2018, pp. 489–501, doi:10.1007/s00446-017-0315-1.","short":"D.-A. Alistarh, J. Aspnes, V. King, J. Saia, Distributed Computing 31 (2018) 489–501."},"publication":"Distributed Computing","page":"489-501","date_published":"2018-11-01T00:00:00Z","scopus_import":1,"has_accepted_license":"1","article_processing_charge":"Yes (via OA deal)","day":"01","_id":"536","user_id":"3E5EF7F0-F248-11E8-B48F-1D18A9856A87","intvolume":" 31","status":"public","ddc":["000"],"title":"Communication-efficient randomized consensus","file":[{"file_size":595707,"content_type":"application/pdf","creator":"dernst","access_level":"open_access","file_name":"2017_DistribComp_Alistarh.pdf","checksum":"69b46e537acdcac745237ddb853fcbb5","date_updated":"2020-07-14T12:46:38Z","date_created":"2019-01-22T07:25:51Z","relation":"main_file","file_id":"5867"}],"oa_version":"Published Version","type":"journal_article","issue":"6","abstract":[{"lang":"eng","text":"We consider the problem of consensus in the challenging classic model. In this model, the adversary is adaptive; it can choose which processors crash at any point during the course of the algorithm. Further, communication is via asynchronous message passing: there is no known upper bound on the time to send a message from one processor to another, and all messages and coin flips are seen by the adversary. We describe a new randomized consensus protocol with expected message complexity O(n2log2n) when fewer than n / 2 processes may fail by crashing. This is an almost-linear improvement over the best previously known protocol, and within logarithmic factors of a known Ω(n2) message lower bound. The protocol further ensures that no process sends more than O(nlog3n) messages in expectation, which is again within logarithmic factors of optimal. We also present a generalization of the algorithm to an arbitrary number of failures t, which uses expected O(nt+t2log2t) total messages. Our approach is to build a message-efficient, resilient mechanism for aggregating individual processor votes, implementing the message-passing equivalent of a weak shared coin. Roughly, in our protocol, a processor first announces its votes to small groups, then propagates them to increasingly larger groups as it generates more and more votes. To bound the number of messages that an individual process might have to send or receive, the protocol progressively increases the weight of generated votes. The main technical challenge is bounding the impact of votes that are still “in flight” (generated, but not fully propagated) on the final outcome of the shared coin, especially since such votes might have different weights. We achieve this by leveraging the structure of the algorithm, and a technical argument based on martingale concentration bounds. Overall, we show that it is possible to build an efficient message-passing implementation of a shared coin, and in the process (almost-optimally) solve the classic consensus problem in the asynchronous message-passing model."}]},{"page":"145-156","citation":{"chicago":"Grubic, Demjan, Leo Tam, Dan-Adrian Alistarh, and Ce Zhang. “Synchronous Multi-GPU Training for Deep Learning with Low-Precision Communications: An Empirical Study.” In Proceedings of the 21st International Conference on Extending Database Technology, 145–56. OpenProceedings, 2018. https://doi.org/10.5441/002/EDBT.2018.14.","mla":"Grubic, Demjan, et al. “Synchronous Multi-GPU Training for Deep Learning with Low-Precision Communications: An Empirical Study.” Proceedings of the 21st International Conference on Extending Database Technology, OpenProceedings, 2018, pp. 145–56, doi:10.5441/002/EDBT.2018.14.","short":"D. Grubic, L. Tam, D.-A. Alistarh, C. Zhang, in:, Proceedings of the 21st International Conference on Extending Database Technology, OpenProceedings, 2018, pp. 145–156.","ista":"Grubic D, Tam L, Alistarh D-A, Zhang C. 2018. Synchronous multi-GPU training for deep learning with low-precision communications: An empirical study. Proceedings of the 21st International Conference on Extending Database Technology. EDBT: Conference on Extending Database Technology, 145–156.","apa":"Grubic, D., Tam, L., Alistarh, D.-A., & Zhang, C. (2018). Synchronous multi-GPU training for deep learning with low-precision communications: An empirical study. In Proceedings of the 21st International Conference on Extending Database Technology (pp. 145–156). Vienna, Austria: OpenProceedings. https://doi.org/10.5441/002/EDBT.2018.14","ieee":"D. Grubic, L. Tam, D.-A. Alistarh, and C. Zhang, “Synchronous multi-GPU training for deep learning with low-precision communications: An empirical study,” in Proceedings of the 21st International Conference on Extending Database Technology, Vienna, Austria, 2018, pp. 145–156.","ama":"Grubic D, Tam L, Alistarh D-A, Zhang C. Synchronous multi-GPU training for deep learning with low-precision communications: An empirical study. In: Proceedings of the 21st International Conference on Extending Database Technology. OpenProceedings; 2018:145-156. doi:10.5441/002/EDBT.2018.14"},"publication":"Proceedings of the 21st International Conference on Extending Database Technology","date_published":"2018-03-26T00:00:00Z","scopus_import":1,"has_accepted_license":"1","article_processing_charge":"No","day":"26","title":"Synchronous multi-GPU training for deep learning with low-precision communications: An empirical study","status":"public","ddc":["000"],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","_id":"7116","oa_version":"Published Version","file":[{"file_name":"2018_OpenProceedings_Grubic.pdf","access_level":"open_access","creator":"dernst","content_type":"application/pdf","file_size":1603204,"file_id":"7118","relation":"main_file","date_updated":"2020-07-14T12:47:49Z","date_created":"2019-11-26T14:23:04Z","checksum":"ec979b56abc71016d6e6adfdadbb4afe"}],"type":"conference","abstract":[{"lang":"eng","text":"Training deep learning models has received tremendous research interest recently. In particular, there has been intensive research on reducing the communication cost of training when using multiple computational devices, through reducing the precision of the underlying data representation. Naturally, such methods induce system trade-offs—lowering communication precision could de-crease communication overheads and improve scalability; but, on the other hand, it can also reduce the accuracy of training. In this paper, we study this trade-off space, and ask:Can low-precision communication consistently improve the end-to-end performance of training modern neural networks, with no accuracy loss?From the performance point of view, the answer to this question may appear deceptively easy: compressing communication through low precision should help when the ratio between communication and computation is high. However, this answer is less straightforward when we try to generalize this principle across various neural network architectures (e.g., AlexNet vs. ResNet),number of GPUs (e.g., 2 vs. 8 GPUs), machine configurations(e.g., EC2 instances vs. NVIDIA DGX-1), communication primitives (e.g., MPI vs. NCCL), and even different GPU architectures(e.g., Kepler vs. Pascal). Currently, it is not clear how a realistic realization of all these factors maps to the speed up provided by low-precision communication. In this paper, we conduct an empirical study to answer this question and report the insights."}],"quality_controlled":"1","oa":1,"tmp":{"name":"Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International (CC BY-NC-ND 4.0)","legal_code_url":"https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode","short":"CC BY-NC-ND (4.0)","image":"/images/cc_by_nc_nd.png"},"language":[{"iso":"eng"}],"doi":"10.5441/002/EDBT.2018.14","conference":{"name":"EDBT: Conference on Extending Database Technology","end_date":"2018-03-29","start_date":"2018-03-26","location":"Vienna, Austria"},"publication_identifier":{"isbn":["9783893180783"],"issn":["2367-2005"]},"month":"03","publisher":"OpenProceedings","department":[{"_id":"DaAl"}],"publication_status":"published","year":"2018","date_updated":"2023-02-23T12:59:17Z","date_created":"2019-11-26T14:19:11Z","author":[{"last_name":"Grubic","first_name":"Demjan","full_name":"Grubic, Demjan"},{"first_name":"Leo","last_name":"Tam","full_name":"Tam, Leo"},{"full_name":"Alistarh, Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","last_name":"Alistarh","first_name":"Dan-Adrian"},{"last_name":"Zhang","first_name":"Ce","full_name":"Zhang, Ce"}],"license":"https://creativecommons.org/licenses/by-nc-nd/4.0/","file_date_updated":"2020-07-14T12:47:49Z"},{"publisher":"Association for Computing Machinery","department":[{"_id":"DaAl"}],"intvolume":" 4","title":"ThreadScan: Automatic and scalable memory reclamation","publication_status":"published","status":"public","_id":"6001","year":"2018","user_id":"3E5EF7F0-F248-11E8-B48F-1D18A9856A87","volume":4,"oa_version":"None","date_updated":"2023-02-23T13:17:54Z","date_created":"2019-02-14T13:24:11Z","related_material":{"record":[{"id":"779","relation":"earlier_version","status":"public"}]},"author":[{"last_name":"Alistarh","first_name":"Dan-Adrian","orcid":"0000-0003-3650-940X","id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","full_name":"Alistarh, Dan-Adrian"},{"last_name":"Leiserson","first_name":"William","full_name":"Leiserson, William"},{"first_name":"Alexander","last_name":"Matveev","full_name":"Matveev, Alexander"},{"first_name":"Nir","last_name":"Shavit","full_name":"Shavit, Nir"}],"type":"journal_article","article_number":"18","issue":"4","abstract":[{"lang":"eng","text":"The concurrent memory reclamation problem is that of devising a way for a deallocating thread to verify that no other concurrent threads hold references to a memory block being deallocated. To date, in the absence of automatic garbage collection, there is no satisfactory solution to this problem; existing tracking methods like hazard pointers, reference counters, or epoch-based techniques like RCU are either prohibitively expensive or require significant programming expertise to the extent that implementing them efficiently can be worthy of a publication. None of the existing techniques are automatic or even semi-automated.\r\nIn this article, we take a new approach to concurrent memory reclamation. Instead of manually tracking access to memory locations as done in techniques like hazard pointers, or restricting shared accesses to specific epoch boundaries as in RCU, our algorithm, called ThreadScan, leverages operating system signaling to automatically detect which memory locations are being accessed by concurrent threads.\r\nInitial empirical evidence shows that ThreadScan scales surprisingly well and requires negligible programming effort beyond the standard use of Malloc and Free."}],"quality_controlled":"1","citation":{"short":"D.-A. Alistarh, W. Leiserson, A. Matveev, N. Shavit, ACM Transactions on Parallel Computing 4 (2018).","mla":"Alistarh, Dan-Adrian, et al. “ThreadScan: Automatic and Scalable Memory Reclamation.” ACM Transactions on Parallel Computing, vol. 4, no. 4, 18, Association for Computing Machinery, 2018, doi:10.1145/3201897.","chicago":"Alistarh, Dan-Adrian, William Leiserson, Alexander Matveev, and Nir Shavit. “ThreadScan: Automatic and Scalable Memory Reclamation.” ACM Transactions on Parallel Computing. Association for Computing Machinery, 2018. https://doi.org/10.1145/3201897.","ama":"Alistarh D-A, Leiserson W, Matveev A, Shavit N. ThreadScan: Automatic and scalable memory reclamation. ACM Transactions on Parallel Computing. 2018;4(4). doi:10.1145/3201897","apa":"Alistarh, D.-A., Leiserson, W., Matveev, A., & Shavit, N. (2018). ThreadScan: Automatic and scalable memory reclamation. ACM Transactions on Parallel Computing. Association for Computing Machinery. https://doi.org/10.1145/3201897","ieee":"D.-A. Alistarh, W. Leiserson, A. Matveev, and N. Shavit, “ThreadScan: Automatic and scalable memory reclamation,” ACM Transactions on Parallel Computing, vol. 4, no. 4. Association for Computing Machinery, 2018.","ista":"Alistarh D-A, Leiserson W, Matveev A, Shavit N. 2018. ThreadScan: Automatic and scalable memory reclamation. ACM Transactions on Parallel Computing. 4(4), 18."},"publication":"ACM Transactions on Parallel Computing","language":[{"iso":"eng"}],"doi":"10.1145/3201897","date_published":"2018-09-01T00:00:00Z","scopus_import":1,"publication_identifier":{"issn":["2329-4949"]},"month":"09","day":"01"},{"quality_controlled":"1","publication":"6th International Conference on Learning Representations","external_id":{"arxiv":["1802.05668"]},"oa":1,"citation":{"ista":"Polino A, Pascanu R, Alistarh D-A. 2018. Model compression via distillation and quantization. 6th International Conference on Learning Representations. ICLR: International Conference on Learning Representations.","apa":"Polino, A., Pascanu, R., & Alistarh, D.-A. (2018). Model compression via distillation and quantization. In 6th International Conference on Learning Representations. Vancouver, Canada.","ieee":"A. Polino, R. Pascanu, and D.-A. Alistarh, “Model compression via distillation and quantization,” in 6th International Conference on Learning Representations, Vancouver, Canada, 2018.","ama":"Polino A, Pascanu R, Alistarh D-A. Model compression via distillation and quantization. In: 6th International Conference on Learning Representations. ; 2018.","chicago":"Polino, Antonio, Razvan Pascanu, and Dan-Adrian Alistarh. “Model Compression via Distillation and Quantization.” In 6th International Conference on Learning Representations, 2018.","mla":"Polino, Antonio, et al. “Model Compression via Distillation and Quantization.” 6th International Conference on Learning Representations, 2018.","short":"A. Polino, R. Pascanu, D.-A. Alistarh, in:, 6th International Conference on Learning Representations, 2018."},"language":[{"iso":"eng"}],"conference":{"end_date":"2018-05-03","location":"Vancouver, Canada","start_date":"2018-04-30","name":"ICLR: International Conference on Learning Representations"},"date_published":"2018-05-01T00:00:00Z","scopus_import":1,"day":"01","month":"05","article_processing_charge":"No","has_accepted_license":"1","publication_status":"published","status":"public","ddc":["000"],"title":"Model compression via distillation and quantization","department":[{"_id":"DaAl"}],"_id":"7812","user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","year":"2018","date_created":"2020-05-10T22:00:51Z","date_updated":"2023-02-23T13:18:41Z","file":[{"content_type":"application/pdf","file_size":308339,"creator":"dernst","file_name":"2018_ICLR_Polino.pdf","access_level":"open_access","date_updated":"2020-07-14T12:48:03Z","date_created":"2020-05-26T13:02:00Z","checksum":"a4336c167978e81891970e4e4517a8c3","relation":"main_file","file_id":"7894"}],"oa_version":"Published Version","author":[{"full_name":"Polino, Antonio","last_name":"Polino","first_name":"Antonio"},{"last_name":"Pascanu","first_name":"Razvan","full_name":"Pascanu, Razvan"},{"id":"4A899BFC-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3650-940X","first_name":"Dan-Adrian","last_name":"Alistarh","full_name":"Alistarh, Dan-Adrian"}],"type":"conference","abstract":[{"lang":"eng","text":"Deep neural networks (DNNs) continue to make significant advances, solving tasks from image classification to translation or reinforcement learning. One aspect of the field receiving considerable attention is efficiently executing deep models in resource-constrained environments, such as mobile or embedded devices. This paper focuses on this problem, and proposes two new compression methods, which jointly leverage weight quantization and distillation of larger teacher networks into smaller student networks. The first method we propose is called quantized distillation and leverages distillation during the training process, by incorporating distillation loss, expressed with respect to the teacher, into the training of a student network whose weights are quantized to a limited set of levels. The second method, differentiable quantization, optimizes the location of quantization points through stochastic gradient descent, to better fit the behavior of the teacher model. We validate both methods through experiments on convolutional and recurrent architectures. We show that quantized shallow students can reach similar accuracy levels to full-precision teacher models, while providing order of magnitude compression, and inference speedup that is linear in the depth reduction. In sum, our results enable DNNs for resource-constrained environments to leverage architecture and accuracy advances developed on more powerful devices."}],"file_date_updated":"2020-07-14T12:48:03Z"},{"oa_version":"None","intvolume":" 53","title":"Harnessing epoch-based reclamation for efficient range queries","status":"public","user_id":"c635000d-4b10-11ee-a964-aac5a93f6ac1","_id":"397","issue":"1","abstract":[{"lang":"eng","text":"Concurrent sets with range query operations are highly desirable in applications such as in-memory databases. However, few set implementations offer range queries. Known techniques for augmenting data structures with range queries (or operations that can be used to build range queries) have numerous problems that limit their usefulness. For example, they impose high overhead or rely heavily on garbage collection. In this work, we show how to augment data structures with highly efficient range queries, without relying on garbage collection. We identify a property of epoch-based memory reclamation algorithms that makes them ideal for implementing range queries, and produce three algorithms, which use locks, transactional memory and lock-free techniques, respectively. Our algorithms are applicable to more data structures than previous work, and are shown to be highly efficient on a large scale Intel system. "}],"alternative_title":["PPoPP"],"type":"conference","date_published":"2018-02-10T00:00:00Z","page":"14 - 27","citation":{"ama":"Arbel Raviv M, Brown TA. Harnessing epoch-based reclamation for efficient range queries. In: Vol 53. ACM; 2018:14-27. doi:10.1145/3178487.3178489","ieee":"M. Arbel Raviv and T. A. Brown, “Harnessing epoch-based reclamation for efficient range queries,” presented at the PPoPP: Principles and Practice of Parallel Programming, Vienna, Austria, 2018, vol. 53, no. 1, pp. 14–27.","apa":"Arbel Raviv, M., & Brown, T. A. (2018). Harnessing epoch-based reclamation for efficient range queries (Vol. 53, pp. 14–27). Presented at the PPoPP: Principles and Practice of Parallel Programming, Vienna, Austria: ACM. https://doi.org/10.1145/3178487.3178489","ista":"Arbel Raviv M, Brown TA. 2018. Harnessing epoch-based reclamation for efficient range queries. PPoPP: Principles and Practice of Parallel Programming, PPoPP, vol. 53, 14–27.","short":"M. Arbel Raviv, T.A. Brown, in:, ACM, 2018, pp. 14–27.","mla":"Arbel Raviv, Maya, and Trevor A. Brown. Harnessing Epoch-Based Reclamation for Efficient Range Queries. Vol. 53, no. 1, ACM, 2018, pp. 14–27, doi:10.1145/3178487.3178489.","chicago":"Arbel Raviv, Maya, and Trevor A Brown. “Harnessing Epoch-Based Reclamation for Efficient Range Queries,” 53:14–27. ACM, 2018. https://doi.org/10.1145/3178487.3178489."},"article_processing_charge":"No","day":"10","scopus_import":"1","volume":53,"date_updated":"2023-09-11T14:10:25Z","date_created":"2018-12-11T11:46:14Z","author":[{"full_name":"Arbel Raviv, Maya","first_name":"Maya","last_name":"Arbel Raviv"},{"full_name":"Brown, Trevor A","id":"3569F0A0-F248-11E8-B48F-1D18A9856A87","last_name":"Brown","first_name":"Trevor A"}],"department":[{"_id":"DaAl"}],"publisher":"ACM","publication_status":"published","year":"2018","publist_id":"7430","language":[{"iso":"eng"}],"doi":"10.1145/3178487.3178489","conference":{"location":"Vienna, Austria","start_date":"2018-02-24","end_date":"2018-02-28","name":"PPoPP: Principles and Practice of Parallel Programming"},"isi":1,"quality_controlled":"1","external_id":{"isi":["000446161100002"]},"publication_identifier":{"isbn":["978-1-4503-4982-6"]},"month":"02"}]