@article{15165, abstract = {Current knowledge suggests a drought Indian monsoon (perhaps a severe one) when the El Nino Southern Oscillation and Pacific Decadal Oscillation each exhibit positive phases (a joint positive phase). For the monsoons, which are exceptions in this regard, we found northeast India often gets excess pre-monsoon rainfall. Further investigation reveals that this excess pre-monsoon rainfall is produced by the interaction of the large-scale circulation associated with the joint phase with the mountains in northeast India. We posit that a warmer troposphere, a consequence of excess rainfall over northeast India, drives a stronger monsoon circulation and enhances monsoon rainfall over central India. Hence, we argue that pre-monsoon rainfall over northeast India can be used for seasonal monsoon rainfall prediction over central India. Most importantly, its predictive value is at its peak when the Pacific Ocean exhibits a joint positive phase and the threat of extreme drought monsoon looms over India.}, author = {Goswami, Bidyut B}, issn = {1944-8007}, journal = {Geophysical Research Letters}, number = {5}, publisher = {Wiley}, title = {{A pre-monsoon signal of false alarms of Indian monsoon droughts}}, doi = {10.1029/2023GL106569}, volume = {51}, year = {2024}, } @article{18708, abstract = {This study investigates the response of Indian summer monsoon (ISM) precipitation to CO2 removal, with a specific focus on regional and subseasonal variations. Following CO2 removal, monsoon circulation weakens throughout the summer owing to the reduced large-scale meridional temperature gradient around India. Weakened monsoon circulation decreases the local-scale thermodynamic stability within India, following monsoon-onset periods. While the frequency of synoptic-scale ISM low-pressure systems (LPSs) decreases overall, the lower thermodynamic stability causes the LPSs to form and resultantly shift west and south from their typical paths, last longer and move more quickly zonally during August and September. Changes in these rain-producing processes induce distinct regional (Western Ghats, south-central-east India, and Tamil Nadu) and subseasonal precipitation responses. Also, extreme precipitation exhibits similar patterns, but is more strongly affected by changes in LPS. Our results suggest that reliable future projections of regional hydroclimate change require a more accurate understanding of multi-scale precipitation processes.}, author = {Paik, Seungmok and Kim, Daehyun and An, Soon Il and Oh, Hyoeun and Shin, Jongsoo and Goswami, Bidyut B and Min, Seung Ki and Mondal, Sanjit Kumar}, issn = {2397-3722}, journal = {npj Climate and Atmospheric Science}, publisher = {Springer Nature}, title = {{Exploring causes of distinct regional and subseasonal Indian summer monsoon precipitation responses to CO2 removal}}, doi = {10.1038/s41612-024-00858-0}, volume = {7}, year = {2024}, } @misc{19307, abstract = {This repository contains the data, scripts, SAM codes and files required to reproduce the results of the manuscript "The Unreasonable Efficiency of Total Rain Evaporation Removal in Triggering Convective Self-Aggregation" submitted to the Geophysical Research Letters (GRL). Brief description of project: This project aims to examine the impact of rain evaporation removal or reduction in the planetary boundary layer (PBL) on convective self aggregation (CSA). Non-rotating radiative-convective equilibrium (RCE) simulations were conducted with the System for Atmospheric Modeling (SAM) cloud resolving model. Rain evaporation in the lowest 1 km was progressively reduced and the effect on CSA was investigated. The physical processes underlying this type of aggregation (referred to in the manuscript as no-evaporation CSA, or NE-CSA) were analyzed and described. The default SAM code base (version 6.10.8) can be downloaded from here: http://rossby.msrc.sunysb.edu/~marat/SAM.html}, author = {Hwong, Yi-Ling and Muller, Caroline J}, publisher = {Zenodo}, title = {{Data - The unreasonable efficiency of total rain evaporation removal in triggering convective self-aggregation}}, doi = {10.5281/ZENODO.10687169}, year = {2024}, } @article{17203, abstract = {The behavior of a rigid body primarily depends on its mass moments, which consist of the mass, center of mass, and moments of inertia. It is possible to manipulate these quantities without altering the geometric appearance of an object by introducing cavities in its interior. Algorithms that find cavities of suitable shapes and sizes have enabled the computational design of spinning tops, yo-yos, wheels, buoys, and statically balanced objects. Previous work is based, for example, on topology optimization on voxel grids, which introduces a large number of optimization variables and box constraints, or offset surface computation, which cannot guarantee that solutions to a feasible problem will always be found. In this work, we provide a mathematical analysis of constrained topology optimization problems that depend only on mass moments. This class of problems covers, among others, all applications mentioned above. Our main result is to show that no matter the outer shape of the rigid body to be optimized or the optimization objective and constraints considered, the optimal solution always features a quadric-shaped interface between material and cavities. This proves that optimal interfaces are always ellipsoids, hyperboloids, paraboloids, or one of a few degenerate cases, such as planes. This insight lets us replace a difficult topology optimization problem with a provably equivalent non-linear equation system in a small number (<10) of variables, which represent the coefficients of the quadric. This system can be solved in a few seconds for most examples, provides insights into the geometric structure of many specific applications, and lets us describe their solution properties. Finally, our method integrates seamlessly into modern fabrication workflows because our solutions are analytical surfaces that are native to the CAD domain.}, author = {Hafner, Christian and Ly, Mickaël and Wojtan, Christopher J}, issn = {1557-7368}, journal = {Transactions on Graphics}, keywords = {Topology Optimization, Mass Moments, Computational Geometry}, location = {Denver, Colorado}, number = {4}, publisher = {Association for Computing Machinery}, title = {{Spin-it faster: Quadrics solve all topology optimization problems that depend only on mass moments}}, doi = {10.1145/3658194}, volume = {43}, year = {2024}, } @article{15313, abstract = {Our goal is to investigate fundamental properties of the system of internally cooled convection. The system consists of an upward thermal flux at the lower boundary, a mean temperature lapse-rate and a constant cooling term in the bulk with the bulk cooling in thermal equilibrium with the input heat flux. This simple model represents idealised dry convection in the atmospheric boundary layer, where the cooling mimics the radiative cooling to space notably through longwave radiation. We perform linear stability analysis of the model for different values of the mean stratification to derive the critical forcing above which the fluid is convectively unstable to small perturbations. The dynamic behavior of the fluid system is described and the scaling of various important measured quantities such as the total vertical convective heat flux and the upward mass flux is measured. We introduce a lapse-rate dependent dimensionless Rayleigh-number Ray that determines the behavior of the system, finding that the convective heat-flux and mass-flux scale approximately as Ray0.5 and Ray0.7 respectively. The area-fraction of the domain that is occupied by upward and downward moving fluid and the skewness of the vertical velocity are studied to understand the asymmetry inherent in the system. We conclude with a short discussion on the relevance to atmospheric convection and the scope for further investigations of atmospheric convection using similar simplified approaches.}, author = {Agasthya, Lokahith N and Muller, Caroline J}, issn = {1007-5704}, journal = {Communications in Nonlinear Science and Numerical Simulation}, publisher = {Elsevier}, title = {{Dynamics and scaling of internally cooled convection}}, doi = {10.1016/j.cnsns.2024.108011}, volume = {134}, year = {2024}, } @inproceedings{17411, abstract = {We present PeFLL, a new personalized federated learning algorithm that improves over the state-of-the-art in three aspects: 1) it produces more accurate models, especially in the low-data regime, and not only for clients present during its training phase, but also for any that may emerge in the future; 2) it reduces the amount of on-client computation and client-server communication by providing future clients with ready-to-use personalized models that require no additional finetuning or optimization; 3) it comes with theoretical guarantees that establish generalization from the observed clients to future ones. At the core of PeFLL lies a learning-to-learn approach that jointly trains an embedding network and a hypernetwork. The embedding network is used to represent clients in a latent descriptor space in a way that reflects their similarity to each other. The hypernetwork takes as input such descriptors and outputs the parameters of fully personalized client models. In combination, both networks constitute a learning algorithm that achieves state-of-the-art performance in several personalized federated learning benchmarks}, author = {Scott, Jonathan A and Zakerinia, Hossein and Lampert, Christoph}, booktitle = {12th International Conference on Learning Representations}, location = {Vienna, Austria}, publisher = {OpenReview}, title = {{PEFLL: Personalized federated learning by learning to learn}}, year = {2024}, } @inproceedings{18120, abstract = {In practice, training using federated learning can be orders of magnitude slower than standard centralized training. This severely limits the amount of experimentation and tuning that can be done, making it challenging to obtain good performance on a given task. Server-side proxy data can be used to run training simulations, for instance for hyperparameter tuning. This can greatly speed up the training pipeline by reducing the number of tuning runs to be performed overall on the true clients. However, it is challenging to ensure that these simulations accurately reflect the dynamics of the real federated training. In particular, the proxy data used for simulations often comes as a single centralized dataset without a partition into distinct clients, and partitioning this data in a naive way can lead to simulations that poorly reflect real federated training. In this paper we address the challenge of how to partition centralized data in a way that reflects the statistical heterogeneity of the true federated clients. We propose a fully federated, theoretically justified, algorithm that efficiently learns the distribution of the true clients and observe improved server-side simulations when using the inferred distribution to create simulated clients from the centralized data.}, author = {Scott, Jonathan A and Cahill, Áine}, booktitle = {Proceedings of the 41st International Conference on Machine Learning}, issn = {2640-3498}, location = {Vienna, Austria}, pages = {44012--44037}, publisher = {ML Research Press}, title = {{Improved modelling of federated datasets using mixtures-of-Dirichlet-multinomials}}, volume = {235}, year = {2024}, } @inproceedings{18118, abstract = {We introduce a new framework for studying meta-learning methods using PAC-Bayesian theory. Its main advantage over previous work is that it allows for more flexibility in how the transfer of knowledge between tasks is realized. For previous approaches, this could only happen indirectly, by means of learning prior distributions over models. In contrast, the new generalization bounds that we prove express the process of meta-learning much more directly as learning the learning algorithm that should be used for future tasks. The flexibility of our framework makes it suitable to analyze a wide range of meta-learning mechanisms and even design new mechanisms. Other than our theoretical contributions we also show empirically that our framework improves the prediction quality in practical meta-learning mechanisms.}, author = {Zakerinia, Hossein and Behjati, Amin and Lampert, Christoph}, booktitle = {Proceedings of the 41st International Conference on Machine Learning}, issn = {2640-3498}, location = {Vienna, Austria}, pages = {58122--58139}, publisher = {ML Research Press}, title = {{More flexible PAC-Bayesian meta-learning by learning learning algorithms}}, volume = {235}, year = {2024}, } @inproceedings{17332, abstract = {Tree data structures, such as red-black trees, quad trees, treaps, or tries, are fundamental tools in computer science. A classical problem in concurrency is to obtain expressive, efficient, and scalable versions of practical tree data structures. We are interested in concurrent trees supporting range queries, i.e., queries that involve multiple consecutive data items. Existing implementations with this capability can list keys in a specific range, but do not support aggregate range queries: for instance, if we want to calculate the number of keys in a range, the only choice is to retrieve a whole list and return its size. This is suboptimal: in the sequential setting, one can augment a balanced search tree with counters and, consequently, perform these aggregate requests in logarithmic rather than linear time.In this paper, we propose a generic approach to implement a broad class of range queries on concurrent trees in a way that is wait-free, asymptotically efficient, and practically scalable. The key idea is a new mechanism for maintaining metadata concurrently at tree nodes, which can be seen as a wait-free variant of hand-over-hand locking (which we call hand-over-hand helping). We did a preliminary implementation of the wait-free binary search tree and preliminary experiments have indicated the soundness of our approach.}, author = {Kokorin, Ilya and Yudov, Victor and Aksenov, Vitaly and Alistarh, Dan-Adrian}, booktitle = {2024 IEEE International Parallel and Distributed Processing Symposium}, isbn = {9798350337662}, issn = {1530-2075}, location = {San Francisco, CA, United States}, pages = {169--179}, publisher = {IEEE}, title = {{Wait-free trees with asymptotically-efficient range queries}}, doi = {10.1109/IPDPS57955.2024.00023}, year = {2024}, } @inproceedings{15011, abstract = {Pruning large language models (LLMs) from the BERT family has emerged as a standard compression benchmark, and several pruning methods have been proposed for this task. The recent “Sparsity May Cry” (SMC) benchmark put into question the validity of all existing methods, exhibiting a more complex setup where many known pruning methods appear to fail. We revisit the question of accurate BERT-pruning during fine-tuning on downstream datasets, and propose a set of general guidelines for successful pruning, even on the challenging SMC benchmark. First, we perform a cost-vs-benefits analysis of pruning model components, such as the embeddings and the classification head; second, we provide a simple-yet-general way of scaling training, sparsification and learning rate schedules relative to the desired target sparsity; finally, we investigate the importance of proper parametrization for Knowledge Distillation in the context of LLMs. Our simple insights lead to state-of-the-art results, both on classic BERT-pruning benchmarks, as well as on the SMC benchmark, showing that even classic gradual magnitude pruning (GMP) can yield competitive results, with the right approach.}, author = {Kurtic, Eldar and Hoefler, Torsten and Alistarh, Dan-Adrian}, booktitle = {Proceedings of Machine Learning Research}, issn = {2640-3498}, location = {Hongkong, China}, pages = {542--553}, publisher = {ML Research Press}, title = {{How to prune your language model: Recovering accuracy on the "Sparsity May Cry" benchmark}}, volume = {234}, year = {2024}, } @inproceedings{18070, abstract = {Parallel SGD in a shared-memory setting is oft-represented by the popular Hogwild! algorithm, in which lock-free updates are asynchronously performed by multiple computing processes. Unfortunately, scaling Hogwild! to distributed workers is largely unexplored. Specifically, it is unknown if any adaptation of Hogwild! to the popular decentralized multi-GPU setting offers any competitive speedup, either empirically or theoretically. In this work, we investigate the potential of decentralizing Hogwild! by incorporating simultaneously (a) asynchronous local gradient updates on the shared memory of GPUs, and (b) non-blocking asynchronous decentralized federated averaging. A naive direct implementation shows degradation in performance, arising from scheduling overheads and concurrent write conflicts on GPUs. To mitigate these drawbacks, we investigate and propose a new method, based on careful block selection rules, which update only portions of the parameter vectors. Our experiments show that the resulting decentralized training method exhibits improved throughput and competitive accuracy for standard image classification benchmarks on the CIFAR-10, CIFAR-100, and Imagenet datasets. On the theoretical side, we prove that our method guarantees sublinear ergodic convergence rates for non-convex objectives.}, author = {Chatterjee, Bapi and Kungurtsev, Vyacheslav and Alistarh, Dan-Adrian}, booktitle = {Proceedings of the 44th International Conference on Distributed Computing Systems}, isbn = {9798350386059}, issn = {2575-8411}, location = {Jersey City, NJ, United States}, pages = {857--868}, publisher = {IEEE}, title = {{Federated SGD with local asynchrony}}, doi = {10.1109/ICDCS60910.2024.00084}, year = {2024}, } @inproceedings{17329, abstract = {We initiate the study of game dynamics in the population protocol model: n agents each maintain a current local strategy and interact in pairs uniformly at random. Upon each interaction, the agents play a two-person game and receive a payoff from an underlying utility function, and they can subsequently update their strategies according to a fixed local algorithm. In this setting, we ask how the distribution over agent strategies evolves over a sequence of interactions, and we introduce a new distributional equilibrium concept to quantify the quality of such distributions. As an initial example, we study a class of repeated prisoner's dilemma games, and we consider a family of simple local update algorithms that yield non-trivial dynamics over the distribution of agent strategies. We show that these dynamics are related to a new class of high-dimensional Ehrenfest random walks, and we derive exact characterizations of their stationary distributions, bounds on their mixing times, and prove their convergence to approximate distributional equilibria. Our results highlight trade-offs between the local state space of each agent, and the convergence rate and approximation factor of the underlying dynamics. Our approach opens the door towards the further characterization of equilibrium computation for other classes of games and dynamics in the population setting.}, author = {Alistarh, Dan-Adrian and Chatterjee, Krishnendu and Karrabi, Mehrdad and Lazarsfeld, John M}, booktitle = {Proceedings of the 43rd Annual ACM Symposium on Principles of Distributed Computing}, isbn = {9798400706684}, location = {Nantes, France}, pages = {40--49}, publisher = {Association for Computing Machinery}, title = {{Game dynamics and equilibrium computation in the population protocol model}}, doi = {10.1145/3662158.3662768}, year = {2024}, } @inproceedings{18121, abstract = {It is known that sparsity can improve interpretability for deep neural networks. However, existing methods in the area either require networks that are pre-trained with sparsity constraints, or impose sparsity after the fact, altering the network’s general behavior. In this paper, we demonstrate, for the first time, that sparsity can instead be incorporated into the interpretation process itself, as a sample-specific preprocessing step. Unlike previous work, this approach, which we call SPADE, does not place constraints on the trained model and does not affect its behavior during inference on the sample. Given a trained model and a target sample, SPADE uses sample-targeted pruning to provide a "trace" of the network’s execution on the sample, reducing the network to the most important connections prior to computing an interpretation. We demonstrate that preprocessing with SPADE significantly increases the accuracy of image saliency maps across several interpretability methods. Additionally, SPADE improves the usefulness of neuron visualizations, aiding humans in reasoning about network behavior. Our code is available at https://github.com/IST-DASLab/SPADE.}, author = {Moakhar, Arshia Soltani and Iofinova, Eugenia B and Frantar, Elias and Alistarh, Dan-Adrian}, booktitle = {Proceedings of the 41st International Conference on Machine Learning}, issn = {2640-3498}, location = {Vienna, Austria}, pages = {45955--45987}, publisher = {ML Research Press}, title = {{SPADE: Sparsity-guided debugging for deep neural networks}}, volume = {235}, year = {2024}, } @inproceedings{18976, abstract = {We analyze asynchronous-type algorithms for distributed SGD in the heterogeneous setting, where each worker has its own computation and communication speeds, as well as data distribution. In these algorithms, workers compute possibly stale and stochastic gradients associated with their local data at some iteration back in history and then return those gradients to the server without synchronizing with other workers. We present a unified convergence theory for non-convex smooth functions in the heterogeneous regime. The proposed analysis provides convergence for pure asynchronous SGD and its various modifications. Moreover, our theory explains what affects the convergence rate and what can be done to improve the performance of asynchronous algorithms. In particular, we introduce a novel asynchronous method based on worker shuffling. As a by-product of our analysis, we also demonstrate convergence guarantees for gradient-type algorithms such as SGD with random reshuffling and shuffle-once mini-batch SGD. The derived rates match the best-known results for those algorithms, highlighting the tightness of our approach. Finally, our numerical evaluations support theoretical findings and show the good practical performance of our method.}, author = {Islamov, Rustem and Safaryan, Mher and Alistarh, Dan-Adrian}, booktitle = {Proceedings of The 27th International Conference on Artificial Intelligence and Statistics}, issn = {2640-3498}, location = {Valencia, Spain}, pages = {649--657}, publisher = {ML Research Press}, title = {{AsGrad: A sharp unified analysis of asynchronous-SGD algorithms}}, volume = {238}, year = {2024}, } @inproceedings{18113, abstract = {The emergence of accurate open large language models (LLMs) has led to a race towards performant quantization techniques which can enable their execution on end-user devices. In this paper, we revisit the problem of “extreme” LLM compression—defined as targeting extremely low bit counts, such as 2 to 3 bits per parameter—from the point of view of classic methods in Multi-Codebook Quantization (MCQ). Our algorithm, called AQLM, generalizes the classic Additive Quantization (AQ) approach for information retrieval to advance the state-of-the-art in LLM compression, via two innovations: 1) learned additive quantization of weight matrices in input-adaptive fashion, and 2) joint optimization of codebook parameters across each transformer blocks. Broadly, AQLM is the first scheme that is Pareto optimal in terms of accuracy-vs-model-size when compressing to less than 3 bits per parameter, and significantly improves upon all known schemes in the extreme compression (2bit) regime. In addition, AQLM is practical: we provide fast GPU and CPU implementations of AQLM for token generation, which enable us to match or outperform optimized FP16 implementations for speed, while executing in a much smaller memory footprint.}, author = {Egiazarian, Vage and Panferov, Andrei and Kuznedelev, Denis and Frantar, Elias and Babenko, Artem and Alistarh, Dan-Adrian}, booktitle = {Proceedings of the 41st International Conference on Machine Learning}, issn = {2640-3498}, location = {Vienna, Austria}, pages = {12284--12303}, publisher = {ML Research Press}, title = {{Extreme compression of large language models via additive quantization}}, volume = {235}, year = {2024}, } @inproceedings{18975, abstract = {Leveraging second-order information about the loss at the scale of deep networks is one of the main lines of approach for improving the performance of current optimizers for deep learning. Yet, existing approaches for accurate full-matrix preconditioning, such as Full-Matrix Adagrad (GGT) or Matrix-Free Approximate Curvature (M-FAC) suffer from massive storage costs when applied even to small-scale models, as they must store a sliding window of gradients, whose memory requirements are multiplicative in the model dimension. In this paper, we address this issue via a novel and efficient error-feedback technique that can be applied to compress preconditioners by up to two orders of magnitude in practice, without loss of convergence. Specifically, our approach compresses the gradient information via sparsification or low-rank compression before it is fed into the preconditioner, feeding the compression error back into future iterations. Extensive experiments on deep neural networks show that this approach can compress full-matrix preconditioners to up to 99% sparsity without accuracy loss, effectively removing the memory overhead of fullmatrix preconditioners such as GGT and M-FAC.}, author = {Modoranu, Ionut-Vlad and Kalinov, Aleksei and Kurtic, Eldar and Frantar, Elias and Alistarh, Dan-Adrian}, booktitle = {41st International Conference on Machine Learning}, issn = {2640-3498}, location = {Vienna, Austria}, pages = {35910--35933}, publisher = {ML Research Press}, title = {{Error feedback can accurately compress preconditioners}}, volume = {235}, year = {2024}, } @inproceedings{18977, abstract = {Recent advances in large language model (LLM) pretraining have led to high-quality LLMs with impressive abilities. By compressing such LLMs via quantization to 3-4 bits per parameter, they can fit into memory-limited devices such as laptops and mobile phones, enabling personalized use. Quantizing models to 3-4 bits per parameter can lead to moderate to high accuracy losses, especially for smaller models (1-10B parameters), which are suitable for edge deployment. To address this accuracy issue, we introduce the Sparse-Quantized Representation (SpQR), a new compressed format and quantization technique that enables for the first time \emph{near-lossless} compression of LLMs across model scales while reaching similar compression levels to previous methods. SpQR works by identifying and isolating \emph{outlier weights}, which cause particularly large quantization errors, and storing them in higher precision while compressing all other weights to 3-4 bits, and achieves relative accuracy losses of less than in perplexity for highly-accurate LLaMA and Falcon LLMs. This makes it possible to run a 33B parameter LLM on a single 24 GB consumer GPU without performance degradation at 15% speedup, thus making powerful LLMs available to consumers without any downsides. SpQR comes with efficient algorithms for both encoding weights into its format, as well as decoding them efficiently at runtime. Specifically, we provide an efficient GPU inference algorithm for SpQR, which yields faster inference than 16-bit baselines at similar accuracy while enabling memory compression gains of more than 4x.}, author = {Dettmers, Tim and Svirschevski, Ruslan A. and Egiazarian, Vage and Kuznedelev, Denis and Frantar, Elias and Ashkboos, Saleh and Borzunov, Alexander and Hoefler, Torsten and Alistarh, Dan-Adrian}, booktitle = {12th International Conference on Learning Representations}, location = {Vienna, Austria}, publisher = {OpenReview}, title = {{SpQR: A sparse-quantized representation for near-lossless LLM weight compression}}, year = {2024}, } @inproceedings{17093, abstract = {Federated Learning (FL) enables large-scale distributed training of machine learning models, while still allowing individual nodes to maintain data locally. However, executing FL at scale comes with inherent practical challenges: 1) heterogeneity of the local node data distributions, 2) heterogeneity of node computational speeds (asynchrony), but also 3) constraints in the amount of communication between the clients and the server. In this work, we present the first variant of the classic federated averaging (FedAvg) algorithm which, at the same time, supports data heterogeneity, partial client asynchrony, and communication compression. Our algorithm comes with a novel, rigorous analysis showing that, in spite of these system relaxations, it can provide similar convergence to FedAvg in interesting parameter regimes. Experimental results in the rigorous LEAF benchmark on setups of up to 300 nodes show that our algorithm ensures fast convergence for standard federated tasks, improving upon prior quantized and asynchronous approaches.}, author = {Zakerinia, Hossein and Talaei, Shayan and Nadiradze, Giorgi and Alistarh, Dan-Adrian}, booktitle = {Proceedings of the 27th International Conference on Artificial Intelligence and Statistics}, issn = {2640-3498}, location = {Valencia, Spain}, pages = {3448--3456}, publisher = {ML Research Press}, title = {{Communication-efficient federated learning with data and client heterogeneity}}, volume = {238}, year = {2024}, } @inproceedings{18117, abstract = {We investigate parameter-efficient fine-tuning (PEFT) methods that can provide good accuracy under limited computational and memory budgets in the context of large language models (LLMs). We present a new PEFT method called Robust Adaptation (RoSA) inspired by robust principal component analysis that jointly trains low-rank and highly-sparse components on top of a set of fixed pretrained weights to efficiently approximate the performance of a full-fine-tuning (FFT) solution. Across a series of challenging generative tasks such as grade-school math and SQL query generation, which require fine-tuning for good performance, we show that RoSA outperforms LoRA, pure sparse fine-tuning, and alternative hybrid methods at the same parameter budget, and can even recover the performance of FFT on some tasks. We provide system support for RoSA to complement the training algorithm, specifically in the form of sparse GPU kernels which enable memory- and computationally-efficient training, and show that it is also compatible with low-precision base weights, resulting in the first joint representation combining quantization, low-rank and sparse approximations. Our code is available at https://github.com/IST-DASLab/RoSA.}, author = {Nikdan, Mahdi and Tabesh, Soroush and Crncevic, Elvir and Alistarh, Dan-Adrian}, booktitle = {Proceedings of the 41st International Conference on Machine Learning}, issn = {2640-3498}, location = {Vienna, Austria}, pages = {38187--38206}, publisher = {ML Research Press}, title = {{RoSA: Accurate parameter-efficient fine-tuning via robust adaptation}}, volume = {235}, year = {2024}, } @article{15315, abstract = {Single and collective cell migration are fundamental processes critical for physiological phenomena ranging from embryonic development and immune response to wound healing and cancer metastasis. To understand cell migration from a physical perspective, a broad variety of models for the underlying physical mechanisms that govern cell motility have been developed. A key challenge in the development of such models is how to connect them to experimental observations, which often exhibit complex stochastic behaviours. In this review, we discuss recent advances in data-driven theoretical approaches that directly connect with experimental data to infer dynamical models of stochastic cell migration. Leveraging advances in nanofabrication, image analysis, and tracking technology, experimental studies now provide unprecedented large datasets on cellular dynamics. In parallel, theoretical efforts have been directed towards integrating such datasets into physical models from the single cell to the tissue scale with the aim of conceptualising the emergent behaviour of cells. We first review how this inference problem has been addressed in both freely migrating and confined cells. Next, we discuss why these dynamics typically take the form of underdamped stochastic equations of motion, and how such equations can be inferred from data. We then review applications of data-driven inference and machine learning approaches to heterogeneity in cell behaviour, subcellular degrees of freedom, and to the collective dynamics of multicellular systems. Across these applications, we emphasise how data-driven methods can be integrated with physical active matter models of migrating cells, and help reveal how underlying molecular mechanisms control cell behaviour. Together, these data-driven approaches are a promising avenue for building physical models of cell migration directly from experimental data, and for providing conceptual links between different length-scales of description.}, author = {Brückner, David and Broedersz, Chase P.}, issn = {1361-6633}, journal = {Reports on Progress in Physics}, number = {5}, publisher = {IOP Publishing}, title = {{Learning dynamical models of single and collective cell migration: a review}}, doi = {10.1088/1361-6633/ad36d2}, volume = {87}, year = {2024}, }