@inproceedings{17411, abstract = {We present PeFLL, a new personalized federated learning algorithm that improves over the state-of-the-art in three aspects: 1) it produces more accurate models, especially in the low-data regime, and not only for clients present during its training phase, but also for any that may emerge in the future; 2) it reduces the amount of on-client computation and client-server communication by providing future clients with ready-to-use personalized models that require no additional finetuning or optimization; 3) it comes with theoretical guarantees that establish generalization from the observed clients to future ones. At the core of PeFLL lies a learning-to-learn approach that jointly trains an embedding network and a hypernetwork. The embedding network is used to represent clients in a latent descriptor space in a way that reflects their similarity to each other. The hypernetwork takes as input such descriptors and outputs the parameters of fully personalized client models. In combination, both networks constitute a learning algorithm that achieves state-of-the-art performance in several personalized federated learning benchmarks}, author = {Scott, Jonathan A and Zakerinia, Hossein and Lampert, Christoph}, booktitle = {12th International Conference on Learning Representations}, location = {Vienna, Austria}, publisher = {OpenReview}, title = {{PEFLL: Personalized federated learning by learning to learn}}, year = {2024}, } @phdthesis{17490, abstract = {Deep learning is essential in numerous applications nowadays, with many recent advancements made possible by training very large models. Despite their broad applicability, training neural networks is often time-intensive, and it is usually impractical to manage large models and datasets on a single machine. To address these issues, distributed deep learning training has become increasingly important. However, distributed training requires synchronization among nodes, and the mini-batch stochastic gradient descent algorithm places a significant load on network connections. A possible solution to tackle the synchronization bottleneck is to reduce a message size by lossy compression. In this thesis, we investigate systems and algorithmic approaches to communication compression during training. From the systems perspective, we demonstrate that a common approach of expensive hardware overprovisioning can be replaced through a thorough system design. We introduce a framework that introduces efficient software support for compressed communication in machine learning applications, applicable to both multi-GPU single-node training and larger-scale multi-node training. Our framework integrates with popular ML frameworks, providing up to 3x speedups for multi-GPU nodes based on commodity hardware and order-of-magnitude improvements in the multi-node setting, with negligible impact on accuracy. Also, we consider an application of our framework to different communication schemes, such as Fully Sharded Data Parallel. We provide strong convergence guarantees for the compression in such a setup. Empirical validation shows that our method preserves model accuracy for GPT-family models with up to 1.3 billion parameters, while completely removing the communication bottlenecks of non-compressed alternatives, providing up to 2.2x speedups end-to-end. From the algorithmic side, we propose a general framework that dynamically adjusts the degree of compression across a model's layers during training. This approach enhances overall compression and results in significant speedups without compromising accuracy. Our algorithm utilizes an adaptive algorithm that automatically selects the optimal compression parameters for model layers, ensuring the best compression ratio while adhering to an error constraint. Our method is effective across all existing families of compression methods. It achieves up to 2.5x faster training and up to a 5x improvement in compression compared to efficient implementations of current approaches. Additionally, LGreCo can complement existing adaptive algorithms. }, author = {Markov, Ilia}, issn = {2663-337X}, pages = {102}, publisher = {Institute of Science and Technology Austria}, title = {{Communication-efficient distributed training of deep neural networks : An algorithms and systems perspective}}, doi = {10.15479/at:ista:17490}, year = {2024}, } @phdthesis{17850, abstract = {Understanding the relationship between a given phenotype and its underlying genotype or genotypes is one of the most pressing challenges of biology, as it lies at the heart of not only basic understanding of evolutionary theory, but also of practical applications in medicine and bioengineering. Understanding this relationship is complicated by the ubiquitous phenomenon of epistasis, wherein mutation effects are dependent on their genetic context. Fitness landscapes — representations of phenotype as a function of genotype — are being increasingly used as a tool to study the effects and interactions of thousands of mutations, but are experimentally limited to exploring a small fraction of a protein’s theoretical sequence space. Furthermore, not all regions of said sequence space are necessarily equally informative. Thus, gene selection for landscape surveys should be carefully considered in order to maximize the usable output of necessarily limited data. In this work, we analyzed the fitness landscapes of orthologous green fluorescent proteins from four different species, by systematically measuring the phenotype, fluorescence, of tens of thousands of mutant genotypes from each protein. These landscapes were highly heterogeneous, with some genes being mutationally robust and displaying epistasis only rarely, and others being highly epistatic and mutationally fragile. We used this data to train machine learning models to predict fluorescence from genotype. Although the training data contained almost exclusively genotypes with less than 3% sequence divergence from the original wild-type sequences, we were able to create novel, functional genotypes with up to 20% sequence divergence. Counterintuitively however, genes with high mutational robustness and rare epistasis were more difficult to introduce large numbers of mutations into, not less. This represents the first study of large-scale fitness landscapes of a protein family, and provides insights into how to approach future landscape surveys and their applications in novel protein design.}, author = {Gonzalez Somermeyer, Louisa}, issn = {2663-337X}, pages = {89}, publisher = {Institute of Science and Technology Austria}, title = {{Fitness landscapes of orthologous green fluorescent proteins}}, doi = {10.15479/at:ista:17850}, year = {2024}, } @phdthesis{17485, abstract = {Large language models (LLMs) have made tremendous progress in the past few years, from being able to generate coherent text to matching or surpassing humans in a wide variety of creative, knowledge or reasoning tasks. Much of this can be attributed to massively increased scale, both in the size of the model as well as the amount of training data, from 100s of millions to 100s of billions, or even trillions. This trend is expected to continue, which, although exciting, also raises major practical concerns. Already today's 100+ billion parameter LLMs require top-of-the-line hardware just to run. Hence, it is clear that sustaining these developments will require significant efficiency advances. Historically, one of the most practical ways of improving model efficiency has been compression, especially in the form of sparsity or quantization. While this has been studied extensively in the past, existing accurate methods are all designed for models around 100 million parameters; scaling them up to ones literally 1000x larger is highly challenging. In this thesis, we introduce a new unified sparsification and quantization approach OBC, which through additional algorithmic enhancements leads to GPTQ and SparseGPT, the first techniques fast and accurate enough to compress 100+ billion parameter models to 4- or even 3-bit precision and 50% weight-sparsity, respectively. Additionally, we show how weight-only quantizion does not just bring space savings but also up to 4.5x faster generation speed, via custom GPU kernels. In fact, we show for the first time that it is possible to develop an FP16 times INT4 mixed-precision matrix multiplication kernel, called Marlin, which comes close to simultaneously maximizing both memory and compute utilization, making weight-only quantization highly practical even for multi-user serving. Further, we demonstrate that GPTQ can be scaled to widely overparametrized trillion-parameter models, where extreme sub-1-bit compression rates can be achieved without any inference slow-down, by co-designing a bespoke entropy coding scheme together with an efficient kernel. Finally, we also study compression from the perspective of someone with access to massive amounts of compute resources for training large models completely from scratch. Here the key questions evolve around the joint scaling behavior between compression, model size, and amount of training data used. Based on extensive experimental results for both vision and text models, we introduce the first scaling law which accurately captures the relationship between weight-sparsity, number of non-zero weights and data. This further allows us to characterize the optimal sparsity, which we find to increase the longer a fixed cost model is being trained. Overall, this thesis presents contributions to three different angles of large model efficiency: affordable but accurate algorithms, highly efficient systems implementations, and fundamental scaling laws for compressed training.}, author = {Frantar, Elias}, issn = {2663-337X}, pages = {129}, publisher = {Institute of Science and Technology Austria}, title = {{Compressing large neural networks : Algorithms, systems and scaling laws}}, doi = {10.15479/at:ista:17485}, year = {2024}, } @inproceedings{17147, abstract = {Efficient utilization of large-scale biobank data is crucial for inferring the genetic basis of disease and predicting health outcomes from the DNA. Yet we lack efficient, accurate methods that scale to data where electronic health records are linked to whole genome sequence information. To address this issue, our paper develops a new algorithmic paradigm based on Approximate Message Passing (AMP), which is specifically tailored for genomic prediction and association testing. Our method yields comparable out-of-sample prediction accuracy to the state of the art on UK Biobank traits, whilst dramatically improving computational complexity, with a 8x-speed up in the run time. In addition, AMP theory provides a joint association testing framework, which outperforms the currently used REGENIE method, in roughly a third of the compute time. This first, truly large-scale application of the AMP framework lays the foundations for a far wider range of statistical analyses for hundreds of millions of variables measured on millions of people.}, author = {Depope, Al and Mondelli, Marco and Robinson, Matthew Richard}, booktitle = {2024 IEEE International Conference on Acoustics, Speech, and Signal Processing}, isbn = {9798350344851}, issn = {1520-6149}, location = {Seoul, Korea}, pages = {13151--13155}, publisher = {IEEE}, title = {{Inference of genetic effects via approximate message passing}}, doi = {10.1109/ICASSP48485.2024.10447198}, year = {2024}, } @phdthesis{18471, abstract = {Spatial omics technologies are enriching our understanding of complex biological samples, by allowing us to study their molecular composition while preserving the spatial relationships between molecules in their native context. As the field continues to advance, there are technical challenges that need to be addressed in order to take full advantage of the spatial capabilities of these methods. In this work, I present two technical developments that I established for multiplexed error robust FISH (MERFISH) throughout my PhD: (1) pushing the spatial resolution limits to the nanoscale, and (2) adding rich tissue context to the mouse brain transcriptome. To achieve nanoscale resolution with MERFISH in cultured cells, I combined it with stimulated emission depletion (STED) and expansion microscopy (ExM) to achieve a spatial resolution as low as ~20 nm, and explored the compatibility of MERFISH with singlemolecule localization microscopy (SMLM) techniques. To visualize targeted mRNAs in mouse brain tissue, I applied the comprehensive analysis of tissues across scales (CATS) toolbox, which provides an unbiased morphological readout by labeling the extracellular domain. I successfully established this method, which we call CATS-MERFISH-ExM, to work with thick mouse brain slices, being able to extract transcriptomics information with 3D tissue context. CATS-MERFISH-ExM enabled us to identify cell types and further visualize the subcellular distribution of transcripts in mouse brain tissue, shedding light on the neuropil-specific transcriptome. This method provides integrated information on cellular structure and transcriptomes in situ, and could potentially be applied with other modalities, opening new avenues for scientific discovery. }, author = {Agudelo Duenas, Nathalie}, isbn = {978-3-99078-044-2}, issn = {2663-337X}, pages = {97}, publisher = {Institute of Science and Technology Austria}, title = {{Visualizing the neuronal transcriptional landscape with tissue context}}, doi = {10.15479/at:ista:18471}, year = {2024}, } @phdthesis{18574, abstract = {Biological vision is unlike a camera; rather than transmitting light information faithfully, early visual circuits process the visual scene to convey only the relevant information in an efficient manner. Consequentially, the nature of this visual processing then depends on what is the relevant information in a scene and on the notion of efficiency. In this work, I study how visual processing is modulated by two different variations in the visual scene. First, I discovered that in the mouse (Mus musculus) retina, Retinal Ganglion Cells in the upper and lower visual field have differences in the center surround structure of their receptive fields. Comparison with models of efficient coding show that this adaptation likely evolved to cope with the brightness gradient from the sky to the ground that is pervasive in natural scenes. In the second project, I study how the downstream neurons in the Superior Colliculus dynamically change their temporal selectivity depending on the ambient luminance and behavioral state. As the scene gets darker or when the animal is is less aroused, the neuronal responses get laggier, while still maintaining their relative timing with respect to the population. Overall, this work emphasises the need to understand visual processing in the context of specific demands of the animal in its the environment. The adaptive changes in the visual system, from the retinal ganglion cells to the superior colliculus, highlight the intricate ways in which biological vision optimizes the processing of visual information. }, author = {Gupta, Divyansh}, isbn = {978-3-99078-050-3}, issn = {2663-337X}, pages = {86}, publisher = {Institute of Science and Technology Austria}, title = {{Visual adaptations to natural statistics}}, doi = {10.15479/at:ista:18574}, year = {2024}, } @article{15323, abstract = {Supercomplexes of the respiratory chain are established constituents of the oxidative phosphorylation system, but their role in mammalian metabolism has been hotly debated. Although recent studies have shown that different tissues/organs are equipped with specific sets of supercomplexes, depending on their metabolic needs, the notion that supercomplexes have a role in the regulation of metabolism has been challenged. However, irrespective of the mechanistic conclusions, the composition of various high molecular weight supercomplexes remains uncertain. Here, using cryogenic electron microscopy, we demonstrate that mammalian (mouse) tissues contain three defined types of ‘respirasome’, supercomplexes made of CI, CIII2 and CIV. The stoichiometry and position of CIV differs in the three respirasomes, of which only one contains the supercomplex-associated factor SCAF1, whose involvement in respirasome formation has long been contended. Our structures confirm that the ‘canonical’ respirasome (the C-respirasome, CICIII2CIV) does not contain SCAF1, which is instead associated to a different respirasome (the CS-respirasome), containing a second copy of CIV. We also identify an alternative respirasome (A-respirasome), with CIV bound to the ‘back’ of CI, instead of the ‘toe’. This structural characterization of mouse mitochondrial supercomplexes allows us to hypothesize a mechanistic basis for their specific role in different metabolic conditions.}, author = {Vercellino, Irene and Sazanov, Leonid A}, issn = {1545-9985}, journal = {Nature Structural and Molecular Biology}, pages = {1061--1071}, publisher = {Springer Nature}, title = {{SCAF1 drives the compositional diversity of mammalian respirasomes}}, doi = {10.1038/s41594-024-01255-0}, volume = {31}, year = {2024}, } @phdthesis{18531, abstract = {Sex chromosomes and autosomes exhibit very different evolutionary dynamics. The Y chromosome usually degenerates, leaving many X-linked loci hemizygous in males. Since recessive X-linked mutations are always exposed to selection in males, selection is more efficient on the X chromosome than on autosomes on recessive mutations, leading to faster adaptation on the X chromosome than other genomic regions, if beneficial mutations are on average recessive (known as the Faster-X effect). In the presence of the functional, but non-recombining gametolog on the Y (as is often the case in young non-recombining regions), recessive mutations are sheltered from selection on the X chromosome. We model this scenario and show that the efficiency of selection is reduced on diploid X loci due to sheltering by the Y chromosome. Reduced efficiency of selection leads to slower adaptation and increased accumulation of deleterious mutations (Slower-X effect). We extended this model to explore the effect of sex-specific selection on degeneration of sex chromosomes, showing theoretically that male-limited genes degenerate on the X chromosome and female-biased genes degenerate on the Y chromosome. This prediction depends on the effective population size and the mutation rate, explaining the variety of sex chromosome degeneration patterns observed in nature. To test for direct evidence of a Slower-X (or Slower-Z) effect, we analyzed the ZW sex chromosomes of the flatworm Schistosoma japonicum, which have a very young non-recombining region with non-degenerated W. Diploid Z-linked genes have higher ratios of non-synonymous to synonymous polymorphisms than autosomal genes, supporting reduced efficiency of selection on the diploid Z region. These results provide evidence of sheltering by the W chromosome, a mechanism that could contribute to Z (X) chromosome degeneration, and illustrate contrasting evolutionary patterns in old and young sex chromosome regions. In addition, genes with sexspecific patterns of expression show opposite patterns of selection in the young (diploid) and old (hemizygous) Z, showing the complex manner in which sex-specific selection shapes the evolutionary patterns of sex chromosomes. }, author = {Mrnjavac, Andrea}, issn = {2663-337X}, keywords = {Sex chromosomes, evolution, selection, sheltering}, pages = {181}, publisher = {Institute of Science and Technology Austria}, title = {{Early stages of sex chromosome evolution}}, doi = {10.15479/at:ista:18531}, year = {2024}, } @unpublished{18648, abstract = {Statistical causal learning in genomics relies on the instrumental variable method of Mendelian Randomization (MR). Currently, an overwhelming number of MR studies purport to show causal relationships among a wide range of risk factors and outcomes. Here, we show that selecting instrument variables from genome-wide association study estimates leads to high false discovery rates for many MR approaches, which can be greatly reduced by employing a graphical inference approach which: (i) explicitly tests instrumental variable assumptions; (ii) distinguishes direct from indirect factors in very high-dimensional data; (iii) discriminates pleiotropic from trait-specific markers, controlling for LD genome-wide; (iv) accommodates rare variants and binary outcomes in a principled way; and (v) identifies potential unobserved latent confounding. For 17 traits and 8.4M variants recorded for 458,747 individuals in the UK Biobank, we show that standard MR analysis gives an abundance of findings that disappear under stringent assumption checks, with many relationships reflecting potential unmeasured confounding. This implies that mixtures of temporal precedence and potential for reverse-causality prohibit understanding the underlying nature of phenotypic and genetic correlations in biobank data. We propose that well-curated longitudinal records are likely needed and that our approach provides a first-step toward robust principled screening for potential causal links. }, author = {Machnik, Nick N and Mahmoudi, Seyed Mahdi and Borczyk, Malgorzata and Krätschmer, Ilse and Bauer, Markus J. and Robinson, Matthew Richard}, booktitle = {bioRxiv}, title = {{Causal inference for multiple risk factors and diseases from genomics data}}, doi = {10.1101/2023.12.06.570392}, year = {2024}, } @phdthesis{17206, abstract = {Males and females exhibit numerous differences, from the initial stages of sex determination to the development of secondary sexual characteristics. In Drosophila, these differences have been thoroughly studied. Extensive research has been performed to understand the role and molecular mode of action of central sex in determining switch genes, such as transformer (tra) and Sex-lethal (Sxl). Furthermore, studies have highlighted differential gene expression as an essential mechanism to create sexual dimorphism. An alternative path to sexual dimorphism that has been less explored is alternative splicing, the mechanism through which genes can produce multiple transcripts with distinct properties and functions. The primary switch sex-determining gene Sxl is a good example of the role of alternative splicing for sex-specific functions: the inclusion of a specific exon determines the male or female form of the protein, which in turn switches on either the male or female developmental pathway. The genes that act upstream of Sxl and determine which form is expressed - the counter genes - have received less attention. This thesis addresses two critical questions about the molecular encoding of sexes in the Drosophila melanogaster genome: First, the use of splice forms in male and female tissues in D. melanogaster is examined, inferring the molecular and evolutionary parameters shaping the diversity of the splicing landscape. Second, the behaviour of counter genes in Drosophila-related species is investigated, shedding light on potential changes leading to their incorporation into the sex-determination pathway. For the alternative splicing analyses, long-read RNA sequencing of testes, ovaries, female and male midguts, heads, and whole bodies was performed. A novel pipeline was developed to assign unique transcript identifiers for each sequence of exons and introns in the read, enabling detailed comparisons of splicing variants in each tissue/sex. Alternative splicing was found to be more pervasive in females than males (22,201 exclusive splice forms in females versus 12,631 in males), especially when comparing ovaries to other tissues. The ovaries alone displayed 15,299 exclusive splice forms, suggesting most female exclusive splice forms originate there. Genome location and gene age were also correlated with the number of splice forms per gene. In particular, the X and 4th chromosomes (Muller elements A and F) showed more splice forms per gene than other chromosomes. Additionally, genes older than 63 million years exhibited more splice forms per gene than younger genes. Our results suggest that alternative splicing is more prevalent than previously believed, with numerous female-exclusive forms, age, and location playing significant roles in shaping its prevalence. For the counter genes analyses, we combined published gene expression, genomic, and gene interaction data from various clades (Bactrocera jarvisi, B. oleae, Ceratitis capitata, Mus musculus, Caenorhabditis elegans, Homo sapiens, and D. melanogaster). The counter genes scute (sc), extra macrochaetae (emc), groucho (gro), deadpan (dpn), daughterless (da), runt (run), Sxl, hermaphrodite (her), and tra maintain conserved Muller element locations between C. capitata and D. melanogaster, which are most of the counter genes identified in the C. capitata genome. Their expression patterns during early embryogenesis in B. jarvisi and D. melanogaster are also similar for counter genes dpn, gro, da, and emc. However, Sxl and sc are also found to have more extreme expression ratios between the species. Lastly, gene interactions within the counter genes are conserved, with da-sc and gro-dpn interactions occurring in Drosophila, worms, humans, and mice. Interactions such as dpn-sc, dpn-da, da-emc, and gro-run are present in Drosophila, mice, and humans, suggesting these genes were recruited by ancestral characteristics, primarily during embryogenesis. The conserved expression, location, and interactions of counter genes suggest serendipitous recruitment of such genes instead of a change in those characteristics as they were recruited for this function. }, author = {Raices, Julia}, issn = {2663-337X}, pages = {82}, publisher = {Institute of Science and Technology Austria}, title = {{Novel approaches to studying alternative splicing in Drosophila Melanogaster : Insights into sex-specific gene expression and the evolution of sex determination}}, doi = {10.15479/at:ista:17206}, year = {2024}, } @phdthesis{18101, abstract = {The Retroviridae family consists of two sub-families, the Orthoretrovirinae and the Spumaretrovirinae. The Orthoretroviruses contain important human pathogens, such as the human immunodeficiency virus 1 (HIV-1). They also harbor other retrovirus species which are regularly used as model systems to study the retroviral life cycle. The main structural component of the retroviruses, is the Gag protein and its truncation derivatives occurring during viral maturation. Orthoretroviral Gag assemblies have been extensively studied to understand the interactions that confer stability and morphology to viral particles. The Spumaretrovirinae subfamily represent an early diverging branch of the Retroviridae. Its members, the Foamy viruses (FV), share most of the conventional features found in retroviruses. However, they also possess multiple characteristics that make them unique. In particular, FV Gag does not get extensively cleaved as in orthoretroviruses. Hence, the Gag architecture deviates from the canonical domain arrangement in FV. They also exhibit a peculiar particle morphology, having no apparent immature state and a seemingly icosahedral mature particle. Due to this, many fundamental questions on FV structural assembly mechanisms remain open. To answer these questions, was the main focus of this thesis. Mainly, it is not known how FV assemble their core in a virus particle and what are the important assembly interaction sites within said core. What is the minimum assembly competent domain of FV Gag? Is there a morphological change in the assembly type of FVGag lattices? If so, what is defining these morphological shifts? Finally, it would be interesting to know what is the evolutionary relationship between FV and the rest of the retrotranscribing elements, from a structural point of view? To answer these questions, membrane-enveloped mammalian cell-derived FV virus-like particles (VLPs) were produced. Cryo-electron tomography (cryo-ET) analysis suggested these FV VLPs do not form a canonical retroviral Gag lattice structure, which is in line with earlier observations. To further evaluate FV Gag assembly competence and morphology, the first bacterial cell-derived in vitro VLP assembly system was designed and optimized. Using this system with different truncation variants, the minimum assembly competent domain of FV Gag was found to be the putative CA300-477 domain. Varying VLP morphologies were also observed and strongly suggested residues upstream of CA300-477 play a role in morphology determination. Finally, a combined cryo-electron microscopy (cryoEM) and cryo-ET approach was taken to analyze tubular assemblies from the minimal assembly competent domain. This revealed an unexpectedly unique non-canonical assembly architecture. Three novel lattice stabilizing interfaces were described which proved to be as unique as the lattice arrangement. Comparison to a newly published FV CA core structure revealed the CA-CA interactions in the atypical assembly do not recapitulate what is described for the FV core lattice. However, the new in vitro VLP assembly system obtained in this thesis also provides an exciting opportunity to study still unresolved FV assembly features in a potentially facilitated approach compared to conventional methods. In summary, this work provided a deeper understanding of the basic FV Gag assembly unit, as well as presenting the first FV Gag-derived in vitro VLP assembly system. This system reveals a novel and unique assembly architecture among retroviral in vitro assemblies.}, author = {Porley, Dario J}, isbn = {978-3-99078-041-1}, issn = {2663-337X}, pages = {131}, publisher = {Institute of Science and Technology Austria}, title = {{Structural characterization of spumavirus capsid assemblies}}, doi = {10.15479/at:ista:18101}, year = {2024}, } @phdthesis{17465, abstract = {In the modern age of machine learning, artificial neural networks have become an integral part of many practical systems. One of the key ingredients of the success of the deep learning approach is recent computational advances which allowed the training of models with billions of parameters on large-scale data. Such over-parameterized and data-hungry regimes pose a challenge for the theoretical analysis of modern models since “classical” statistical wisdom is no longer applicable. In this view, it is paramount to extend or develop new machinery that will allow tackling the neural network analysis under new challenging asymptotic regimes, which is the focus of this thesis. Large neural network systems are usually optimized via “local” search algorithms, such as stochastic gradient descent (SGD). However, given the high-dimensional nature of the parameter space, it is a priori not clear why such a crude “local” approach works so remarkably well in practice. We take a step towards demystifying this phenomenon by showing that the landscape of the SGD training dynamics exhibits a few beneficial properties for the optimization. First, we show that along the SGD trajectory an over-parameterized network is dropout stable. The emergence of dropout stability allows to conclude that the minima found by SGD are connected via a continuous path of small loss. This in turn means that the high-dimensional landscape of the neural network optimization problem is provably not so unfavourable to gradient-based training, due to mode connectivity. Next, we show that SGD for an over-parameterized network tends to find solutions that are functionally more “simple”. This in turn means that the SGD minima are more robust, since a less complicated solution will less likely overfit the data. More formally, for a prototypical example of a wide two-layer ReLU network on a 1d regression task we show that the SGD algorithm is implicitly selective in its choice of an interpolating solution. Namely, at convergence the neural network implements a piece-wise linear function with the number of linear regions depending only on the amount of training data. This is in contrast to a “smooth”-like behaviour which one would expect given such a severe over-parameterization of the model. Diverging from the generic supervised setting of classification and regression problems, we analyze an auto-encoder model that is commonly used for representation learning and data compression. Despite the wide applicability of the auto-encoding paradigm, the theoretical understanding of their behaviour is limited even in the simplistic shallow case. The related work is restricted to extreme asymptotic regimes in which the auto-encoder is either severely over-parameterized or under-parameterized. In contrast, we provide a tight characterization for the 1-bit compression of Gaussian signals in the challenging proportional regime, i.e., the input dimension and the size of the compressed representation obey the same asymptotics. We also show that gradient-based methods are able to find a globally optimal solution and that the predictions made for Gaussian data extrapolate beyond - to the case of compression of natural images. Next, we relax the Gaussian assumption and study more structured input sources. We show that the shallow model is sometimes agnostic to the structure of the data vii which results in a Gaussian-like behaviour. We prove that making the decoding component slightly less shallow is already enough to escape the “curse” of Gaussian performance. }, author = {Shevchenko, Aleksandr}, issn = {2663-337X}, pages = {232}, publisher = {Institute of Science and Technology Austria}, title = {{High-dimensional limits in artificial neural networks}}, doi = {10.15479/at:ista:17465}, year = {2024}, } @article{17890, abstract = {Our understanding of the molecular pathways that regulate oogenesis and define cellular identity in the Arthropod female reproductive system and the extent of their conservation is currently very limited. This is due to the focus on model systems, including Drosophila and Daphnia, which do not reflect the observed diversity of morphologies, reproductive modes, and sex chromosome systems. We use single-nucleus RNA and ATAC sequencing to produce a comprehensive single nucleus atlas of the adult Artemia franciscana female reproductive system. We map our data to the Fly Cell Atlas single-nucleus dataset of the Drosophila melanogaster ovary, shedding light on the conserved regulatory programs between the two distantly related Arthropod species. We identify the major cell types known to be present in the Artemia ovary, including germ cells, follicle cells, and ovarian muscle cells. Additionally, we use the germ cells to explore gene regulation and expression of the Z chromosome during meiosis, highlighting its unique regulatory dynamics and allowing us to explore the presence of meiotic sex chromosome silencing in this group.}, author = {Elkrewi, Marwan N and Vicoso, Beatriz}, issn = {1553-7404}, journal = {PLoS Genetics}, number = {8}, publisher = {Public Library of Science}, title = {{Single-nucleus atlas of the Artemia female reproductive system suggests germline repression of the Z chromosome}}, doi = {10.1371/journal.pgen.1011376}, volume = {20}, year = {2024}, } @inproceedings{18121, abstract = {It is known that sparsity can improve interpretability for deep neural networks. However, existing methods in the area either require networks that are pre-trained with sparsity constraints, or impose sparsity after the fact, altering the network’s general behavior. In this paper, we demonstrate, for the first time, that sparsity can instead be incorporated into the interpretation process itself, as a sample-specific preprocessing step. Unlike previous work, this approach, which we call SPADE, does not place constraints on the trained model and does not affect its behavior during inference on the sample. Given a trained model and a target sample, SPADE uses sample-targeted pruning to provide a "trace" of the network’s execution on the sample, reducing the network to the most important connections prior to computing an interpretation. We demonstrate that preprocessing with SPADE significantly increases the accuracy of image saliency maps across several interpretability methods. Additionally, SPADE improves the usefulness of neuron visualizations, aiding humans in reasoning about network behavior. Our code is available at https://github.com/IST-DASLab/SPADE.}, author = {Moakhar, Arshia Soltani and Iofinova, Eugenia B and Frantar, Elias and Alistarh, Dan-Adrian}, booktitle = {Proceedings of the 41st International Conference on Machine Learning}, issn = {2640-3498}, location = {Vienna, Austria}, pages = {45955--45987}, publisher = {ML Research Press}, title = {{SPADE: Sparsity-guided debugging for deep neural networks}}, volume = {235}, year = {2024}, } @article{12334, abstract = {Regulation of the Arp2/3 complex is required for productive nucleation of branched actin networks. An emerging aspect of regulation is the incorporation of subunit isoforms into the Arp2/3 complex. Specifically, both ArpC5 subunit isoforms, ArpC5 and ArpC5L, have been reported to fine-tune nucleation activity and branch junction stability. We have combined reverse genetics and cellular structural biology to describe how ArpC5 and ArpC5L differentially affect cell migration. Both define the structural stability of ArpC1 in branch junctions and, in turn, by determining protrusion characteristics, affect protein dynamics and actin network ultrastructure. ArpC5 isoforms also affect the positioning of members of the Ena/Vasodilator-stimulated phosphoprotein (VASP) family of actin filament elongators, which mediate ArpC5 isoform–specific effects on the actin assembly level. Our results suggest that ArpC5 and Ena/VASP proteins are part of a signaling pathway enhancing cell migration.}, author = {Fäßler, Florian and Javoor, Manjunath and Datler, Julia and Döring, Hermann and Hofer, Florian and Dimchev, Georgi A and Hodirnau, Victor-Valentin and Faix, Jan and Rottner, Klemens and Schur, Florian KM}, issn = {2375-2548}, journal = {Science Advances}, keywords = {Multidisciplinary}, number = {3}, publisher = {American Association for the Advancement of Science}, title = {{ArpC5 isoforms regulate Arp2/3 complex–dependent protrusion through differential Ena/VASP positioning}}, doi = {10.1126/sciadv.add6495}, volume = {9}, year = {2023}, } @phdthesis{12732, abstract = {Nonergodic systems, whose out-of-equilibrium dynamics fail to thermalize, provide a fascinating research direction both for fundamental reasons and for application in state of the art quantum devices. Going beyond the description of statistical mechanics, ergodicity breaking yields a new paradigm in quantum many-body physics, introducing novel phases of matter with no counterpart at equilibrium. In this Thesis, we address different open questions in the field, focusing on disorder-induced many-body localization (MBL) and on weak ergodicity breaking in kinetically constrained models. In particular, we contribute to the debate about transport in kinetically constrained models, studying the effect of $U(1)$ conservation and inversion-symmetry breaking in a family of quantum East models. Using tensor network techniques, we analyze the dynamics of large MBL systems beyond the limit of exact numerical methods. In this setting, we approach the debated topic of the coexistence of localized and thermal eigenstates separated by energy thresholds known as many-body mobility edges. Inspired by recent experiments, our work further investigates the localization of a small bath induced by the coupling to a large localized chain, the so-called MBL proximity effect. In the first Chapter, we introduce a family of particle-conserving kinetically constrained models, inspired by the quantum East model. The system we study features strong inversion-symmetry breaking, due to the nature of the correlated hopping. We show that these models host so-called quantum Hilbert space fragmentation, consisting of disconnected subsectors in an entangled basis, and further provide an analytical description of this phenomenon. We further probe its effect on dynamics of simple product states, showing revivals in fidelity and local observalbes. The study of dynamics within the largest subsector reveals an anomalous transient superdiffusive behavior crossing over to slow logarithmic dynamics at later times. This work suggests that particle conserving constrained models with inversion-symmetry breaking realize new universality classes of dynamics and invite their further theoretical and experimental studies. Next, we use kinetic constraints and disorder to design a model with many-body mobility edges in particle density. This feature allows to study the dynamics of localized and thermal states in large systems beyond the limitations of previous studies. The time-evolution shows typical signatures of localization at small densities, replaced by thermal behavior at larger densities. Our results provide evidence in favor of the stability of many-body mobility edges, which was recently challenged by a theoretical argument. To support our findings, we probe the mechanism proposed as a cause of delocalization in many-body localized systems with mobility edges suggesting its ineffectiveness in the model studied. In the last Chapter of this Thesis, we address the topic of many-body localization proximity effect. We study a model inspired by recent experiments, featuring Anderson localized coupled to a small bath of free hard-core bosons. The interaction among the two particle species results in non-trivial dynamics, which we probe using tensor network techniques. Our simulations show convincing evidence of many-body localization proximity effect when the bath is composed by a single free particle and interactions are strong. We furthter observe an anomalous entanglement dynamics, which we explain through a phenomenological theory. Finally, we extract highly excited eigenstates of large systems, providing supplementary evidence in favor of our findings.}, author = {Brighi, Pietro}, issn = {2663-337X}, pages = {158}, publisher = {Institute of Science and Technology Austria}, title = {{Ergodicity breaking in disordered and kinetically constrained quantum many-body systems}}, doi = {10.15479/at:ista:12732}, year = {2023}, } @misc{12817, abstract = {3D-reconstruction of living brain tissue down to individual synapse level would create opportunities for decoding the dynamics and structure-function relationships of the brain’s complex and dense information processing network. However, it has been hindered by insufficient 3D-resolution, inadequate signal-to-noise-ratio, and prohibitive light burden in optical imaging, whereas electron microscopy is inherently static. Here we solved these challenges by developing an integrated optical/machine learning technology, LIONESS (Live Information-Optimized Nanoscopy Enabling Saturated Segmentation). It leverages optical modifications to stimulated emission depletion (STED) microscopy in comprehensively, extracellularly labelled tissue and prior information on sample structure via machine learning to simultaneously achieve isotropic super-resolution, high signal-to-noise-ratio, and compatibility with living tissue. This allows dense deep-learning-based instance segmentation and 3D-reconstruction at synapse level incorporating molecular, activity, and morphodynamic information. LIONESS opens up avenues for studying the dynamic functional (nano-)architecture of living brain tissue.}, author = {Danzl, Johann G}, publisher = {Institute of Science and Technology Austria}, title = {{Research data for the publication "Dense 4D nanoscale reconstruction of living brain tissue"}}, doi = {10.15479/AT:ISTA:12817}, year = {2023}, } @inproceedings{13053, abstract = {Deep neural networks (DNNs) often have to be compressed, via pruning and/or quantization, before they can be deployed in practical settings. In this work we propose a new compression-aware minimizer dubbed CrAM that modifies the optimization step in a principled way, in order to produce models whose local loss behavior is stable under compression operations such as pruning. Thus, dense models trained via CrAM should be compressible post-training, in a single step, without significant accuracy loss. Experimental results on standard benchmarks, such as residual networks for ImageNet classification and BERT models for language modelling, show that CrAM produces dense models that can be more accurate than the standard SGD/Adam-based baselines, but which are stable under weight pruning: specifically, we can prune models in one-shot to 70-80% sparsity with almost no accuracy loss, and to 90% with reasonable (∼1%) accuracy loss, which is competitive with gradual compression methods. Additionally, CrAM can produce sparse models which perform well for transfer learning, and it also works for semi-structured 2:4 pruning patterns supported by GPU hardware. The code for reproducing the results is available at this https URL .}, author = {Peste, Elena-Alexandra and Vladu, Adrian and Kurtic, Eldar and Lampert, Christoph and Alistarh, Dan-Adrian}, booktitle = {11th International Conference on Learning Representations }, location = {Kigali, Rwanda }, publisher = {OpenReview}, title = {{CrAM: A Compression-Aware Minimizer}}, year = {2023}, } @phdthesis{13074, abstract = {Deep learning has become an integral part of a large number of important applications, and many of the recent breakthroughs have been enabled by the ability to train very large models, capable to capture complex patterns and relationships from the data. At the same time, the massive sizes of modern deep learning models have made their deployment to smaller devices more challenging; this is particularly important, as in many applications the users rely on accurate deep learning predictions, but they only have access to devices with limited memory and compute power. One solution to this problem is to prune neural networks, by setting as many of their parameters as possible to zero, to obtain accurate sparse models with lower memory footprint. Despite the great research progress in obtaining sparse models that preserve accuracy, while satisfying memory and computational constraints, there are still many challenges associated with efficiently training sparse models, as well as understanding their generalization properties. The focus of this thesis is to investigate how the training process of sparse models can be made more efficient, and to understand the differences between sparse and dense models in terms of how well they can generalize to changes in the data distribution. We first study a method for co-training sparse and dense models, at a lower cost compared to regular training. With our method we can obtain very accurate sparse networks, and dense models that can recover the baseline accuracy. Furthermore, we are able to more easily analyze the differences, at prediction level, between the sparse-dense model pairs. Next, we investigate the generalization properties of sparse neural networks in more detail, by studying how well different sparse models trained on a larger task can adapt to smaller, more specialized tasks, in a transfer learning scenario. Our analysis across multiple pruning methods and sparsity levels reveals that sparse models provide features that can transfer similarly to or better than the dense baseline. However, the choice of the pruning method plays an important role, and can influence the results when the features are fixed (linear finetuning), or when they are allowed to adapt to the new task (full finetuning). Using sparse models with fixed masks for finetuning on new tasks has an important practical advantage, as it enables training neural networks on smaller devices. However, one drawback of current pruning methods is that the entire training cycle has to be repeated to obtain the initial sparse model, for every sparsity target; in consequence, the entire training process is costly and also multiple models need to be stored. In the last part of the thesis we propose a method that can train accurate dense models that are compressible in a single step, to multiple sparsity levels, without additional finetuning. Our method results in sparse models that can be competitive with existing pruning methods, and which can also successfully generalize to new tasks.}, author = {Peste, Elena-Alexandra}, issn = {2663-337X}, pages = {147}, publisher = {Institute of Science and Technology Austria}, title = {{Efficiency and generalization of sparse neural networks}}, doi = {10.15479/at:ista:13074}, year = {2023}, }