@article{12538,
  abstract     = {In this paper, we study the compression of a target two-layer neural network with N nodes into a compressed network with M<N nodes. More precisely, we consider the setting in which the weights of the target network are i.i.d. sub-Gaussian, and we minimize the population L_2 loss between the outputs of the target and of the compressed network, under the assumption of Gaussian inputs. By using tools from high-dimensional probability, we show that this non-convex problem can be simplified when the target network is sufficiently over-parameterized, and provide the error rate of this approximation as a function of the input dimension and N. In this mean-field limit, the simplified objective, as well as the optimal weights of the compressed network, does not depend on the realization of the target network, but only on expected scaling factors. Furthermore, for networks with ReLU activation, we conjecture that the optimum of the simplified optimization problem is achieved by taking weights on the Equiangular Tight Frame (ETF), while the scaling of the weights and the orientation of the ETF depend on the parameters of the target network. Numerical evidence is provided to support this conjecture.},
  author       = {Amani, Mohammad Hossein and Bombari, Simone and Mondelli, Marco and Pukdee, Rattana and Rini, Stefano},
  isbn         = {9781665483414},
  journal      = {IEEE Information Theory Workshop},
  location     = {Mumbai, India},
  pages        = {588--593},
  publisher    = {IEEE},
  title        = {{Sharp asymptotics on the compression of two-layer neural networks}},
  doi          = {10.1109/ITW54588.2022.9965870},
  year         = {2022},
}

@inproceedings{12540,
  abstract     = {We consider the problem of signal estimation in generalized linear models defined via rotationally invariant design matrices. Since these matrices can have an arbitrary spectral distribution, this model is well suited for capturing complex correlation structures which often arise in applications. We propose a novel family of approximate message passing (AMP) algorithms for signal estimation, and rigorously characterize their performance in the high-dimensional limit via a state evolution recursion. Our rotationally invariant AMP has complexity of the same order as the existing AMP derived under the restrictive assumption of a Gaussian design; our algorithm also recovers this existing AMP as a special case. Numerical results showcase a performance close to Vector AMP (which is conjectured to be Bayes-optimal in some settings), but obtained with a much lower complexity, as the proposed algorithm does not require a computationally expensive singular value decomposition.},
  author       = {Venkataramanan, Ramji and Kögler, Kevin and Mondelli, Marco},
  booktitle    = {Proceedings of the 39th International Conference on Machine Learning},
  location     = {Baltimore, MD, United States},
  publisher    = {ML Research Press},
  title        = {{Estimation in rotationally invariant generalized linear models via approximate message passing}},
  volume       = {162},
  year         = {2022},
}

@inproceedings{12568,
  abstract     = {We treat the problem of risk-aware control for stochastic shortest path (SSP) on Markov decision processes (MDP). Typically, expectation is considered for SSP, which however is oblivious to the incurred risk. We present an alternative view, instead optimizing conditional value-at-risk (CVaR), an established risk measure. We treat both Markov chains as well as MDP and introduce, through novel insights, two algorithms, based on linear programming and value iteration, respectively. Both algorithms offer precise and provably correct solutions. Evaluation of our prototype implementation shows that risk-aware control is feasible on several moderately sized models.},
  author       = {Meggendorfer, Tobias},
  booktitle    = {Proceedings of the 36th AAAI Conference on Artificial Intelligence, AAAI 2022},
  isbn         = {1577358767},
  issn         = {2374-3468},
  location     = {Virtual},
  number       = {9},
  pages        = {9858--9867},
  publisher    = {Association for the Advancement of Artificial Intelligence},
  title        = {{Risk-aware stochastic shortest path}},
  doi          = {10.1609/aaai.v36i9.21222},
  volume       = {36},
  year         = {2022},
}

@article{12670,
  abstract     = {DNA methylation plays essential homeostatic functions in eukaryotic genomes. In animals, DNA methylation is also developmentally regulated and, in turn, regulates development. In the past two decades, huge research effort has endorsed the understanding that DNA methylation plays a similar role in plant development, especially during sexual reproduction. The power of whole-genome sequencing and cell isolation techniques, as well as bioinformatics tools, have enabled recent studies to reveal dynamic changes in DNA methylation during germline development. Furthermore, the combination of these technological advances with genetics, developmental biology and cell biology tools has revealed functional methylation reprogramming events that control gene and transposon activities in flowering plant germlines. In this review, we discuss the major advances in our knowledge of DNA methylation dynamics during male and female germline development in flowering plants.},
  author       = {He, Shengbo and Feng, Xiaoqi},
  issn         = {1744-7909},
  journal      = {Journal of Integrative Plant Biology},
  keywords     = {Plant Science, General Biochemistry, Genetics and Molecular Biology, Biochemistry},
  number       = {12},
  pages        = {2240--2251},
  publisher    = {Wiley},
  title        = {{DNA methylation dynamics during germline development}},
  doi          = {10.1111/jipb.13422},
  volume       = {64},
  year         = {2022},
}

@article{12671,
  abstract     = {Sperm chromatin is typically transformed by protamines into a compact and transcriptionally inactive state1,2. Sperm cells of flowering plants lack protamines, yet they have small, transcriptionally active nuclei with chromatin condensed through an unknown mechanism3,4. Here we show that a histone variant, H2B.8, mediates sperm chromatin and nuclear condensation in Arabidopsis thaliana. Loss of H2B.8 causes enlarged sperm nuclei with dispersed chromatin, whereas ectopic expression in somatic cells produces smaller nuclei with aggregated chromatin. This result demonstrates that H2B.8 is sufficient for chromatin condensation. H2B.8 aggregates transcriptionally inactive AT-rich chromatin into phase-separated condensates, which facilitates nuclear compaction without reducing transcription. Reciprocal crosses show that mutation of h2b.8 reduces male transmission, which suggests that H2B.8-mediated sperm compaction is important for fertility. Altogether, our results reveal a new mechanism of nuclear compaction through global aggregation of unexpressed chromatin. We propose that H2B.8 is an evolutionary innovation of flowering plants that achieves nuclear condensation compatible with active transcription.},
  author       = {Buttress, Toby and He, Shengbo and Wang, Liang and Zhou, Shaoli and Saalbach, Gerhard and Vickers, Martin and Li, Guohong and Li, Pilong and Feng, Xiaoqi},
  issn         = {1476-4687},
  journal      = {Nature},
  number       = {7936},
  pages        = {614--622},
  publisher    = {Springer Nature},
  title        = {{Histone H2B.8 compacts flowering plant sperm through chromatin phase separation}},
  doi          = {10.1038/s41586-022-05386-6},
  volume       = {611},
  year         = {2022},
}

@unpublished{12677,
  abstract     = {In modern sample-driven Prophet Inequality, an adversary chooses a sequence of n items with values v1,v2,…,vn to be presented to a decision maker (DM). The process follows in two phases. In the first phase (sampling phase), some items, possibly selected at random, are revealed to the DM, but she can never accept them. In the second phase, the DM is presented with the other items in a random order and online fashion. For each item, she must make an irrevocable decision to either accept the item and stop the process or reject the item forever and proceed to the next item. The goal of the DM is to maximize the expected value as compared to a Prophet (or offline algorithm) that has access to all information. In this setting, the sampling phase has no cost and is not part of the optimization process. However, in many scenarios, the samples are obtained as part of the decision-making process.
We model this aspect as a two-phase Prophet Inequality where an adversary chooses a sequence of 2n items with values v1,v2,…,v2n and the items are randomly ordered. Finally, there are two phases of the Prophet Inequality problem with the first n-items and the rest of the items, respectively. We show that some basic algorithms achieve a ratio of at most 0.450. We present an algorithm that achieves a ratio of at least 0.495. Finally, we show that for every algorithm the ratio it can achieve is at most 0.502. Hence our algorithm is near-optimal.},
  author       = {Chatterjee, Krishnendu and Mohammadi, Mona and Saona Urmeneta, Raimundo J},
  booktitle    = {arXiv},
  title        = {{Repeated prophet inequality with near-optimal bounds}},
  doi          = {10.48550/ARXIV.2209.14368},
  year         = {2022},
}

@unpublished{12750,
  abstract     = {Quantum kinetically constrained models have recently attracted significant attention due to their anomalous dynamics and thermalization. In this work, we introduce a hitherto unexplored family of kinetically constrained models featuring a conserved particle number and strong inversion-symmetry breaking due to facilitated hopping. We demonstrate that these models provide a generic example of so-called quantum Hilbert space fragmentation, that is manifested in disconnected sectors in the Hilbert space that are not apparent in the computational basis. Quantum Hilbert space fragmentation leads to an exponential in system size number of eigenstates with exactly zero entanglement entropy across several bipartite cuts. These eigenstates can be probed dynamically using quenches from simple initial product states. In addition, we study the particle spreading under unitary dynamics launched from the domain wall state, and find faster than diffusive dynamics at high particle densities, that crosses over into logarithmically slow relaxation at smaller densities. Using a classically simulable cellular automaton, we reproduce the logarithmic dynamics observed in the quantum case. Our work suggests that particle conserving constrained models with inversion symmetry breaking realize so far unexplored universality classes of dynamics and invite their further theoretical and experimental studies.},
  author       = {Brighi, Pietro and Ljubotina, Marko and Serbyn, Maksym},
  booktitle    = {arXiv},
  title        = {{Hilbert space fragmentation and slow dynamics in particle-conserving quantum East models}},
  doi          = {10.48550/arXiv.2210.15607},
  year         = {2022},
}

@article{12776,
  abstract     = {An improved asymptotic formula is established for the number of rational points of bounded height on the split smooth del Pezzo surface of degree 5. The proof uses the five conic bundle structures on the surface.},
  author       = {Browning, Timothy D},
  issn         = {1076-9803},
  journal      = {New York Journal of Mathematics},
  pages        = {1193 -- 1229},
  publisher    = {State University of New York},
  title        = {{Revisiting the Manin–Peyre conjecture for the split del Pezzo surface of degree 5}},
  volume       = {28},
  year         = {2022},
}

@unpublished{12860,
  abstract     = {Memorization of the relation between entities in a dataset can lead to privacy issues when using a trained model for question answering. We introduce Relational Memorization (RM) to understand, quantify and control this phenomenon. While bounding general memorization can have detrimental effects on the performance of a trained model, bounding RM does not prevent effective learning. The difference is most pronounced when the data distribution is long-tailed, with many queries having only few training examples: Impeding general memorization prevents effective learning, while impeding only relational memorization still allows learning general properties of the underlying concepts. We formalize the notion of Relational Privacy (RP) and, inspired by Differential Privacy (DP), we provide a possible definition of Differential Relational Privacy (DrP). These notions can be used to describe and compute bounds on the amount of RM in a trained model. We illustrate Relational Privacy concepts in experiments with large-scale models for Question Answering.},
  author       = {Bombari, Simone and Achille, Alessandro and Wang, Zijian and Wang, Yu-Xiang and Xie, Yusheng and Singh, Kunwar Yashraj and Appalaraju, Srikar and Mahadevan, Vijay and Soatto, Stefano},
  booktitle    = {arXiv},
  title        = {{Towards differential relational privacy and its use in question answering}},
  doi          = {10.48550/arXiv.2203.16701},
  year         = {2022},
}

@misc{13064,
  abstract     = {Genetically informed, deep-phenotyped biobanks are an important research resource and it is imperative that the most powerful, versatile, and efficient analysis approaches are used. Here, we apply our recently developed Bayesian grouped mixture of regressions model (GMRM) in the UK and Estonian Biobanks and obtain the highest genomic prediction accuracy reported to date across 21 heritable traits. When compared to other approaches, GMRM accuracy was greater than annotation prediction models run in the LDAK or LDPred-funct software by 15% (SE 7%) and 14% (SE 2%), respectively, and was 18% (SE 3%) greater than a baseline BayesR model without single-nucleotide polymorphism (SNP) markers grouped into minor allele frequency–linkage disequilibrium (MAF-LD) annotation categories. For height, the prediction accuracy R 2 was 47% in a UK Biobank holdout sample, which was 76% of the estimated h SNP 2 . We then extend our GMRM prediction model to provide mixed-linear model association (MLMA) SNP marker estimates for genome-wide association (GWAS) discovery, which increased the independent loci detected to 16,162 in unrelated UK Biobank individuals, compared to 10,550 from BoltLMM and 10,095 from Regenie, a 62 and 65% increase, respectively. The average χ2 value of the leading markers increased by 15.24 (SE 0.41) for every 1% increase in prediction accuracy gained over a baseline BayesR model across the traits. Thus, we show that modeling genetic associations accounting for MAF and LD differences among SNP markers, and incorporating prior knowledge of genomic function, is important for both genomic prediction and discovery in large-scale individual-level studies.},
  author       = {Orliac, Etienne and Trejo Banos, Daniel and Ojavee, Sven and Läll, Kristi and Mägi, Reedik and Visscher, Peter and Robinson, Matthew Richard},
  publisher    = {Dryad},
  title        = {{Improving genome-wide association discovery and genomic prediction accuracy in biobank data}},
  doi          = {10.5061/DRYAD.GTHT76HMZ},
  year         = {2022},
}

@misc{13066,
  abstract     = {Chromosomal inversions have been shown to play a major role in local adaptation by suppressing recombination between alternative arrangements and maintaining beneficial allele combinations. However, so far, their importance relative to the remaining genome remains largely unknown. Understanding the genetic architecture of adaptation requires better estimates of how loci of different effect sizes contribute to phenotypic variation. Here, we used three Swedish islands where the marine snail Littorina saxatilis has repeatedly evolved into two distinct ecotypes along a habitat transition. We estimated the contribution of inversion polymorphisms to phenotypic divergence while controlling for polygenic effects in the remaining genome using a quantitative genetics framework. We confirmed the importance of inversions but showed that contributions of loci outside inversions are of similar magnitude, with variable proportions dependent on the trait and the population. Some inversions showed consistent effects across all sites, whereas others exhibited site-specific effects, indicating that the genomic basis for replicated phenotypic divergence is only partly shared. The contributions of sexual dimorphism as well as environmental factors to phenotypic variation were significant but minor compared to inversions and polygenic background. Overall, this integrated approach provides insight into the multiple mechanisms contributing to parallel phenotypic divergence.},
  author       = {Koch, Eva and Ravinet, Mark and Westram, Anja M and Jonannesson, Kerstin and Butlin, Roger},
  publisher    = {Dryad},
  title        = {{Data from: Genetic architecture of repeated phenotypic divergence in Littorina saxatilis ecotype evolution}},
  doi          = {10.5061/DRYAD.M905QFV4B},
  year         = {2022},
}

@misc{13076,
  abstract     = {The source code for replicating experiments presented in the paper.

The implementation of the designed priority schedulers can be found in Galois-2.2.1/include/Galois/WorkList/:
StealingMultiQueue.h is the StealingMultiQueue.
MQOptimized/ contains MQ Optimized variants.

We provide images that contain all the dependencies and datasets. Images can be pulled from npostnikova/mq-based-schedulers repository, or downloaded from Zenodo. See readme for more detail.},
  author       = {Postnikova, Anastasiia and Koval, Nikita and Nadiradze, Giorgi and Alistarh, Dan-Adrian},
  publisher    = {Zenodo},
  title        = {{Multi-queues can be state-of-the-art priority schedulers}},
  doi          = {10.5281/ZENODO.5733408},
  year         = {2022},
}

@inproceedings{13239,
  abstract     = {Brains are thought to engage in predictive learning - learning to predict upcoming stimuli - to construct an internal model of their environment. This is especially notable for spatial navigation, as first described by Tolman’s latent learning tasks. However, predictive learning has also been observed in sensory cortex, in settings unrelated to spatial navigation. Apart from normative frameworks such as active inference or efficient coding, what could be the utility of learning to predict the patterns of occurrence of correlated stimuli? Here we show that prediction, and thereby the construction of an internal model of sequential stimuli, can bootstrap the learning process of a working memory task in a recurrent neural network. We implemented predictive learning alongside working memory match-tasks, and networks emerged to solve the prediction task first by encoding information across time to predict upcoming stimuli, and then eavesdropped on this solution to solve the matching task. Eavesdropping was most beneficial when neural resources were limited. Hence, predictive learning acts as a general neural mechanism to learn to store sensory information that can later be essential for working memory tasks.},
  author       = {Van Der Plas, Thijs L. and Vogels, Tim P and Manohar, Sanjay G.},
  booktitle    = {Proceedings of Machine Learning Research},
  issn         = {2640-3498},
  pages        = {518--531},
  publisher    = {ML Research Press},
  title        = {{Predictive learning enables neural networks to learn complex working memory tasks}},
  volume       = {199},
  year         = {2022},
}

@article{13240,
  abstract     = {Ustilago maydis is a biotrophic phytopathogenic fungus that causes corn smut disease. As a well-established model system, U. maydis is genetically fully accessible with large omics datasets available and subject to various biological questions ranging from DNA-repair, RNA-transport, and protein secretion to disease biology. For many genetic approaches, tight control of transgene regulation is important. Here we established an optimised version of the Tetracycline-ON (TetON) system for U. maydis. We demonstrate the Tetracycline concentration-dependent expression of fluorescent protein transgenes and the system’s suitability for the induced expression of the toxic protein BCL2 Associated X-1 (Bax1). The Golden Gate compatible vector system contains a native minimal promoter from the mating factor a-1 encoding gene, mfa with ten copies of the tet-regulated operator (tetO) and a codon optimised Tet-repressor (tetR*) which is translationally fused to the native transcriptional corepressor Mql1 (UMAG_05501). The metabolism-independent transcriptional regulator system is functional both, in liquid culture as well as on solid media in the presence of the inducer and can become a useful tool for toxin-antitoxin studies, identification of antifungal proteins, and to study functions of toxic gene products in Ustilago maydis.},
  author       = {Ingole, Kishor D. and Nagarajan, Nithya and Uhse, Simon and Giannini, Caterina and Djamei, Armin},
  issn         = {2673-6128},
  journal      = {Frontiers in Fungal Biology},
  publisher    = {Frontiers Media},
  title        = {{Tetracycline-controlled (TetON) gene expression system for the smut fungus Ustilago maydis}},
  doi          = {10.3389/ffunb.2022.1029114},
  volume       = {3},
  year         = {2022},
}

@inproceedings{13241,
  abstract     = {Addressing fairness concerns about machine learning models is a crucial step towards their long-term adoption in real-world automated systems. Many approaches for training fair models from data have been developed and an implicit assumption about such algorithms is that they are able to recover a fair model, despite potential historical biases in the data. In this work we show a number of impossibility results that indicate that there is no learning algorithm that can recover a fair model when a proportion of the dataset is subject to arbitrary manipulations. Specifically, we prove that there are situations in which an adversary can force any learner to return a biased classifier, with or without degrading accuracy, and that the strength of this bias increases for learning problems with underrepresented protected groups in the data. Our results emphasize on the importance of studying further data corruption models of various strength and of establishing stricter data collection practices for fairness-aware learning.},
  author       = {Konstantinov, Nikola H and Lampert, Christoph},
  booktitle    = {Proceedings of Machine Learning Research},
  issn         = {2640-3498},
  pages        = {59--83},
  publisher    = {ML Research Press},
  title        = {{On the impossibility of fairness-aware learning from corrupted data}},
  volume       = {171},
  year         = {2022},
}

@inproceedings{14093,
  abstract     = { We propose a stochastic conditional gradient method (CGM) for minimizing convex finite-sum objectives formed as a sum of smooth and non-smooth terms. Existing CGM variants for this template either suffer from slow convergence rates, or require carefully increasing the batch size over the course of the algorithm’s execution, which leads to computing full gradients. In contrast, the proposed method, equipped with a stochastic average gradient (SAG) estimator, requires only one sample per iteration. Nevertheless, it guarantees fast convergence rates on par with more sophisticated variance reduction techniques. In applications we put special emphasis on problems with a large number of separable constraints. Such problems are prevalent among semidefinite programming (SDP) formulations arising in machine learning and theoretical computer science. We provide numerical experiments on matrix completion, unsupervised clustering, and sparsest-cut SDPs. },
  author       = {Dresdner, Gideon and Vladarean, Maria-Luiza and Rätsch, Gunnar and Locatello, Francesco and Cevher, Volkan and Yurtsever, Alp},
  booktitle    = {Proceedings of the 25th International Conference on Artificial Intelligence and Statistics},
  issn         = {2640-3498},
  location     = {Virtual},
  pages        = {8439--8457},
  publisher    = {ML Research Press},
  title        = {{ Faster one-sample stochastic conditional gradient method for composite convex minimization}},
  volume       = {151},
  year         = {2022},
}

@inproceedings{14106,
  abstract     = {We show that deep networks trained to satisfy demographic parity often do so
through a form of race or gender awareness, and that the more we force a network
to be fair, the more accurately we can recover race or gender from the internal state
of the network. Based on this observation, we investigate an alternative fairness
approach: we add a second classification head to the network to explicitly predict
the protected attribute (such as race or gender) alongside the original task. After
training the two-headed network, we enforce demographic parity by merging the
two heads, creating a network with the same architecture as the original network.
We establish a close relationship between existing approaches and our approach
by showing (1) that the decisions of a fair classifier are well-approximated by our
approach, and (2) that an unfair and optimally accurate classifier can be recovered
from a fair classifier and our second head predicting the protected attribute. We use
our explicit formulation to argue that the existing fairness approaches, just as ours,
demonstrate disparate treatment and that they are likely to be unlawful in a wide
range of scenarios under US law.},
  author       = {Lohaus, Michael and Kleindessner, Matthäus and Kenthapadi, Krishnaram and Locatello, Francesco and Russell, Chris},
  booktitle    = {36th Conference on Neural Information Processing Systems},
  isbn         = {9781713871088},
  location     = {New Orleans, LA, United States},
  pages        = {16548--16562},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Are two heads the same as one? Identifying disparate treatment in fair neural networks}},
  volume       = {35},
  year         = {2022},
}

@inproceedings{14107,
  abstract     = {Amodal perception requires inferring the full shape of an object that is partially occluded. This task is particularly challenging on two levels: (1) it requires more information than what is contained in the instant retina or imaging sensor, (2) it is difficult to obtain enough well-annotated amodal labels for supervision. To this end, this paper develops a new framework of
Self-supervised amodal Video object segmentation (SaVos). Our method efficiently leverages the visual information of video temporal sequences to infer the amodal mask of objects. The key intuition is that the occluded part of an object can be explained away if that part is visible in other frames, possibly deformed as long as the deformation can be reasonably learned.
Accordingly, we derive a novel self-supervised learning paradigm that efficiently utilizes the visible object parts as the supervision to guide the training on videos. In addition to learning type prior to complete masks for known types, SaVos also learns the spatiotemporal prior, which is also useful for the amodal task and could generalize to unseen types. The proposed
framework achieves the state-of-the-art performance on the synthetic amodal segmentation benchmark FISHBOWL and the real world benchmark KINS-Video-Car. Further, it lends itself well to being transferred to novel distributions using test-time adaptation, outperforming existing models even after the transfer to a new distribution.},
  author       = {Yao, Jian and Hong, Yuxin and Wang, Chiyu and Xiao, Tianjun and He, Tong and Locatello, Francesco and Wipf, David and Fu, Yanwei and Zhang, Zheng},
  booktitle    = {36th Conference on Neural Information Processing Systems},
  location     = {New Orleans, LA, United States},
  title        = {{Self-supervised amodal video object segmentation}},
  doi          = {10.48550/arXiv.2210.12733},
  year         = {2022},
}

@inproceedings{14114,
  abstract     = {Algorithmic fairness is frequently motivated in terms of a trade-off in which overall performance is decreased so as to improve performance on disadvantaged groups where the algorithm would otherwise be less accurate. Contrary to this, we find that applying existing fairness approaches to computer vision improve fairness by degrading the performance of classifiers across all groups (with increased degradation on the best performing groups). Extending the bias-variance decomposition for classification to fairness, we theoretically explain why the majority of fairness methods designed for low capacity models should not be used in settings involving high-capacity models, a scenario common to computer vision. We corroborate this analysis with extensive experimental support that shows that many of the fairness heuristics used in computer vision also degrade performance on the most disadvantaged groups. Building on these insights, we propose an adaptive augmentation strategy that, uniquely, of all methods tested, improves performance for the disadvantaged groups.},
  author       = {Zietlow, Dominik and Lohaus, Michael and Balakrishnan, Guha and Kleindessner, Matthaus and Locatello, Francesco and Scholkopf, Bernhard and Russell, Chris},
  booktitle    = {2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  isbn         = {9781665469470},
  issn         = {2575-7075},
  location     = {New Orleans, LA, United States},
  pages        = {10400--10411},
  publisher    = {Institute of Electrical and Electronics Engineers},
  title        = {{Leveling down in computer vision: Pareto inefficiencies in fair deep classifiers}},
  doi          = {10.1109/cvpr52688.2022.01016},
  year         = {2022},
}

@inproceedings{14168,
  abstract     = {Recent work has seen the development of general purpose neural architectures
that can be trained to perform tasks across diverse data modalities. General
purpose models typically make few assumptions about the underlying
data-structure and are known to perform well in the large-data regime. At the
same time, there has been growing interest in modular neural architectures that
represent the data using sparsely interacting modules. These models can be more
robust out-of-distribution, computationally efficient, and capable of
sample-efficient adaptation to new data. However, they tend to make
domain-specific assumptions about the data, and present challenges in how
module behavior (i.e., parameterization) and connectivity (i.e., their layout)
can be jointly learned. In this work, we introduce a general purpose, yet
modular neural architecture called Neural Attentive Circuits (NACs) that
jointly learns the parameterization and a sparse connectivity of neural modules
without using domain knowledge. NACs are best understood as the combination of
two systems that are jointly trained end-to-end: one that determines the module
configuration and the other that executes it on an input. We demonstrate
qualitatively that NACs learn diverse and meaningful module configurations on
the NLVR2 dataset without additional supervision. Quantitatively, we show that
by incorporating modularity in this way, NACs improve upon a strong non-modular
baseline in terms of low-shot adaptation on CIFAR and CUBs dataset by about
10%, and OOD robustness on Tiny ImageNet-R by about 2.5%. Further, we find that
NACs can achieve an 8x speedup at inference time while losing less than 3%
performance. Finally, we find NACs to yield competitive results on diverse data
modalities spanning point-cloud classification, symbolic processing and
text-classification from ASCII bytes, thereby confirming its general purpose
nature.},
  author       = {Rahaman, Nasim and Weiss, Martin and Locatello, Francesco and Pal, Chris and Bengio, Yoshua and Schölkopf, Bernhard and Li, Li Erran and Ballas, Nicolas},
  booktitle    = {36th Conference on Neural Information Processing Systems},
  location     = {New Orleans, United States},
  title        = {{Neural attentive circuits}},
  volume       = {35},
  year         = {2022},
}