@inproceedings{22827,
  abstract     = {Quantized training of Large Language Models (LLMs) remains an open challenge, as maintaining accuracy while performing all matrix multiplications in low precision has proven difficult. This is particularly the case when fine-tuning pre-trained models, which can have large weight, activation, and error (output gradient) outlier values that make lower-precision optimization difficult. To address this, we present HALO, a new quantization-aware training approach for Transformers that enables accurate and efficient low-precision training by combining 1) strategic placement of Hadamard rotations in both forward and backward passes, which mitigate outliers, 2) high-performance kernel support, and 3) FSDP integration for low-precision communication. Our approach ensures that all large matrix multiplications during the forward and backward passes are executed in lower precision. Applied to LLaMa models, HALO achieves near-full-precision-equivalent results during fine-tuning on various tasks, while delivering up to 1.41x end-to-end speedup for full fine-tuning on RTX 4090 GPUs. HALO efficiently supports both standard and parameter-efficient fine-tuning (PEFT). Our results demonstrate the first practical approach to fully quantized LLM fine-tuning that maintains accuracy in INT8 and FP6 precision, while delivering performance benefits.},
  author       = {Ashkboos, Saleh and Nikdan, Mahdi and Tabesh, Soroush and Lopez Castro, Roberto and Hoefler, Torsten and Alistarh, Dan-Adrian},
  booktitle    = {39th Conference on Neural Information Processing Systems},
  isbn         = {9798331338275},
  issn         = {1049-5258},
  location     = {San Diego, CA, United States},
  pages        = {131755--131780},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{HALO: Hadamard-assisted lower-precision optimization for LLMs}},
  doi          = {10.52202/085713-3966},
  volume       = {38},
  year         = {2025},
}

@inproceedings{22831,
  abstract     = {Neural models learn representations of high-dimensional data on low-dimensional
manifolds. Multiple factors, including stochasticities in the training process, model
architectures, and additional inductive biases, may induce different representations,
even when learning the same task on the same data. However, it has recently been
shown that when a latent structure is shared between distinct latent spaces, relative
distances between representations can be preserved, up to distortions. Building
on this idea, we demonstrate that exploiting the differential-geometric structure of
latent spaces of neural models, it is possible to capture precisely the transformations
between representational spaces trained on similar data distributions. Specifically,
we assume that distinct neural models parametrize approximately the same underlying manifold, and introduce a representation based on the pullback metric
that captures the intrinsic structure of the latent space, while scaling efficiently
to large models. We validate experimentally our method on model stitching and
retrieval tasks, covering autoencoders and vision foundation discriminative models,
across diverse architectures, datasets, pretraining schemes and modalities. Code is
available at https://github.com/marc0git/RelativeGeodesics.},
  author       = {Yu, Hanlin and Inal, Berfin and Arvanitidis, Georgios and Hauberg, Søren and Locatello, Francesco and Fumero, Marco},
  booktitle    = {39th Conference on Neural Information Processing Systems},
  isbn         = {9798331338275},
  issn         = {1049-5258},
  location     = {San Diego, CA, United States},
  pages        = {125316--125360},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Connecting neural models latent geometries with relative geodesic representations}},
  doi          = {10.52202/085713-3769},
  volume       = {38},
  year         = {2025},
}

@inproceedings{21076,
  abstract     = {In many scientific experiments, the data annotating cost constraints the pace for testing novel hypotheses. Yet, modern machine learning pipelines offer a promising solution—provided their predictions yield correct conclusions. We focus on Prediction-Powered Causal Inferences (PPCI), i.e., estimating the treatment effect in an unlabeled target experiment, relying on training data with the same outcome annotated but potentially different treatment or effect modifiers. We first show that conditional calibration guarantees valid PPCI at population level. Then, we introduce a sufficient representation constraint transferring validity across experiments, which we propose to enforce in practice in Deconfounded Empirical Risk Minimization, our new model-agnostic training objective. We validate our method on synthetic and real-world scientific data, solving impossible problem instances for Empirical Risk Minimization even with standard invariance constraints. In particular, for the first time, we achieve valid causal inference on a scientific experiment with complex recording and no human annotations, fine-tuning a foundational model on our similar annotated experiment.},
  author       = {Cadei, Riccardo and Demirel, Ilker and De Bartolomeis, Piersilvio and Lindorfer, Lukas and Cremer, Sylvia and Schmid, Cordelia and Locatello, Francesco},
  booktitle    = {39th Annual Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {San Diego, CA, United States},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Prediction-powered causal inferences}},
  doi          = {10.52202/085713-2479},
  volume       = {38},
  year         = {2025},
}

@inproceedings{21072,
  abstract     = {Language and vision-language models have shown impressive performance across a wide range of tasks, but their internal mechanisms remain only partly understood. In this work, we study how individual attention heads in text-generative models specialize in specific semantic or visual attributes. Building on an established interpretability method, we reinterpret the practice of probing intermediate activations with the final decoding layer through the lens of signal processing. This lets us analyze multiple samples in a principled way and rank attention heads based on their relevance to target concepts. Our results show consistent patterns of specialization at the head level across both unimodal and multimodal transformers. Remarkably, we find that editing as few as 1% of the heads, selected using our method, can reliably suppress or enhance targeted concepts in the model output. We validate our approach on language tasks such as question answering and toxicity mitigation, as well as vision-language tasks including image classification and captioning. Our findings highlight an interpretable and controllable structure within attention layers, offering simple tools for understanding and editing large-scale generative models.},
  author       = {Basile, Lorenzo and Maiorca, Valentino and Doimo, Diego and Locatello, Francesco and Cazzaniga, Alberto},
  booktitle    = {39th Annual Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {San Diego, CA, United States},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Head pursuit: Probing attention specialization in multimodal transformers}},
  doi          = {10.52202/085713-3752},
  volume       = {38},
  year         = {2025},
}

@inproceedings{21068,
  abstract     = {Causal reasoning and discovery, two fundamental tasks of causal analysis,
often face challenges in applications due to the complexity, noisiness, and highdimensionality of real-world data. Despite recent progress in identifying latent
causal structures using causal representation learning (CRL), what makes learned
representations useful for causal downstream tasks and how to evaluate them are
still not well understood. In this paper, we reinterpret CRL using a measurement
model framework, where the learned representations are viewed as proxy measurements of the latent causal variables. Our approach clarifies the conditions under
which learned representations support downstream causal reasoning and provides
a principled basis for quantitatively assessing the quality of representations using
a new Test-based Measurement EXclusivity (T-MEX) score. We validate T-MEX
across diverse causal inference scenarios, including numerical simulations and
real-world ecological video analysis, demonstrating that the proposed framework
and corresponding score effectively assess the identification of learned representations and their usefulness for causal downstream tasks. Reproducible code can
be found at https://github.com/shimenghuang/a-measurement-perspective-of-crl.},
  author       = {Yao, Dingling and Huang, Shimeng and Cadei, Riccardo and Zhang, Kun and Locatello, Francesco},
  booktitle    = {39th Annual Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {San Diego, CA, United States},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{The third pillar of causal analysis? A measurement perspective on causal representations}},
  doi          = {10.52202/085713-1444},
  volume       = {38},
  year         = {2025},
}

@inproceedings{21070,
  abstract     = {Deep learning systems deployed in real-world applications often encounter data that is different from their in-distribution (ID). A reliable model should ideally abstain from making decisions in this out-of-distribution (OOD) setting. Existing state-of-the-art methods primarily focus on feature distances, such as k-th nearest neighbors and distances to decision boundaries, either overlooking or ineffectively using in-distribution statistics. In this work, we propose a novel angle-based metric for OOD detection that is computed relative to the in-distribution structure. We demonstrate that the angles between feature representations and decision boundaries, viewed from the mean of in-distribution features, serve as an effective discriminative factor between ID and OOD data. We evaluate our method on nine ImageNet-pretrained models. Our approach achieves the lowest FPR in 5 out of 9 ImageNet models, obtains the best average FPR overall, and consistently ranking among the top 3 across all evaluated models. Furthermore, we highlight the benefits of contrastive representations by showing strong performance with ResNet SCL and CLIP architectures. Finally, we demonstrate that the scale-invariant nature of our score enables an ensemble strategy via simple score summation. },
  author       = {Demirel, Berker and Fumero, Marco  and Locatello, Francesco},
  booktitle    = {39th Annual Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {San Diego, CA, United States},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Out-of-Distribution detection with relative angles}},
  doi          = {10.52202/085713-3125},
  volume       = {38},
  year         = {2025},
}

@inproceedings{22823,
  abstract     = {This position paper argues that setting the privacy budget in differential privacy should not be viewed as an important limitation of differential privacy compared to alternative methods for privacy-preserving machine learning. The so-called problem of interpreting the privacy budget is often presented as a major hindrance to the wider adoption of differential privacy in real-world deployments and is sometimes used to promote alternative mitigation techniques for data protection. We believe this misleads decision-makers into choosing unsafe methods. We argue that the difficulty in interpreting privacy budgets does not stem from the definition of differential privacy itself, but from the intrinsic difficulty of estimating privacy risks in context, a challenge that any rigorous method for privacy risk assessment face. Moreover, we claim that any sound method for estimating privacy risks should, given the current state of research, be expressible within the differential privacy framework or justify why it cannot.},
  author       = {Cyffers, Edwige Audrey Lucienne},
  booktitle    = {39th Conference on Neural Information Processing Systems},
  isbn         = {9798331338275},
  issn         = {1049-5258},
  location     = {San Diego, CA, United States},
  pages        = {159016--159028},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Setting ε is not the issue in differential privacy}},
  doi          = {10.52202/085713-4798},
  volume       = {38},
  year         = {2025},
}

@inproceedings{18890,
  abstract     = {Deep Neural Collapse (DNC) refers to the surprisingly rigid structure of the data representations in the final layers of Deep Neural Networks (DNNs). Though the phenomenon has been measured in a variety of settings, its emergence is typically explained via data-agnostic approaches, such as the unconstrained features model. In this work, we introduce a data-dependent setting where DNC forms due to feature learning through the average gradient outer product (AGOP). The AGOP is defined with respect to a learned predictor and is equal to the uncentered covariance matrix of its input-output gradients averaged over the training dataset. The Deep Recursive Feature Machine (Deep RFM) is a method that constructs a neural network by iteratively mapping the data with the AGOP and applying an untrained random feature map. We demonstrate empirically that DNC occurs in Deep RFM across standard settings as a consequence of the projection with the AGOP matrix computed at each layer. Further, we theoretically explain DNC in Deep RFM in an asymptotic setting and as a result of kernel learning. We then provide evidence that this mechanism holds for neural networks more generally. In particular, we show that the right singular vectors and values of the weights can be responsible for the majority of within-class variability collapse for DNNs trained in the feature learning regime. As observed in recent work, this singular structure is highly correlated with that of the AGOP.},
  author       = {Beaglehole, Daniel and Súkeník, Peter and Mondelli, Marco and Belkin, Mikhail},
  booktitle    = {38th Annual Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {Vancouver, Canada},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Average gradient outer product as a mechanism for deep neural collapse}},
  doi          = {10.52202/079017-4156},
  volume       = {37},
  year         = {2024},
}

@inproceedings{18875,
  abstract     = {Current state-of-the-art methods for differentially private model training are based on matrix factorization techniques. However, these methods suffer from high computational overhead because they require numerically solving a demanding optimization problem to determine an approximately optimal factorization prior to the actual model training. In this work, we present a new matrix factorization approach, BSR, which overcomes this computational bottleneck. By exploiting properties of the standard matrix square root, BSR allows to efficiently handle also large-scale problems. For the key scenario of stochastic gradient descent with momentum and weight decay, we even derive analytical expressions for BSR that render the computational overhead negligible. We prove bounds on the approximation quality that hold both in the centralized and in the federated learning setting. Our numerical experiments demonstrate that models trained using BSR perform on par with the best existing methods, while completely avoiding their computational overhead.},
  author       = {Kalinin, Nikita and Lampert, Christoph},
  booktitle    = {38th Annual Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {Vancouver, Canada},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Banded square root matrix factorization for differentially private model training}},
  doi          = {10.52202/079017-0560},
  volume       = {37},
  year         = {2024},
}

@inproceedings{18996,
  abstract     = {We consider the linear causal representation learning setting where we observe a linear mixing of d unknown latent factors, which follow a linear structural causal model. Recent work has shown that it is possible to recover the latent factors as well as the underlying structural causal model over them, up to permutation and scaling, provided that we have at least d environments, each of which corresponds to perfect interventions on a single latent node (factor). After this powerful result, a key open problem faced by the community has been to relax these conditions: allow for coarser than perfect single-node interventions, and allow for fewer than d of them, since the number of latent factors d could be very large. In this work, we consider precisely such a setting, where we allow a smaller than d number of environments, and also allow for very coarse interventions that can very coarsely \textit{change the entire causal graph over the latent factors}. On the flip side, we relax what we wish to extract to simply the \textit{list of nodes that have shifted between one or more environments}. We provide a surprising identifiability result that it is indeed possible, under some very mild standard assumptions, to identify the set of shifted nodes. Our identifiability proof moreover is a constructive one: we explicitly provide necessary and sufficient conditions for a node to be a shifted node, and show that we can check these conditions given observed data. Our algorithm lends itself very naturally to the sample setting where instead of just interventional distributions, we are provided datasets of samples from each of these distributions. We corroborate our results on both synthetic experiments as well as an interesting psychometric dataset. The code can be found at https://github.com/TianyuCodings/iLCS.},
  author       = {Chen, Tianyu and Bello, Kevin and Locatello, Francesco and Aragam, Bryon and Ravikumar, Pradeep Kumar},
  booktitle    = {38th Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {Vancouver, Canada},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Identifying general mechanism shifts in linear causal representations}},
  doi          = {10.52202/079017-1343},
  volume       = {37},
  year         = {2024},
}

@inproceedings{19005,
  abstract     = {Causal representation learning promises to extend causal models to hidden causal
variables from raw entangled measurements. However, most progress has focused
on proving identifiability results in different settings, and we are not aware of any
successful real-world application. At the same time, the field of dynamical systems
benefited from deep learning and scaled to countless applications but does not allow
parameter identification. In this paper, we draw a clear connection between the two
and their key assumptions, allowing us to apply identifiable methods developed
in causal representation learning to dynamical systems. At the same time, we can
leverage scalable differentiable solvers developed for differential equations to build
models that are both identifiable and practical. Overall, we learn explicitly controllable models that isolate the trajectory-specific parameters for further downstream
tasks such as out-of-distribution classification or treatment effect estimation. We
experiment with a wind simulator with partially known factors of variation. We
also apply the resulting model to real-world climate data and successfully answer
downstream causal questions in line with existing literature on climate change.
Code is available at https://github.com/CausalLearningAI/crl-dynamical-systems.},
  author       = {Yao, Dingling and Muller, Caroline J and Locatello, Francesco},
  booktitle    = {38th Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {Vancouver, Canada},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Marrying causal representation learning with dynamical systems for science}},
  doi          = {10.52202/079017-2290},
  volume       = {37},
  year         = {2024},
}

@inproceedings{19512,
  abstract     = {Differential privacy with gradual expiration models the setting where data items
arrive in a stream and at a given time t the privacy loss guaranteed for a data item
seen at time (t − d) is εg(d), where g is a monotonically non-decreasing function.
We study the fundamental continual (binary) counting problem where each data
item consists of a bit, and the algorithm needs to output at each time step the sum of
all the bits streamed so far. For a stream of length T and privacy without expiration
continual counting is possible with maximum (over all time steps) additive error
O(log2
(T)/ε) and the best known lower bound is Ω(log(T)/ε); closing this gap
is a challenging open problem.
We show that the situation is very different for privacy with gradual expiration by
giving upper and lower bounds for a large set of expiration functions g. Specifically,
our algorithm achieves an additive error of O(log(T)/ε) for a large set of privacy
expiration functions. We also give a lower bound that shows that if C is the additive
error of any ε-DP algorithm for this problem, then the product of C and the privacy
expiration function after 2C steps must be Ω(log(T)/ε). Our algorithm matches
this lower bound as its additive error is O(log(T)/ε), even when g(2C) = O(1).
Our empirical evaluation shows that we achieve a slowly growing privacy loss
with significantly smaller empirical privacy loss for large values of d than a natural
baseline algorithm.},
  author       = {Andersson, Joel Daniel and Henzinger, Monika H and Pagh, Rasmus and Steiner, Teresa Anna and Upadhyay, Jalaj},
  booktitle    = {38th Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {Vancouver, Canada},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Continual counting with gradual privacy expiration}},
  doi          = {10.52202/079017-0299},
  volume       = {37},
  year         = {2024},
}

@inproceedings{19515,
  abstract     = {Neural models learn data representations that lie on low-dimensional manifolds,
yet modeling the relation between these representational spaces is an ongoing challenge. By integrating spectral geometry principles into neural modeling, we show
that this problem can be better addressed in the functional domain, mitigating complexity, while enhancing interpretability and performances on downstream tasks.
To this end, we introduce a multi-purpose framework to the representation learning
community, which allows to: (i) compare different spaces in an interpretable way
and measure their intrinsic similarity; (ii) find correspondences between them, both
in unsupervised and weakly supervised settings, and (iii) to effectively transfer
representations between distinct spaces. We validate our framework on various
applications, ranging from stitching to retrieval tasks, and on multiple modalities,
demonstrating that Latent Functional Maps can serve as a swiss-army knife for
representation alignment},
  author       = {Fumero, Marco and Pegoraro, Marco and Maiorca, Valentino and Locatello, Francesco and Rodolà, Emanuele},
  booktitle    = {38th Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {Vancouver, Canada},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Latent functional maps: A spectral framework for representation alignment}},
  doi          = {10.52202/079017-2115},
  volume       = {37},
  year         = {2024},
}

@inproceedings{19509,
  abstract     = {Machine Learning and AI have the potential to transform data-driven scientific discovery,
enabling accurate predictions for several scientific phenomena. As many scientific questions
are inherently causal, this paper looks at the causal inference task of treatment effect
estimation, where the outcome of interest is recorded in high-dimensional observations
in a Randomized Controlled Trial (RCT). Despite being the simplest possible causal setting
and a perfect fit for deep learning, we theoretically find that many common choices in the
literature may lead to biased estimates. To test the practical impact of these considerations,
we recorded ISTAnt, the first real-world benchmark for causal inference downstream tasks
on high-dimensional observations as an RCT studying how garden ants (Lasius neglectus)
respond to microparticles applied onto their colony members by hygienic grooming.
Comparing 6 480 models fine-tuned from state-of-the-art visual backbones, we find that
the sampling and modeling choices significantly affect the accuracy of the causal estimate,
and that classification accuracy is not a proxy thereof. We further validated the analysis,
repeating it on a synthetically generated visual data set controlling the causal model. Our
results suggest that future benchmarks should carefully consider real downstream scientific
questions, especially causal ones. Further, we highlight guidelines for representation
learning methods to help answer causal questions in the sciences.
Code: https://github.com/CausalLearningAI/ISTAnt
Data: https://doi.org/10.6084/m9.figshare.26484934.v2},
  author       = {Cadei, Riccardo and Lindorfer, Lukas and Cremer, Sylvia and Schmid, Cordelia and Locatello, Francesco},
  booktitle    = {38th Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {Vancouver, Canada},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Smoke and mirrors in causal downstream tasks}},
  doi          = {10.52202/079017-0820},
  volume       = {37},
  year         = {2024},
}

@inproceedings{14921,
  abstract     = {Neural collapse (NC) refers to the surprising structure of the last layer of deep neural networks in the terminal phase of gradient descent training. Recently, an increasing amount of experimental evidence has pointed to the propagation of NC to earlier layers of neural networks. However, while the NC in the last layer is well studied theoretically, much less is known about its multi-layered counterpart - deep neural collapse (DNC). In particular, existing work focuses either on linear layers or only on the last two layers at the price of an extra assumption. Our paper fills this gap by generalizing the established analytical framework for NC - the unconstrained features model - to multiple non-linear layers. Our key technical contribution is to show that, in a deep unconstrained features model, the unique global optimum for binary classification exhibits all the properties typical of DNC. This explains the existing experimental evidence of DNC. We also empirically show that (i) by optimizing deep unconstrained features models via gradient descent, the resulting solution agrees well with our theory, and (ii) trained networks recover the unconstrained features suitable for the occurrence of DNC, thus supporting the validity of this modeling principle.},
  author       = {Súkeník, Peter and Mondelli, Marco and Lampert, Christoph},
  booktitle    = {37th Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {New Orleans, LA, United States},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Deep neural collapse is provably optimal for the deep unconstrained features model}},
  year         = {2023},
}

@inproceedings{14954,
  abstract     = {When domain knowledge is limited and experimentation is restricted by ethical,
financial, or time constraints, practitioners turn to observational causal discovery
methods to recover the causal structure, exploiting the statistical properties of their
data. Because causal discovery without further assumptions is an ill-posed problem,
each algorithm comes with its own set of usually untestable assumptions, some
of which are hard to meet in real datasets. Motivated by these considerations, this
paper extensively benchmarks the empirical performance of recent causal discovery
methods on observational iid data generated under different background conditions,
allowing for violations of the critical assumptions required by each selected approach. Our experimental findings show that score matching-based methods demonstrate surprising performance in the false positive and false negative rate of the
inferred graph in these challenging scenarios, and we provide theoretical insights
into their performance. This work is also the first effort to benchmark the stability of
causal discovery algorithms with respect to the values of their hyperparameters. Finally, we hope this paper will set a new standard for the evaluation of causal discovery methods and can serve as an accessible entry point for practitioners interested
in the field, highlighting the empirical implications of different algorithm choices.},
  author       = {Montagna, Francesco and Mastakouri, Atalanti A. and Eulig, Elias and Noceti, Nicoletta and Rosasco, Lorenzo and Janzing, Dominik and Aragam, Bryon and Locatello, Francesco},
  booktitle    = {37th Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {New Orleans, LA, United States},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Assumption violations in causal discovery and the robustness of score matching}},
  doi          = {10.52202/075280-2050},
  volume       = {36},
  year         = {2023},
}

@inproceedings{15023,
  abstract     = {Reinforcement learning has shown promising results in learning neural network policies for complicated control tasks. However, the lack of formal guarantees about the behavior of such policies remains an impediment to their deployment. We propose a novel method for learning a composition of neural network policies in stochastic environments, along with a formal certificate which guarantees that a specification over the policy's behavior is satisfied with the desired probability. Unlike prior work on verifiable RL, our approach leverages the compositional nature of logical specifications provided in SpectRL, to learn over graphs of probabilistic reach-avoid specifications. The formal guarantees are provided by learning neural network policies together with reach-avoid supermartingales (RASM) for the graph’s sub-tasks and then composing them into a global policy. We also derive a tighter lower bound compared to previous work on the probability of reach-avoidance implied by a RASM, which is required to find a compositional policy with an acceptable probabilistic threshold for complex tasks with multiple edge policies. We implement a prototype of our approach and evaluate it on a Stochastic Nine Rooms environment.},
  author       = {Zikelic, Dorde and Lechner, Mathias and Verma, Abhinav and Chatterjee, Krishnendu and Henzinger, Thomas A},
  booktitle    = {37th Conference on Neural Information Processing Systems},
  issn         = {1049-5258},
  location     = {New Orleans, LA, United States},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Compositional policy learning in stochastic control systems with formal guarantees}},
  year         = {2023},
}

@inproceedings{14953,
  abstract     = {This paper provides statistical sample complexity bounds for score-matching and
its applications in causal discovery. We demonstrate that accurate estimation of the
score function is achievable by training a standard deep ReLU neural network using
stochastic gradient descent. We establish bounds on the error rate of recovering
causal relationships using the score-matching-based causal discovery method of
Rolland et al. [2022], assuming a sufficiently good estimation of the score function.
Finally, we analyze the upper bound of score-matching estimation within the scorebased generative modeling, which has been applied for causal discovery but is also
of independent interest within the domain of generative models.y},
  author       = {Zhu, Zhenyu and Locatello, Francesco and Cevher, Volkan},
  booktitle    = {37th Conference on Neural Information Processing Systems},
  isbn         = {9781713899921},
  issn         = {1049-5258},
  location     = {New Orleans, LA, United States},
  pages        = {3325--3337},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Sample complexity bounds for score-matching: Causal discovery and generative modeling}},
  doi          = {10.52202/075280-0147},
  volume       = {36},
  year         = {2023},
}

@inproceedings{12537,
  abstract     = {The Neural Tangent Kernel (NTK) has emerged as a powerful tool to provide memorization, optimization and generalization guarantees in deep neural networks. A line of work has studied the NTK spectrum for two-layer and deep networks with at least a layer with Ω(N) neurons, N being the number of training samples. Furthermore, there is increasing evidence suggesting that deep networks with sub-linear layer widths are powerful memorizers and optimizers, as long as the number of parameters exceeds the number of samples. Thus, a natural open question is whether the NTK is well conditioned in such a challenging sub-linear setup. In this paper, we answer this question in the affirmative. Our key technical contribution is a lower bound on the smallest NTK eigenvalue for deep networks with the minimum possible over-parameterization: the number of parameters is roughly Ω(N) and, hence, the number of neurons is as little as Ω(N−−√). To showcase the applicability of our NTK bounds, we provide two results concerning memorization capacity and optimization guarantees for gradient descent training.},
  author       = {Bombari, Simone and Amani, Mohammad Hossein and Mondelli, Marco},
  booktitle    = {36th Conference on Neural Information Processing Systems},
  isbn         = {9781713871088},
  issn         = {1049-5258},
  location     = {New Orleans, LA, United States},
  pages        = {7628--7640},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Memorization and optimization in deep neural networks with minimum over-parameterization}},
  volume       = {35},
  year         = {2022},
}

@inproceedings{14326,
  abstract     = {Learning object-centric representations of complex scenes is a promising step towards enabling efficient abstract reasoning from low-level perceptual features. Yet, most deep learning approaches learn distributed representations that do not capture the compositional properties of natural scenes. In this paper, we present the Slot Attention module, an architectural component that interfaces with perceptual representations such as the output of a convolutional neural network and produces a set of task-dependent abstract representations which we call slots. These slots are exchangeable and can bind to any object in the input by specializing through a competitive procedure over multiple rounds of attention. We empirically demonstrate that Slot Attention can extract object-centric representations that enable generalization to unseen compositions when trained on unsupervised object discovery and supervised property prediction tasks.

},
  author       = {Locatello, Francesco and Weissenborn, Dirk and Unterthiner, Thomas and Mahendran, Aravindh and Heigold, Georg and Uszkoreit, Jakob and Dosovitskiy, Alexey and Kipf, Thomas},
  booktitle    = {34th International Conference on Neural Information Processing Systems},
  isbn         = {9781713829546},
  issn         = {1049-5258},
  location     = {Virtual},
  pages        = {11525--11538},
  publisher    = {Neural Information Processing Systems Foundation},
  title        = {{Object-centric learning with slot attention}},
  volume       = {33},
  year         = {2020},
}