[{"page":"222","month":"10","ddc":["519"],"oa_version":"Published Version","_id":"23072","supervisor":[{"last_name":"Mondelli","id":"27EB676C-8706-11E9-9510-7717E6697425","first_name":"Marco","full_name":"Mondelli, Marco","orcid":"0000-0002-3242-7020"}],"type":"dissertation","publisher":"Institute of Science and Technology Austria","publication_status":"published","user_id":"8b945eb4-e2f2-11eb-945a-df72226e66a9","language":[{"iso":"eng"}],"related_material":{"record":[{"status":"public","relation":"part_of_dissertation","id":"14459"},{"relation":"part_of_dissertation","id":"12540","status":"public"},{"status":"public","id":"17469","relation":"part_of_dissertation"}]},"fulldoi":"https://doi.org/10.15479/AT-ISTA-23072","OA_place":"publisher","department":[{"_id":"MaMo"},{"_id":"GradSch"}],"acknowledged_ssus":[{"_id":"ScienComp"}],"corr_author":"1","publication_identifier":{"issn":["2663-337X"]},"file":[{"file_name":"2026_Koegler_Kevin_Thesis.zip","file_size":8504109,"checksum":"5fb2ff02dc9d8c659b2fb5e0802b06ff","file_id":"23084","date_updated":"2026-10-09T12:22:55Z","creator":"kkoegler","access_level":"closed","content_type":"application/zip","date_created":"2026-10-09T11:20:09Z","relation":"source_file"},{"access_level":"open_access","date_updated":"2026-10-09T13:25:49Z","creator":"kkoegler","file_id":"23085","checksum":"a6df3b04c97628a1703f63356fb56f96","file_size":6692605,"file_name":"2026_Koegler_Kevin_Thesis.pdf","success":1,"relation":"main_file","date_created":"2026-10-09T13:25:49Z","content_type":"application/pdf"}],"doi":"10.15479/AT-ISTA-23072","degree_awarded":"PhD","file_date_updated":"2026-10-09T13:25:49Z","status":"public","date_published":"2026-10-08T00:00:00Z","has_accepted_license":"1","alternative_title":["ISTA Thesis"],"day":"08","citation":{"ama":"Kögler K. Compression and inference in high dimensions: An approximate message passing perspective. 2026. doi:<a href=\"https://doi.org/10.15479/AT-ISTA-23072\">10.15479/AT-ISTA-23072</a>","ieee":"K. Kögler, “Compression and inference in high dimensions: An approximate message passing perspective,” Institute of Science and Technology Austria, 2026.","apa":"Kögler, K. (2026). <i>Compression and inference in high dimensions: An approximate message passing perspective</i>. Institute of Science and Technology Austria. <a href=\"https://doi.org/10.15479/AT-ISTA-23072\">https://doi.org/10.15479/AT-ISTA-23072</a>","ista":"Kögler K. 2026. Compression and inference in high dimensions: An approximate message passing perspective. Institute of Science and Technology Austria.","chicago":"Kögler, Kevin. “Compression and Inference in High Dimensions: An Approximate Message Passing Perspective.” Institute of Science and Technology Austria, 2026. <a href=\"https://doi.org/10.15479/AT-ISTA-23072\">https://doi.org/10.15479/AT-ISTA-23072</a>.","mla":"Kögler, Kevin. <i>Compression and Inference in High Dimensions: An Approximate Message Passing Perspective</i>. Institute of Science and Technology Austria, 2026, doi:<a href=\"https://doi.org/10.15479/AT-ISTA-23072\">10.15479/AT-ISTA-23072</a>.","short":"K. Kögler, Compression and Inference in High Dimensions: An Approximate Message Passing Perspective, Institute of Science and Technology Austria, 2026."},"project":[{"_id":"059876FA-7A3F-11EA-A408-12923DDC885E","name":"Prix Lopez-Loretta 2019 - Marco Mondelli"}],"abstract":[{"text":"Inferring signal from large quantities of data is one of the most important problems in today’s\r\nworld, with neural networks emerging as the most powerful solution. While there have been\r\nmassive breakthroughs in the ML space, there are two main drawbacks. The first is that\r\nmost of these models are “black boxes”, with very little understanding of why they work,\r\nwhich makes it hard to find the correct architecture for a given problem. The architecture is\r\ncrucial, as the widely used training algorithms are “local” and thus rely on the loss landscape\r\nbeing benign enough to find a global optimum. This brings us to the second problem: while\r\nthere is a lot of evidence that this can be overcome with large enough data, this is less clear\r\nfor datasets which are noisy, do not have enough samples, or have a non-standard structure.\r\nHistorically, on a smaller scale, the alternative has been simple (Bayesian) statistical methods,\r\nwhich also have the advantage of being interpretable and adaptable. However, many of the\r\nclassical results cannot be easily extended to the high-dimensional regime in which modern\r\ndata lives.\r\nIn this work we want to consider two different settings relating to the above. The first is\r\napproximate message passing (AMP), an iterative algorithm for high-dimensional (generalized)\r\nlinear models that easily lends itself to Bayesian setups, and which in many cases is provably\r\nBayes-optimal. The second is an autoencoder with a shallow encoder, for which we can\r\nanalyze the weights along the training trajectory of gradient descent. Surprisingly, we will find\r\na connection between the two, as we recover certain structures of AMP.\r\nWe will first introduce the AMP framework by studying an extension that can be applied\r\nto a wider variety of data: rotationally invariant generalized approximate message passing\r\n(RI-GAMP). This shows that one can extend the AMP framework to a much wider class of\r\nalgorithms, relevant to more practical settings. The results are then also confirmed on real\r\ndata that go beyond the theoretical setting.\r\nWe then move to the setup of an autoencoder with a shallow encoder and a Bayesian prior in\r\nthe high-dimensional regime, in which the input and compression dimensions are large and\r\nproportional; this is also a first step towards opening one of the black boxes above, as we\r\ncan say precisely when and why the network works. While autoencoders are widely used in\r\nmodern ML architectures, most fundamental research in this direction has focused on extreme\r\nasymptotic regimes, where the autoencoder is either severely over- or under-parameterized,\r\nand which do not extend to the proportional high-dimensional regime. We show that for\r\nGaussian priors the network finds the global optimum, and these theoretical findings extrapolate\r\nstrikingly well to real image data. Next we extend this analysis to more structured priors and\r\na deeper decoder. For sparse Gaussian inputs in the training regime studied here, the shallow\r\ndecoder does not exploit sparsity and yields the same dynamics as in the Gaussian setting.\r\nWe then show how a deeper decoder can break this behavior, and interestingly, if we treat the\r\nproblem as an AMP problem, the network eventually learns the same decoding as AMP. This\r\ntoo extrapolates to real image data.\r\nLastly, we will show that the AMP setup has real practical use, by using a practical variant,\r\ngenetic vector approximate message passing (gVAMP), for predicting plasma protein levels\r\nfrom genetic information using the UKB GWAS data. This is a perfect example of the problems\r\nin the high-dimensional regime: approximately 8.4 million dimensions and 40,000 samples,\r\nwhere the high noise, low sample count, and large size make it hard for ML methods to work.\r\nTo get the optimal prediction accuracy we consider a two-step inference. We first consider a\r\npre-processed version of the data, of about 880,000 dimensions, on which we run the gVAMP\r\nalgorithm; this allows us to find predictive regions, taking into account joint effects between\r\ndifferent regions, and for most proteins we end up selecting only about 0.1% of the dimensions.\r\nWe then consider the analogue of this small subset, on which we can now run a standard tree\r\nalgorithm (XGB) that no longer suffers from the aforementioned problems. This pipeline yields\r\nbetter prediction accuracy than recent nonlinear deep learning approaches applied to this data,\r\nas we are able to handle the pre-processing in a principled Bayesian way.","lang":"eng"}],"author":[{"first_name":"Kevin","last_name":"Kögler","id":"94ec913c-dc85-11ea-9058-e5051ab2428b","full_name":"Kögler, Kevin"}],"doi_confirm":"1","title":"Compression and inference in high dimensions: An approximate message passing perspective","date_updated":"2026-10-12T13:04:49Z","article_processing_charge":"No","year":"2026","date_created":"2026-10-07T11:20:26Z","acknowledgement":"This project was partially supported by the 2019 Lopez-Loreta Prize and\r\nthe ERC INF2 project (project number 101161364). Furthermore, the candidate acknowledges\r\nthe support from the Scientific Service Units of the Institute of Science and Technology Austria\r\nthrough resources provided by Scientific Computing.","oa":1}]