[{"title":"Can LLMs separate instructions from data? And what do we even mean by that?","date_published":"2024-03-01T00:00:00Z","citation":{"ieee":"E. Zverev, S. Abdelnabi, S. Tabesh, M. Fritz, and C. Lampert, “Can LLMs separate instructions from data? And what do we even mean by that?,” <i>arXiv</i>. .","chicago":"Zverev, Egor, Sahar Abdelnabi, Soroush Tabesh, Mario Fritz, and Christoph Lampert. “Can LLMs Separate Instructions from Data? And What Do We Even Mean by That?” <i>ArXiv</i>, n.d. <a href=\"https://doi.org/10.48550/arXiv.2403.06833\">https://doi.org/10.48550/arXiv.2403.06833</a>.","ista":"Zverev E, Abdelnabi S, Tabesh S, Fritz M, Lampert C. Can LLMs separate instructions from data? And what do we even mean by that? arXiv, 2403.06833.","apa":"Zverev, E., Abdelnabi, S., Tabesh, S., Fritz, M., &#38; Lampert, C. (n.d.). Can LLMs separate instructions from data? And what do we even mean by that? <i>arXiv</i>. <a href=\"https://doi.org/10.48550/arXiv.2403.06833\">https://doi.org/10.48550/arXiv.2403.06833</a>","ama":"Zverev E, Abdelnabi S, Tabesh S, Fritz M, Lampert C. Can LLMs separate instructions from data? And what do we even mean by that? <i>arXiv</i>. doi:<a href=\"https://doi.org/10.48550/arXiv.2403.06833\">10.48550/arXiv.2403.06833</a>","mla":"Zverev, Egor, et al. “Can LLMs Separate Instructions from Data? And What Do We Even Mean by That?” <i>ArXiv</i>, 2403.06833, doi:<a href=\"https://doi.org/10.48550/arXiv.2403.06833\">10.48550/arXiv.2403.06833</a>.","short":"E. Zverev, S. Abdelnabi, S. Tabesh, M. Fritz, C. Lampert, ArXiv (n.d.)."},"OA_type":"green","status":"public","month":"03","type":"preprint","arxiv":1,"year":"2024","acknowledged_ssus":[{"_id":"ScienComp"}],"file_date_updated":"2025-02-20T10:11:45Z","date_updated":"2026-08-13T07:26:54Z","acknowledgement":"The authors would like to sincerely thank Juan Rocamonde for valuable feedback to our manuscript. We acknowledge the support from the Scientific Service Units (SSU) of ISTA through resources provided by Scientific Computing (SciComp). We thank Dan Alistarh for providing us with computational resources. This work was partially funded by the German Federal Ministry of Education and Research (BMBF) under the grant AIgenCY (16KIS2012) and ELSA – European Lighthouse on Secure and Safe AI funded by the European Union under grant agreement No. 101070617. Views and opinions expressed are however those of the authors only and do not necessarily reflect those of the European Union or European Commission. Neither the European Union nor the European Commission can be held responsible for them.","department":[{"_id":"GradSch"},{"_id":"ChLa"}],"fulldoi":"https://doi.org/10.48550/arXiv.2403.06833","tmp":{"short":"CC BY (4.0)","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","image":"/images/cc_by.png","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode"},"abstract":[{"text":"Instruction-tuned Large Language Models (LLMs) show impressive results in numerous practical applications, but they lack essential safety features that are common in other areas of computer science, particularly an explicit separation of instructions and data. This makes them vulnerable to manipulations such as indirect prompt injections and generally unsuitable for safety-critical tasks. Surprisingly, there is currently no established definition or benchmark to quantify this phenomenon. In this work, we close this gap by introducing a formal measure for instruction-data separation and an empirical variant that is calculable from a model's outputs. We also present a new dataset, SEP, that allows estimating the measure for real-world models. Our results on various LLMs show that the problem of instruction-data separation is real: all models fail to achieve high separation, and canonical mitigation techniques, such as prompt engineering and fine-tuning, either fail to substantially improve separation or reduce model utility. The source code and SEP dataset are openly accessible at https://github.com/egozverev/Shold-It-Be-Executed-Or-Processed.\r\n","lang":"eng"}],"doi":"10.48550/arXiv.2403.06833","external_id":{"arxiv":["2403.06833"]},"publication_status":"submitted","publication":"arXiv","article_processing_charge":"No","_id":"19063","oa_version":"Preprint","corr_author":"1","day":"01","language":[{"iso":"eng"}],"related_material":{"link":[{"relation":"software","url":" https://github.com/egozverev/Shold-It-Be-Executed-Or-Processed"}]},"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","author":[{"last_name":"Zverev","id":"05162b19-1340-11ed-8f02-fa94e0e8c3bc","full_name":"Zverev, Egor","first_name":"Egor"},{"last_name":"Abdelnabi","full_name":"Abdelnabi, Sahar","first_name":"Sahar"},{"last_name":"Tabesh","id":"06000900-6068-11ef-8d61-c2472ef2e752","full_name":"Tabesh, Soroush","first_name":"Soroush","orcid":"0009-0003-4119-6281"},{"last_name":"Fritz","first_name":"Mario","full_name":"Fritz, Mario"},{"id":"40C20FD2-F248-11E8-B48F-1D18A9856A87","last_name":"Lampert","orcid":"0000-0001-8622-7887","first_name":"Christoph","full_name":"Lampert, Christoph"}],"file":[{"date_created":"2025-02-20T10:11:45Z","file_name":"2403.06833v3.pdf","relation":"main_file","success":1,"content_type":"application/pdf","date_updated":"2025-02-20T10:11:45Z","file_id":"19064","access_level":"open_access","file_size":530972,"creator":"ezverev","checksum":"35eb43968684b87be59144603ef10af0"}],"OA_place":"repository","ddc":["000"],"article_number":"2403.06833","date_created":"2025-02-20T10:13:42Z","has_accepted_license":"1","oa":1}]