Behavioral Analysis of Information Salience in Large Language Models. Trienes, J., Schlötterer, J., Li, J. J., & Seifert, C. In The 63rd Annual Meeting of the Association for Computational Linguistics (ACL 2025), pages 23428–23454, Vienna, Austria, May, 2025. ACL 2025. doi abstract bibtex Large Language Models (LLMs) excel at text summarization, a task that requires models to select content based on its importance. However, the exact notion of salience that LLMs have internalized remains unclear. To bridge this gap, we introduce an explainable framework to systematically derive and investigate information salience in LLMs through their summarization behavior. Using length-controlled summarization as a behavioral probe into the content selection process, and tracing the answerability of Questions Under Discussion throughout, we derive a proxy for how models prioritize information. Our experiments on 13 models across four datasets reveal that LLMs have a nuanced, hierarchical notion of salience, generally consistent across model families and sizes. While models show highly consistent behavior and hence salience patterns, this notion of salience cannot be accessed through introspection, and only weakly correlates with human perceptions of information salience.
@inproceedings{trienes_behavioral_2025,
address = {Vienna, Austria},
title = {Behavioral {Analysis} of {Information} {Salience} in {Large} {Language} {Models}},
doi = {https://doi.org/10.48550/arXiv.2502.14613},
abstract = {Large Language Models (LLMs) excel at text summarization, a task that requires models to select content based on its importance. However, the exact notion of salience that LLMs have internalized remains unclear. To bridge this gap, we introduce an explainable framework to systematically derive and investigate information salience in LLMs through their summarization behavior. Using length-controlled summarization as a behavioral probe into the content selection process, and tracing the answerability of Questions Under Discussion throughout, we derive a proxy for how models prioritize information. Our experiments on 13 models across four datasets reveal that LLMs have a nuanced, hierarchical notion of salience, generally consistent across model families and sizes. While models show highly consistent behavior and hence salience patterns, this notion of salience cannot be accessed through introspection, and only weakly correlates with human perceptions of information salience.},
language = {en},
booktitle = {The 63rd {Annual} {Meeting} of the {Association} for {Computational} {Linguistics} ({ACL} 2025)},
publisher = {ACL 2025},
author = {Trienes, Jan and Schlötterer, Jörg and Li, Junyi Jessy and Seifert, Christin},
month = may,
year = {2025},
keywords = {Foundational, SYS: CosmicAI Contact Author, WG: Explorable},
pages = {23428--23454},
}
Downloads: 0
{"_id":"j59B8qkXXwcDEppaZ","bibbaseid":"trienes-schltterer-li-seifert-behavioralanalysisofinformationsalienceinlargelanguagemodels-2025","author_short":["Trienes, J.","Schlötterer, J.","Li, J. J.","Seifert, C."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","address":"Vienna, Austria","title":"Behavioral Analysis of Information Salience in Large Language Models","doi":"https://doi.org/10.48550/arXiv.2502.14613","abstract":"Large Language Models (LLMs) excel at text summarization, a task that requires models to select content based on its importance. However, the exact notion of salience that LLMs have internalized remains unclear. To bridge this gap, we introduce an explainable framework to systematically derive and investigate information salience in LLMs through their summarization behavior. Using length-controlled summarization as a behavioral probe into the content selection process, and tracing the answerability of Questions Under Discussion throughout, we derive a proxy for how models prioritize information. Our experiments on 13 models across four datasets reveal that LLMs have a nuanced, hierarchical notion of salience, generally consistent across model families and sizes. While models show highly consistent behavior and hence salience patterns, this notion of salience cannot be accessed through introspection, and only weakly correlates with human perceptions of information salience.","language":"en","booktitle":"The 63rd Annual Meeting of the Association for Computational Linguistics (ACL 2025)","publisher":"ACL 2025","author":[{"propositions":[],"lastnames":["Trienes"],"firstnames":["Jan"],"suffixes":[]},{"propositions":[],"lastnames":["Schlötterer"],"firstnames":["Jörg"],"suffixes":[]},{"propositions":[],"lastnames":["Li"],"firstnames":["Junyi","Jessy"],"suffixes":[]},{"propositions":[],"lastnames":["Seifert"],"firstnames":["Christin"],"suffixes":[]}],"month":"May","year":"2025","keywords":"Foundational, SYS: CosmicAI Contact Author, WG: Explorable","pages":"23428–23454","bibtex":"@inproceedings{trienes_behavioral_2025,\n\taddress = {Vienna, Austria},\n\ttitle = {Behavioral {Analysis} of {Information} {Salience} in {Large} {Language} {Models}},\n\tdoi = {https://doi.org/10.48550/arXiv.2502.14613},\n\tabstract = {Large Language Models (LLMs) excel at text summarization, a task that requires models to select content based on its importance. However, the exact notion of salience that LLMs have internalized remains unclear. To bridge this gap, we introduce an explainable framework to systematically derive and investigate information salience in LLMs through their summarization behavior. Using length-controlled summarization as a behavioral probe into the content selection process, and tracing the answerability of Questions Under Discussion throughout, we derive a proxy for how models prioritize information. Our experiments on 13 models across four datasets reveal that LLMs have a nuanced, hierarchical notion of salience, generally consistent across model families and sizes. While models show highly consistent behavior and hence salience patterns, this notion of salience cannot be accessed through introspection, and only weakly correlates with human perceptions of information salience.},\n\tlanguage = {en},\n\tbooktitle = {The 63rd {Annual} {Meeting} of the {Association} for {Computational} {Linguistics} ({ACL} 2025)},\n\tpublisher = {ACL 2025},\n\tauthor = {Trienes, Jan and Schlötterer, Jörg and Li, Junyi Jessy and Seifert, Christin},\n\tmonth = may,\n\tyear = {2025},\n\tkeywords = {Foundational, SYS: CosmicAI Contact Author, WG: Explorable},\n\tpages = {23428--23454},\n}\n\n\n\n","author_short":["Trienes, J.","Schlötterer, J.","Li, J. J.","Seifert, C."],"key":"trienes_behavioral_2025","id":"trienes_behavioral_2025","bibbaseid":"trienes-schltterer-li-seifert-behavioralanalysisofinformationsalienceinlargelanguagemodels-2025","role":"author","urls":{},"keyword":["Foundational","SYS: CosmicAI Contact Author","WG: Explorable"],"metadata":{"authorlinks":{}}},"bibtype":"inproceedings","biburl":"https://bibbase.org/zotero-group/pratikmhatre/5933976","dataSources":["yJr5AAtJ5Sz3Q4WT4"],"keywords":["foundational","sys: cosmicai contact author","wg: explorable"],"search_terms":["behavioral","analysis","information","salience","large","language","models","trienes","schlötterer","li","seifert"],"title":"Behavioral Analysis of Information Salience in Large Language Models","year":2025}