Why Do Multi-Agent LLM Systems Fail?. Cemri, M., Pan, M. Z., Yang, S., Agrawal, L. A., Chopra, B., Tiwari, R., Keutzer, K., Parameswaran, A., Klein, D., Ramchandran, K., Zaharia, M., Gonzalez, J. E., & Stoica, I. April, 2025. arXiv:2503.13657 [cs]![Why Do Multi-Agent LLM Systems Fail? [link]](https://bibbase.org/img/filetypes/link.svg "link")
Paper doi abstract bibtex Despite growing enthusiasm for Multi-Agent LLM Systems (MAS), their performance gains on popular benchmarks often remain minimal compared with single-agent frameworks. This gap highlights the need to systematically analyze the challenges hindering MAS effectiveness. We present MAST (Multi-Agent System Failure Taxonomy), the first empirically grounded taxonomy designed to understand MAS failures. We analyze seven popular MAS frameworks across over 200 tasks, involving six expert human annotators. Through this process, we identify 14 unique failure modes, organized into 3 overarching categories, (i) specification issues, (ii) inter-agent misalignment, and (iii) task verification. MAST emerges iteratively from rigorous inter-annotator agreement studies, achieving a Cohen's Kappa score of 0.88. To support scalable evaluation, we develop a validated LLM-as-a-Judge pipeline integrated with MAST. We leverage two case studies to demonstrate MAST's practical utility in analyzing failures and guiding MAS development. Our findings reveal that identified failures require more complex solutions, highlighting a clear roadmap for future research. We open source our comprehensive dataset and LLM annotator to facilitate further development of MAS.
@misc{cemri_why_2025,
title = {Why {Do} {Multi}-{Agent} {LLM} {Systems} {Fail}?},
url = {http://arxiv.org/abs/2503.13657},
doi = {10.48550/arXiv.2503.13657},
abstract = {Despite growing enthusiasm for Multi-Agent LLM Systems (MAS), their performance gains on popular benchmarks often remain minimal compared with single-agent frameworks. This gap highlights the need to systematically analyze the challenges hindering MAS effectiveness. We present MAST (Multi-Agent System Failure Taxonomy), the first empirically grounded taxonomy designed to understand MAS failures. We analyze seven popular MAS frameworks across over 200 tasks, involving six expert human annotators. Through this process, we identify 14 unique failure modes, organized into 3 overarching categories, (i) specification issues, (ii) inter-agent misalignment, and (iii) task verification. MAST emerges iteratively from rigorous inter-annotator agreement studies, achieving a Cohen's Kappa score of 0.88. To support scalable evaluation, we develop a validated LLM-as-a-Judge pipeline integrated with MAST. We leverage two case studies to demonstrate MAST's practical utility in analyzing failures and guiding MAS development. Our findings reveal that identified failures require more complex solutions, highlighting a clear roadmap for future research. We open source our comprehensive dataset and LLM annotator to facilitate further development of MAS.},
urldate = {2025-07-28},
publisher = {arXiv},
author = {Cemri, Mert and Pan, Melissa Z. and Yang, Shuyi and Agrawal, Lakshya A. and Chopra, Bhavya and Tiwari, Rishabh and Keutzer, Kurt and Parameswaran, Aditya and Klein, Dan and Ramchandran, Kannan and Zaharia, Matei and Gonzalez, Joseph E. and Stoica, Ion},
month = apr,
year = {2025},
note = {arXiv:2503.13657 [cs]},
keywords = {Computer Science - Artificial Intelligence},
}
Downloads: 0
{"_id":"GjSKALPxfL3TSvd7w","bibbaseid":"cemri-pan-yang-agrawal-chopra-tiwari-keutzer-parameswaran-etal-whydomultiagentllmsystemsfail-2025","author_short":["Cemri, M.","Pan, M. Z.","Yang, S.","Agrawal, L. A.","Chopra, B.","Tiwari, R.","Keutzer, K.","Parameswaran, A.","Klein, D.","Ramchandran, K.","Zaharia, M.","Gonzalez, J. E.","Stoica, I."],"bibdata":{"bibtype":"misc","type":"misc","title":"Why Do Multi-Agent LLM Systems Fail?","url":"http://arxiv.org/abs/2503.13657","doi":"10.48550/arXiv.2503.13657","abstract":"Despite growing enthusiasm for Multi-Agent LLM Systems (MAS), their performance gains on popular benchmarks often remain minimal compared with single-agent frameworks. This gap highlights the need to systematically analyze the challenges hindering MAS effectiveness. We present MAST (Multi-Agent System Failure Taxonomy), the first empirically grounded taxonomy designed to understand MAS failures. We analyze seven popular MAS frameworks across over 200 tasks, involving six expert human annotators. Through this process, we identify 14 unique failure modes, organized into 3 overarching categories, (i) specification issues, (ii) inter-agent misalignment, and (iii) task verification. MAST emerges iteratively from rigorous inter-annotator agreement studies, achieving a Cohen's Kappa score of 0.88. To support scalable evaluation, we develop a validated LLM-as-a-Judge pipeline integrated with MAST. We leverage two case studies to demonstrate MAST's practical utility in analyzing failures and guiding MAS development. Our findings reveal that identified failures require more complex solutions, highlighting a clear roadmap for future research. We open source our comprehensive dataset and LLM annotator to facilitate further development of MAS.","urldate":"2025-07-28","publisher":"arXiv","author":[{"propositions":[],"lastnames":["Cemri"],"firstnames":["Mert"],"suffixes":[]},{"propositions":[],"lastnames":["Pan"],"firstnames":["Melissa","Z."],"suffixes":[]},{"propositions":[],"lastnames":["Yang"],"firstnames":["Shuyi"],"suffixes":[]},{"propositions":[],"lastnames":["Agrawal"],"firstnames":["Lakshya","A."],"suffixes":[]},{"propositions":[],"lastnames":["Chopra"],"firstnames":["Bhavya"],"suffixes":[]},{"propositions":[],"lastnames":["Tiwari"],"firstnames":["Rishabh"],"suffixes":[]},{"propositions":[],"lastnames":["Keutzer"],"firstnames":["Kurt"],"suffixes":[]},{"propositions":[],"lastnames":["Parameswaran"],"firstnames":["Aditya"],"suffixes":[]},{"propositions":[],"lastnames":["Klein"],"firstnames":["Dan"],"suffixes":[]},{"propositions":[],"lastnames":["Ramchandran"],"firstnames":["Kannan"],"suffixes":[]},{"propositions":[],"lastnames":["Zaharia"],"firstnames":["Matei"],"suffixes":[]},{"propositions":[],"lastnames":["Gonzalez"],"firstnames":["Joseph","E."],"suffixes":[]},{"propositions":[],"lastnames":["Stoica"],"firstnames":["Ion"],"suffixes":[]}],"month":"April","year":"2025","note":"arXiv:2503.13657 [cs]","keywords":"Computer Science - Artificial Intelligence","bibtex":"@misc{cemri_why_2025,\n\ttitle = {Why {Do} {Multi}-{Agent} {LLM} {Systems} {Fail}?},\n\turl = {http://arxiv.org/abs/2503.13657},\n\tdoi = {10.48550/arXiv.2503.13657},\n\tabstract = {Despite growing enthusiasm for Multi-Agent LLM Systems (MAS), their performance gains on popular benchmarks often remain minimal compared with single-agent frameworks. This gap highlights the need to systematically analyze the challenges hindering MAS effectiveness. We present MAST (Multi-Agent System Failure Taxonomy), the first empirically grounded taxonomy designed to understand MAS failures. We analyze seven popular MAS frameworks across over 200 tasks, involving six expert human annotators. Through this process, we identify 14 unique failure modes, organized into 3 overarching categories, (i) specification issues, (ii) inter-agent misalignment, and (iii) task verification. MAST emerges iteratively from rigorous inter-annotator agreement studies, achieving a Cohen's Kappa score of 0.88. To support scalable evaluation, we develop a validated LLM-as-a-Judge pipeline integrated with MAST. We leverage two case studies to demonstrate MAST's practical utility in analyzing failures and guiding MAS development. Our findings reveal that identified failures require more complex solutions, highlighting a clear roadmap for future research. We open source our comprehensive dataset and LLM annotator to facilitate further development of MAS.},\n\turldate = {2025-07-28},\n\tpublisher = {arXiv},\n\tauthor = {Cemri, Mert and Pan, Melissa Z. and Yang, Shuyi and Agrawal, Lakshya A. and Chopra, Bhavya and Tiwari, Rishabh and Keutzer, Kurt and Parameswaran, Aditya and Klein, Dan and Ramchandran, Kannan and Zaharia, Matei and Gonzalez, Joseph E. and Stoica, Ion},\n\tmonth = apr,\n\tyear = {2025},\n\tnote = {arXiv:2503.13657 [cs]},\n\tkeywords = {Computer Science - Artificial Intelligence},\n}\n\n\n\n\n\n\n\n","author_short":["Cemri, M.","Pan, M. Z.","Yang, S.","Agrawal, L. A.","Chopra, B.","Tiwari, R.","Keutzer, K.","Parameswaran, A.","Klein, D.","Ramchandran, K.","Zaharia, M.","Gonzalez, J. E.","Stoica, I."],"key":"cemri_why_2025","id":"cemri_why_2025","bibbaseid":"cemri-pan-yang-agrawal-chopra-tiwari-keutzer-parameswaran-etal-whydomultiagentllmsystemsfail-2025","role":"author","urls":{"Paper":"http://arxiv.org/abs/2503.13657"},"keyword":["Computer Science - Artificial Intelligence"],"metadata":{"authorlinks":{}},"html":""},"bibtype":"misc","biburl":"https://bibbase.org/zotero/andreasmartin","dataSources":["jurZeGzSpYdkQ8rm4"],"keywords":["computer science - artificial intelligence"],"search_terms":["multi","agent","llm","systems","fail","cemri","pan","yang","agrawal","chopra","tiwari","keutzer","parameswaran","klein","ramchandran","zaharia","gonzalez","stoica"],"title":"Why Do Multi-Agent LLM Systems Fail?","year":2025}