Emergent Abilities of Large Language Models

Emergent Abilities of Large Language Models. Wei, J., Tay, Y., Bommasani, R., Raffel, C., Zoph, B., Borgeaud, S., Yogatama, D., Bosma, M., Zhou, D., Metzler, D., Chi, E. H., Hashimoto, T., Vinyals, O., Liang, P., Dean, J., & Fedus, W. arXiv, October, 2022.

Paper doi abstract bibtex 2 downloads

Scaling up language models has been shown to predictably improve performance and sample efficiency on a wide range of downstream tasks. This paper instead discusses an unpredictable phenomenon that we refer to as emergent abilities of large language models. We consider an ability to be emergent if it is not present in smaller models but is present in larger models. Thus, emergent abilities cannot be predicted simply by extrapolating the performance of smaller models. The existence of such emergence implies that additional scaling could further expand the range of capabilities of language models.

@article{weiEmergentAbilitiesLarge2022,
	title = {Emergent {Abilities} of {Large} {Language} {Models}},
	url = {http://arxiv.org/abs/2206.07682},
	doi = {10.48550/arXiv.2206.07682},
	abstract = {Scaling up language models has been shown to predictably improve performance and sample efficiency on a wide range of downstream tasks. This paper instead discusses an unpredictable phenomenon that we refer to as emergent abilities of large language models. We consider an ability to be emergent if it is not present in smaller models but is present in larger models. Thus, emergent abilities cannot be predicted simply by extrapolating the performance of smaller models. The existence of such emergence implies that additional scaling could further expand the range of capabilities of language models.},
	urldate = {2024-09-24},
	publisher = {arXiv},
	author = {Wei, Jason and Tay, Yi and Bommasani, Rishi and Raffel, Colin and Zoph, Barret and Borgeaud, Sebastian and Yogatama, Dani and Bosma, Maarten and Zhou, Denny and Metzler, Donald and Chi, Ed H. and Hashimoto, Tatsunori and Vinyals, Oriol and Liang, Percy and Dean, Jeff and Fedus, William},
	month = oct,
	year = {2022},
	keywords = {Computer Science - Computation and Language},
}

Downloads: 2

{"_id":"viWqqWaxLi2rpbrvZ","bibbaseid":"wei-tay-bommasani-raffel-zoph-borgeaud-yogatama-bosma-etal-emergentabilitiesoflargelanguagemodels-2022","author_short":["Wei, J.","Tay, Y.","Bommasani, R.","Raffel, C.","Zoph, B.","Borgeaud, S.","Yogatama, D.","Bosma, M.","Zhou, D.","Metzler, D.","Chi, E. H.","Hashimoto, T.","Vinyals, O.","Liang, P.","Dean, J.","Fedus, W."],"bibdata":{"bibtype":"article","type":"article","title":"Emergent Abilities of Large Language Models","url":"http://arxiv.org/abs/2206.07682","doi":"10.48550/arXiv.2206.07682","abstract":"Scaling up language models has been shown to predictably improve performance and sample efficiency on a wide range of downstream tasks. This paper instead discusses an unpredictable phenomenon that we refer to as emergent abilities of large language models. We consider an ability to be emergent if it is not present in smaller models but is present in larger models. Thus, emergent abilities cannot be predicted simply by extrapolating the performance of smaller models. The existence of such emergence implies that additional scaling could further expand the range of capabilities of language models.","urldate":"2024-09-24","publisher":"arXiv","author":[{"propositions":[],"lastnames":["Wei"],"firstnames":["Jason"],"suffixes":[]},{"propositions":[],"lastnames":["Tay"],"firstnames":["Yi"],"suffixes":[]},{"propositions":[],"lastnames":["Bommasani"],"firstnames":["Rishi"],"suffixes":[]},{"propositions":[],"lastnames":["Raffel"],"firstnames":["Colin"],"suffixes":[]},{"propositions":[],"lastnames":["Zoph"],"firstnames":["Barret"],"suffixes":[]},{"propositions":[],"lastnames":["Borgeaud"],"firstnames":["Sebastian"],"suffixes":[]},{"propositions":[],"lastnames":["Yogatama"],"firstnames":["Dani"],"suffixes":[]},{"propositions":[],"lastnames":["Bosma"],"firstnames":["Maarten"],"suffixes":[]},{"propositions":[],"lastnames":["Zhou"],"firstnames":["Denny"],"suffixes":[]},{"propositions":[],"lastnames":["Metzler"],"firstnames":["Donald"],"suffixes":[]},{"propositions":[],"lastnames":["Chi"],"firstnames":["Ed","H."],"suffixes":[]},{"propositions":[],"lastnames":["Hashimoto"],"firstnames":["Tatsunori"],"suffixes":[]},{"propositions":[],"lastnames":["Vinyals"],"firstnames":["Oriol"],"suffixes":[]},{"propositions":[],"lastnames":["Liang"],"firstnames":["Percy"],"suffixes":[]},{"propositions":[],"lastnames":["Dean"],"firstnames":["Jeff"],"suffixes":[]},{"propositions":[],"lastnames":["Fedus"],"firstnames":["William"],"suffixes":[]}],"month":"October","year":"2022","keywords":"Computer Science - Computation and Language","bibtex":"@article{weiEmergentAbilitiesLarge2022,\n\ttitle = {Emergent {Abilities} of {Large} {Language} {Models}},\n\turl = {http://arxiv.org/abs/2206.07682},\n\tdoi = {10.48550/arXiv.2206.07682},\n\tabstract = {Scaling up language models has been shown to predictably improve performance and sample efficiency on a wide range of downstream tasks. This paper instead discusses an unpredictable phenomenon that we refer to as emergent abilities of large language models. We consider an ability to be emergent if it is not present in smaller models but is present in larger models. Thus, emergent abilities cannot be predicted simply by extrapolating the performance of smaller models. The existence of such emergence implies that additional scaling could further expand the range of capabilities of language models.},\n\turldate = {2024-09-24},\n\tpublisher = {arXiv},\n\tauthor = {Wei, Jason and Tay, Yi and Bommasani, Rishi and Raffel, Colin and Zoph, Barret and Borgeaud, Sebastian and Yogatama, Dani and Bosma, Maarten and Zhou, Denny and Metzler, Donald and Chi, Ed H. and Hashimoto, Tatsunori and Vinyals, Oriol and Liang, Percy and Dean, Jeff and Fedus, William},\n\tmonth = oct,\n\tyear = {2022},\n\tkeywords = {Computer Science - Computation and Language},\n}\n\n\n\n\n\n\n\n","author_short":["Wei, J.","Tay, Y.","Bommasani, R.","Raffel, C.","Zoph, B.","Borgeaud, S.","Yogatama, D.","Bosma, M.","Zhou, D.","Metzler, D.","Chi, E. H.","Hashimoto, T.","Vinyals, O.","Liang, P.","Dean, J.","Fedus, W."],"key":"weiEmergentAbilitiesLarge2022","id":"weiEmergentAbilitiesLarge2022","bibbaseid":"wei-tay-bommasani-raffel-zoph-borgeaud-yogatama-bosma-etal-emergentabilitiesoflargelanguagemodels-2022","role":"author","urls":{"Paper":"http://arxiv.org/abs/2206.07682"},"keyword":["Computer Science - Computation and Language"],"metadata":{"authorlinks":{}},"downloads":2,"html":""},"bibtype":"article","biburl":"https://bibbase.org/zotero/saurabhr","dataSources":["nxjWwW7fWbb5tfpKz","HQzSNhug84wo985ng"],"keywords":["computer science - computation and language"],"search_terms":["emergent","abilities","large","language","models","wei","tay","bommasani","raffel","zoph","borgeaud","yogatama","bosma","zhou","metzler","chi","hashimoto","vinyals","liang","dean","fedus"],"title":"Emergent Abilities of Large Language Models","year":2022,"downloads":2}