Neural language models as psycholinguistic subjects: Representations of syntactic state

Neural language models as psycholinguistic subjects: Representations of syntactic state. Futrell, R., Wilcox, E., Morita, T., Qian, P., Ballesteros, M., & Levy, R. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pages 32–42, Stroudsburg, PA, USA, 2019. Association for Computational Linguistics.

Paper doi abstract bibtex

We deploy the methods of controlled psycholinguistic experimentation to shed light on the extent to which the behavior of neural network language models reflects incremental representations of syntactic state. To do so, we examine model behavior on artificial sentences containing a variety of syntactically complex structures. We test four models: two publicly available LSTM sequence models of English (Jozefowicz et al., 2016; Gulordava et al., 2018) trained on large datasets; an RNNG (Dyer et al., 2016) trained on a small, parsed dataset; and an LSTM trained on the same small corpus as the RNNG. We find evidence that the LSTMs trained on large datasets represent syntactic state over large spans of text in a way that is comparable to the RNNG, while the LSTM trained on the small dataset does not or does so only weakly.

@inproceedings{Futrell2019,
abstract = {We deploy the methods of controlled psycholinguistic experimentation to shed light on the extent to which the behavior of neural network language models reflects incremental representations of syntactic state. To do so, we examine model behavior on artificial sentences containing a variety of syntactically complex structures. We test four models: two publicly available LSTM sequence models of English (Jozefowicz et al., 2016; Gulordava et al., 2018) trained on large datasets; an RNNG (Dyer et al., 2016) trained on a small, parsed dataset; and an LSTM trained on the same small corpus as the RNNG. We find evidence that the LSTMs trained on large datasets represent syntactic state over large spans of text in a way that is comparable to the RNNG, while the LSTM trained on the small dataset does not or does so only weakly.},
address = {Stroudsburg, PA, USA},
archivePrefix = {arXiv},
arxivId = {1903.03260},
author = {Futrell, Richard and Wilcox, Ethan and Morita, Takashi and Qian, Peng and Ballesteros, Miguel and Levy, Roger},
booktitle = {Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
doi = {10.18653/v1/N19-1004},
eprint = {1903.03260},
file = {:Users/shanest/Documents/Library/Futrell et al/Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics Human Language Technologie./Futrell et al. - 2019 - Neural language models as psycholinguistic subjects Representations of syntactic state.pdf:pdf},
keywords = {method: psycholinguistic,phenomenon: incremental syntax},
pages = {32--42},
publisher = {Association for Computational Linguistics},
title = {{Neural language models as psycholinguistic subjects: Representations of syntactic state}},
url = {http://aclweb.org/anthology/N19-1004},
year = {2019}
}

Downloads: 0

{"_id":"XrR8msjZFPwo5LCuF","bibbaseid":"futrell-wilcox-morita-qian-ballesteros-levy-neurallanguagemodelsaspsycholinguisticsubjectsrepresentationsofsyntacticstate-2019","authorIDs":[],"author_short":["Futrell, R.","Wilcox, E.","Morita, T.","Qian, P.","Ballesteros, M.","Levy, R."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","abstract":"We deploy the methods of controlled psycholinguistic experimentation to shed light on the extent to which the behavior of neural network language models reflects incremental representations of syntactic state. To do so, we examine model behavior on artificial sentences containing a variety of syntactically complex structures. We test four models: two publicly available LSTM sequence models of English (Jozefowicz et al., 2016; Gulordava et al., 2018) trained on large datasets; an RNNG (Dyer et al., 2016) trained on a small, parsed dataset; and an LSTM trained on the same small corpus as the RNNG. We find evidence that the LSTMs trained on large datasets represent syntactic state over large spans of text in a way that is comparable to the RNNG, while the LSTM trained on the small dataset does not or does so only weakly.","address":"Stroudsburg, PA, USA","archiveprefix":"arXiv","arxivid":"1903.03260","author":[{"propositions":[],"lastnames":["Futrell"],"firstnames":["Richard"],"suffixes":[]},{"propositions":[],"lastnames":["Wilcox"],"firstnames":["Ethan"],"suffixes":[]},{"propositions":[],"lastnames":["Morita"],"firstnames":["Takashi"],"suffixes":[]},{"propositions":[],"lastnames":["Qian"],"firstnames":["Peng"],"suffixes":[]},{"propositions":[],"lastnames":["Ballesteros"],"firstnames":["Miguel"],"suffixes":[]},{"propositions":[],"lastnames":["Levy"],"firstnames":["Roger"],"suffixes":[]}],"booktitle":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)","doi":"10.18653/v1/N19-1004","eprint":"1903.03260","file":":Users/shanest/Documents/Library/Futrell et al/Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics Human Language Technologie./Futrell et al. - 2019 - Neural language models as psycholinguistic subjects Representations of syntactic state.pdf:pdf","keywords":"method: psycholinguistic,phenomenon: incremental syntax","pages":"32–42","publisher":"Association for Computational Linguistics","title":"Neural language models as psycholinguistic subjects: Representations of syntactic state","url":"http://aclweb.org/anthology/N19-1004","year":"2019","bibtex":"@inproceedings{Futrell2019,\nabstract = {We deploy the methods of controlled psycholinguistic experimentation to shed light on the extent to which the behavior of neural network language models reflects incremental representations of syntactic state. To do so, we examine model behavior on artificial sentences containing a variety of syntactically complex structures. We test four models: two publicly available LSTM sequence models of English (Jozefowicz et al., 2016; Gulordava et al., 2018) trained on large datasets; an RNNG (Dyer et al., 2016) trained on a small, parsed dataset; and an LSTM trained on the same small corpus as the RNNG. We find evidence that the LSTMs trained on large datasets represent syntactic state over large spans of text in a way that is comparable to the RNNG, while the LSTM trained on the small dataset does not or does so only weakly.},\naddress = {Stroudsburg, PA, USA},\narchivePrefix = {arXiv},\narxivId = {1903.03260},\nauthor = {Futrell, Richard and Wilcox, Ethan and Morita, Takashi and Qian, Peng and Ballesteros, Miguel and Levy, Roger},\nbooktitle = {Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},\ndoi = {10.18653/v1/N19-1004},\neprint = {1903.03260},\nfile = {:Users/shanest/Documents/Library/Futrell et al/Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics Human Language Technologie./Futrell et al. - 2019 - Neural language models as psycholinguistic subjects Representations of syntactic state.pdf:pdf},\nkeywords = {method: psycholinguistic,phenomenon: incremental syntax},\npages = {32--42},\npublisher = {Association for Computational Linguistics},\ntitle = {{Neural language models as psycholinguistic subjects: Representations of syntactic state}},\nurl = {http://aclweb.org/anthology/N19-1004},\nyear = {2019}\n}\n","author_short":["Futrell, R.","Wilcox, E.","Morita, T.","Qian, P.","Ballesteros, M.","Levy, R."],"key":"Futrell2019","id":"Futrell2019","bibbaseid":"futrell-wilcox-morita-qian-ballesteros-levy-neurallanguagemodelsaspsycholinguisticsubjectsrepresentationsofsyntacticstate-2019","role":"author","urls":{"Paper":"http://aclweb.org/anthology/N19-1004"},"keyword":["method: psycholinguistic","phenomenon: incremental syntax"],"metadata":{"authorlinks":{}},"downloads":0},"bibtype":"inproceedings","biburl":"https://www.shane.st/teaching/575/win20/MachineLearning-interpretability.bib","creationDate":"2020-01-05T22:03:38.520Z","downloads":0,"keywords":["method: psycholinguistic","phenomenon: incremental syntax"],"search_terms":["neural","language","models","psycholinguistic","subjects","representations","syntactic","state","futrell","wilcox","morita","qian","ballesteros","levy"],"title":"Neural language models as psycholinguistic subjects: Representations of syntactic state","year":2019,"dataSources":["okYcdTpf4JJ2zkj7A"]}