Investigating BERT's Knowledge of Language: Five Analysis Methods with NPIs. Warstadt, A., Cao, Y., Grosu, I., Peng, W., Blix, H., Nie, Y., Alsop, A., Bordia, S., Liu, H., Parrish, A., Wang, S., Phang, J., Mohananey, A., Htut, P. M., Jeretič, P., & Bowman, S. R. In Empirical Methods in Natural Language Processing (EMNLP), 2019. ![Investigating BERT's Knowledge of Language: Five Analysis Methods with NPIs [link]](https://bibbase.org/img/filetypes/link.svg "link")
Paper abstract bibtex Though state-of-the-art sentence representation models can perform tasks requiring significant knowledge of grammar, it is an open question how best to evaluate their grammatical knowledge. We explore five experimental methods inspired by prior work evaluating pretrained sentence representation models. We use a single linguistic phenomenon, negative polarity item (NPI) licensing in English, as a case study for our experiments. NPIs like "any" are grammatical only if they appear in a licensing environment like negation ("Sue doesn't have any cats" vs. "Sue has any cats"). This phenomenon is challenging because of the variety of NPI licensing environments that exist. We introduce an artificially generated dataset that manipulates key features of NPI licensing for the experiments. We find that BERT has significant knowledge of these features, but its success varies widely across different experimental methods. We conclude that a variety of methods is necessary to reveal all relevant aspects of a model's grammatical knowledge in a given domain.
@inproceedings{Warstadt2019,
abstract = {Though state-of-the-art sentence representation models can perform tasks requiring significant knowledge of grammar, it is an open question how best to evaluate their grammatical knowledge. We explore five experimental methods inspired by prior work evaluating pretrained sentence representation models. We use a single linguistic phenomenon, negative polarity item (NPI) licensing in English, as a case study for our experiments. NPIs like "any" are grammatical only if they appear in a licensing environment like negation ("Sue doesn't have any cats" vs. "Sue has any cats"). This phenomenon is challenging because of the variety of NPI licensing environments that exist. We introduce an artificially generated dataset that manipulates key features of NPI licensing for the experiments. We find that BERT has significant knowledge of these features, but its success varies widely across different experimental methods. We conclude that a variety of methods is necessary to reveal all relevant aspects of a model's grammatical knowledge in a given domain.},
archivePrefix = {arXiv},
arxivId = {1909.02597},
author = {Warstadt, Alex and Cao, Yu and Grosu, Ioana and Peng, Wei and Blix, Hagen and Nie, Yining and Alsop, Anna and Bordia, Shikha and Liu, Haokun and Parrish, Alicia and Wang, Sheng-Fu and Phang, Jason and Mohananey, Anhad and Htut, Phu Mon and Jereti{\v{c}}, Paloma and Bowman, Samuel R.},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
eprint = {1909.02597},
file = {:Users/shanest/Documents/Library/Warstadt et al/Empirical Methods in Natural Language Processing (EMNLP)/Warstadt et al. - 2019 - Investigating BERT's Knowledge of Language Five Analysis Methods with NPIs.pdf:pdf},
keywords = {method: method comparison,phenomenon: NPIs},
title = {{Investigating BERT's Knowledge of Language: Five Analysis Methods with NPIs}},
url = {http://arxiv.org/abs/1909.02597},
year = {2019}
}
Downloads: 0
{"_id":"KT865MbF3xqEt6HjS","bibbaseid":"warstadt-cao-grosu-peng-blix-nie-alsop-bordia-etal-investigatingbertsknowledgeoflanguagefiveanalysismethodswithnpis-2019","authorIDs":[],"author_short":["Warstadt, A.","Cao, Y.","Grosu, I.","Peng, W.","Blix, H.","Nie, Y.","Alsop, A.","Bordia, S.","Liu, H.","Parrish, A.","Wang, S.","Phang, J.","Mohananey, A.","Htut, P. M.","Jeretič, P.","Bowman, S. R."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","abstract":"Though state-of-the-art sentence representation models can perform tasks requiring significant knowledge of grammar, it is an open question how best to evaluate their grammatical knowledge. We explore five experimental methods inspired by prior work evaluating pretrained sentence representation models. We use a single linguistic phenomenon, negative polarity item (NPI) licensing in English, as a case study for our experiments. NPIs like \"any\" are grammatical only if they appear in a licensing environment like negation (\"Sue doesn't have any cats\" vs. \"Sue has any cats\"). This phenomenon is challenging because of the variety of NPI licensing environments that exist. We introduce an artificially generated dataset that manipulates key features of NPI licensing for the experiments. We find that BERT has significant knowledge of these features, but its success varies widely across different experimental methods. We conclude that a variety of methods is necessary to reveal all relevant aspects of a model's grammatical knowledge in a given domain.","archiveprefix":"arXiv","arxivid":"1909.02597","author":[{"propositions":[],"lastnames":["Warstadt"],"firstnames":["Alex"],"suffixes":[]},{"propositions":[],"lastnames":["Cao"],"firstnames":["Yu"],"suffixes":[]},{"propositions":[],"lastnames":["Grosu"],"firstnames":["Ioana"],"suffixes":[]},{"propositions":[],"lastnames":["Peng"],"firstnames":["Wei"],"suffixes":[]},{"propositions":[],"lastnames":["Blix"],"firstnames":["Hagen"],"suffixes":[]},{"propositions":[],"lastnames":["Nie"],"firstnames":["Yining"],"suffixes":[]},{"propositions":[],"lastnames":["Alsop"],"firstnames":["Anna"],"suffixes":[]},{"propositions":[],"lastnames":["Bordia"],"firstnames":["Shikha"],"suffixes":[]},{"propositions":[],"lastnames":["Liu"],"firstnames":["Haokun"],"suffixes":[]},{"propositions":[],"lastnames":["Parrish"],"firstnames":["Alicia"],"suffixes":[]},{"propositions":[],"lastnames":["Wang"],"firstnames":["Sheng-Fu"],"suffixes":[]},{"propositions":[],"lastnames":["Phang"],"firstnames":["Jason"],"suffixes":[]},{"propositions":[],"lastnames":["Mohananey"],"firstnames":["Anhad"],"suffixes":[]},{"propositions":[],"lastnames":["Htut"],"firstnames":["Phu","Mon"],"suffixes":[]},{"propositions":[],"lastnames":["Jeretič"],"firstnames":["Paloma"],"suffixes":[]},{"propositions":[],"lastnames":["Bowman"],"firstnames":["Samuel","R."],"suffixes":[]}],"booktitle":"Empirical Methods in Natural Language Processing (EMNLP)","eprint":"1909.02597","file":":Users/shanest/Documents/Library/Warstadt et al/Empirical Methods in Natural Language Processing (EMNLP)/Warstadt et al. - 2019 - Investigating BERT's Knowledge of Language Five Analysis Methods with NPIs.pdf:pdf","keywords":"method: method comparison,phenomenon: NPIs","title":"Investigating BERT's Knowledge of Language: Five Analysis Methods with NPIs","url":"http://arxiv.org/abs/1909.02597","year":"2019","bibtex":"@inproceedings{Warstadt2019,\nabstract = {Though state-of-the-art sentence representation models can perform tasks requiring significant knowledge of grammar, it is an open question how best to evaluate their grammatical knowledge. We explore five experimental methods inspired by prior work evaluating pretrained sentence representation models. We use a single linguistic phenomenon, negative polarity item (NPI) licensing in English, as a case study for our experiments. NPIs like \"any\" are grammatical only if they appear in a licensing environment like negation (\"Sue doesn't have any cats\" vs. \"Sue has any cats\"). This phenomenon is challenging because of the variety of NPI licensing environments that exist. We introduce an artificially generated dataset that manipulates key features of NPI licensing for the experiments. We find that BERT has significant knowledge of these features, but its success varies widely across different experimental methods. We conclude that a variety of methods is necessary to reveal all relevant aspects of a model's grammatical knowledge in a given domain.},\narchivePrefix = {arXiv},\narxivId = {1909.02597},\nauthor = {Warstadt, Alex and Cao, Yu and Grosu, Ioana and Peng, Wei and Blix, Hagen and Nie, Yining and Alsop, Anna and Bordia, Shikha and Liu, Haokun and Parrish, Alicia and Wang, Sheng-Fu and Phang, Jason and Mohananey, Anhad and Htut, Phu Mon and Jereti{\\v{c}}, Paloma and Bowman, Samuel R.},\nbooktitle = {Empirical Methods in Natural Language Processing (EMNLP)},\neprint = {1909.02597},\nfile = {:Users/shanest/Documents/Library/Warstadt et al/Empirical Methods in Natural Language Processing (EMNLP)/Warstadt et al. - 2019 - Investigating BERT's Knowledge of Language Five Analysis Methods with NPIs.pdf:pdf},\nkeywords = {method: method comparison,phenomenon: NPIs},\ntitle = {{Investigating BERT's Knowledge of Language: Five Analysis Methods with NPIs}},\nurl = {http://arxiv.org/abs/1909.02597},\nyear = {2019}\n}\n","author_short":["Warstadt, A.","Cao, Y.","Grosu, I.","Peng, W.","Blix, H.","Nie, Y.","Alsop, A.","Bordia, S.","Liu, H.","Parrish, A.","Wang, S.","Phang, J.","Mohananey, A.","Htut, P. M.","Jeretič, P.","Bowman, S. R."],"key":"Warstadt2019","id":"Warstadt2019","bibbaseid":"warstadt-cao-grosu-peng-blix-nie-alsop-bordia-etal-investigatingbertsknowledgeoflanguagefiveanalysismethodswithnpis-2019","role":"author","urls":{"Paper":"http://arxiv.org/abs/1909.02597"},"keyword":["method: method comparison","phenomenon: NPIs"],"metadata":{"authorlinks":{}},"downloads":0},"bibtype":"inproceedings","biburl":"https://www.shane.st/teaching/575/win20/MachineLearning-interpretability.bib","creationDate":"2020-01-05T04:04:02.863Z","downloads":0,"keywords":["method: method comparison","phenomenon: npis"],"search_terms":["investigating","bert","knowledge","language","five","analysis","methods","npis","warstadt","cao","grosu","peng","blix","nie","alsop","bordia","liu","parrish","wang","phang","mohananey","htut","jeretič","bowman"],"title":"Investigating BERT's Knowledge of Language: Five Analysis Methods with NPIs","year":2019,"dataSources":["okYcdTpf4JJ2zkj7A","znj7izS5PeehdLR3G"]}