What do you learn from context? Probing for sentence structure in contextualized word representations. Tenney, I., Xia, P., Chen, B., Wang, A., Poliak, A., McCoy, R T., Kim, N., Van Durme, B., Bowman, S. R, Das, D., & Pavlick, E. In International Conference of Learning Representations (ICLR 2019), pages 1–17, 2019. ![What do you learn from context? Probing for sentence structure in contextualized word representations [link]](https://bibbase.org/img/filetypes/link.svg "link")
Paper abstract bibtex Contextualized representation models such as ELMo (Peters et al., 2018a) and BERT (Devlin et al., 2018) have recently achieved state-of-the-art results on a diverse array of downstream NLP tasks. Building on recent token-level probing work, we introduce a novel edge probing task design and construct a broad suite of sub-sentence tasks derived from the traditional structured NLP pipeline. We probe word-level contextual representations from four recent models and investigate how they encode sentence structure across a range of syntactic, semantic, local, and long-range phenomena. We find that existing models trained on language modeling and translation produce strong representations for syntactic phenomena, but only offer comparably small improvements on semantic tasks over a non-contextual baseline.
@inproceedings{Tenney2019,
abstract = {Contextualized representation models such as ELMo (Peters et al., 2018a) and BERT (Devlin et al., 2018) have recently achieved state-of-the-art results on a diverse array of downstream NLP tasks. Building on recent token-level probing work, we introduce a novel edge probing task design and construct a broad suite of sub-sentence tasks derived from the traditional structured NLP pipeline. We probe word-level contextual representations from four recent models and investigate how they encode sentence structure across a range of syntactic, semantic, local, and long-range phenomena. We find that existing models trained on language modeling and translation produce strong representations for syntactic phenomena, but only offer comparably small improvements on semantic tasks over a non-contextual baseline.},
archivePrefix = {arXiv},
arxivId = {1905.06316},
author = {Tenney, Ian and Xia, Patrick and Chen, Berlin and Wang, Alex and Poliak, Adam and McCoy, R Thomas and Kim, Najoung and {Van Durme}, Benjamin and Bowman, Samuel R and Das, Dipanjan and Pavlick, Ellie},
booktitle = {International Conference of Learning Representations (ICLR 2019)},
eprint = {1905.06316},
file = {:Users/shanest/Documents/Library/Tenney et al/International Conference of Learning Representations (ICLR 2019)/Tenney et al. - 2019 - What do you learn from context Probing for sentence structure in contextualized word representations.pdf:pdf},
keywords = {method: diagnostic classifier,method: model comparison,phenomenon: various},
pages = {1--17},
title = {{What do you learn from context? Probing for sentence structure in contextualized word representations}},
url = {http://arxiv.org/abs/1905.06316},
year = {2019}
}
Downloads: 0
{"_id":"s2LbEKdDu534DvZ9k","bibbaseid":"tenney-xia-chen-wang-poliak-mccoy-kim-vandurme-etal-whatdoyoulearnfromcontextprobingforsentencestructureincontextualizedwordrepresentations-2019","authorIDs":[],"author_short":["Tenney, I.","Xia, P.","Chen, B.","Wang, A.","Poliak, A.","McCoy, R T.","Kim, N.","Van Durme, B.","Bowman, S. R","Das, D.","Pavlick, E."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","abstract":"Contextualized representation models such as ELMo (Peters et al., 2018a) and BERT (Devlin et al., 2018) have recently achieved state-of-the-art results on a diverse array of downstream NLP tasks. Building on recent token-level probing work, we introduce a novel edge probing task design and construct a broad suite of sub-sentence tasks derived from the traditional structured NLP pipeline. We probe word-level contextual representations from four recent models and investigate how they encode sentence structure across a range of syntactic, semantic, local, and long-range phenomena. We find that existing models trained on language modeling and translation produce strong representations for syntactic phenomena, but only offer comparably small improvements on semantic tasks over a non-contextual baseline.","archiveprefix":"arXiv","arxivid":"1905.06316","author":[{"propositions":[],"lastnames":["Tenney"],"firstnames":["Ian"],"suffixes":[]},{"propositions":[],"lastnames":["Xia"],"firstnames":["Patrick"],"suffixes":[]},{"propositions":[],"lastnames":["Chen"],"firstnames":["Berlin"],"suffixes":[]},{"propositions":[],"lastnames":["Wang"],"firstnames":["Alex"],"suffixes":[]},{"propositions":[],"lastnames":["Poliak"],"firstnames":["Adam"],"suffixes":[]},{"propositions":[],"lastnames":["McCoy"],"firstnames":["R","Thomas"],"suffixes":[]},{"propositions":[],"lastnames":["Kim"],"firstnames":["Najoung"],"suffixes":[]},{"propositions":[],"lastnames":["Van Durme"],"firstnames":["Benjamin"],"suffixes":[]},{"propositions":[],"lastnames":["Bowman"],"firstnames":["Samuel","R"],"suffixes":[]},{"propositions":[],"lastnames":["Das"],"firstnames":["Dipanjan"],"suffixes":[]},{"propositions":[],"lastnames":["Pavlick"],"firstnames":["Ellie"],"suffixes":[]}],"booktitle":"International Conference of Learning Representations (ICLR 2019)","eprint":"1905.06316","file":":Users/shanest/Documents/Library/Tenney et al/International Conference of Learning Representations (ICLR 2019)/Tenney et al. - 2019 - What do you learn from context Probing for sentence structure in contextualized word representations.pdf:pdf","keywords":"method: diagnostic classifier,method: model comparison,phenomenon: various","pages":"1–17","title":"What do you learn from context? Probing for sentence structure in contextualized word representations","url":"http://arxiv.org/abs/1905.06316","year":"2019","bibtex":"@inproceedings{Tenney2019,\nabstract = {Contextualized representation models such as ELMo (Peters et al., 2018a) and BERT (Devlin et al., 2018) have recently achieved state-of-the-art results on a diverse array of downstream NLP tasks. Building on recent token-level probing work, we introduce a novel edge probing task design and construct a broad suite of sub-sentence tasks derived from the traditional structured NLP pipeline. We probe word-level contextual representations from four recent models and investigate how they encode sentence structure across a range of syntactic, semantic, local, and long-range phenomena. We find that existing models trained on language modeling and translation produce strong representations for syntactic phenomena, but only offer comparably small improvements on semantic tasks over a non-contextual baseline.},\narchivePrefix = {arXiv},\narxivId = {1905.06316},\nauthor = {Tenney, Ian and Xia, Patrick and Chen, Berlin and Wang, Alex and Poliak, Adam and McCoy, R Thomas and Kim, Najoung and {Van Durme}, Benjamin and Bowman, Samuel R and Das, Dipanjan and Pavlick, Ellie},\nbooktitle = {International Conference of Learning Representations (ICLR 2019)},\neprint = {1905.06316},\nfile = {:Users/shanest/Documents/Library/Tenney et al/International Conference of Learning Representations (ICLR 2019)/Tenney et al. - 2019 - What do you learn from context Probing for sentence structure in contextualized word representations.pdf:pdf},\nkeywords = {method: diagnostic classifier,method: model comparison,phenomenon: various},\npages = {1--17},\ntitle = {{What do you learn from context? Probing for sentence structure in contextualized word representations}},\nurl = {http://arxiv.org/abs/1905.06316},\nyear = {2019}\n}\n","author_short":["Tenney, I.","Xia, P.","Chen, B.","Wang, A.","Poliak, A.","McCoy, R T.","Kim, N.","Van Durme, B.","Bowman, S. R","Das, D.","Pavlick, E."],"key":"Tenney2019","id":"Tenney2019","bibbaseid":"tenney-xia-chen-wang-poliak-mccoy-kim-vandurme-etal-whatdoyoulearnfromcontextprobingforsentencestructureincontextualizedwordrepresentations-2019","role":"author","urls":{"Paper":"http://arxiv.org/abs/1905.06316"},"keyword":["method: diagnostic classifier","method: model comparison","phenomenon: various"],"metadata":{"authorlinks":{}},"downloads":0},"bibtype":"inproceedings","biburl":"https://www.shane.st/teaching/575/win20/MachineLearning-interpretability.bib","creationDate":"2020-01-05T04:04:02.882Z","downloads":0,"keywords":["method: diagnostic classifier","method: model comparison","phenomenon: various"],"search_terms":["learn","context","probing","sentence","structure","contextualized","word","representations","tenney","xia","chen","wang","poliak","mccoy","kim","van durme","bowman","das","pavlick"],"title":"What do you learn from context? Probing for sentence structure in contextualized word representations","year":2019,"dataSources":["okYcdTpf4JJ2zkj7A"]}