A Structural Probe for Finding Syntax in Word Representations. Hewitt, J. & Manning, C. D. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pages 4129–4138, Stroudsburg, PA, USA, 2019. Association for Computational Linguistics. ![A Structural Probe for Finding Syntax in Word Representations [link]](https://bibbase.org/img/filetypes/link.svg "link")
Paper doi abstract bibtex Recent work has improved our ability to detect linguistic knowledge in word representations. However, current methods for detecting syntactic knowledge do not test whether syntax trees are represented in their entirety. In this work, we propose a structural probe, which evaluates whether syntax trees are embedded in a linear transformation of a neural network's word representation space. The probe identifies a linear transformation under which squared L2 distance encodes the distance between words in the parse tree, and one in which squared L2 norm encodes depth in the parse tree. Using our probe, we show that such transformations exist for both ELMo and BERT but not in baselines, providing evidence that entire syntax trees are embedded implicitly in deep models' vector geometry.
@inproceedings{Hewitt2019,
abstract = {Recent work has improved our ability to detect linguistic knowledge in word representations. However, current methods for detecting syntactic knowledge do not test whether syntax trees are represented in their entirety. In this work, we propose a structural probe, which evaluates whether syntax trees are embedded in a linear transformation of a neural network's word representation space. The probe identifies a linear transformation under which squared L2 distance encodes the distance between words in the parse tree, and one in which squared L2 norm encodes depth in the parse tree. Using our probe, we show that such transformations exist for both ELMo and BERT but not in baselines, providing evidence that entire syntax trees are embedded implicitly in deep models' vector geometry.},
address = {Stroudsburg, PA, USA},
author = {Hewitt, John and Manning, Christopher D.},
booktitle = {Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
doi = {10.18653/v1/N19-1419},
file = {:Users/shanest/Documents/Library/Hewitt, Manning/Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics Human Language Technologie./Hewitt, Manning - 2019 - A Structural Probe for Finding Syntax in Word Representations.pdf:pdf},
keywords = {method: geometry,phenomenon: dependency parsing},
pages = {4129--4138},
publisher = {Association for Computational Linguistics},
title = {{A Structural Probe for Finding Syntax in Word Representations}},
url = {http://aclweb.org/anthology/N19-1419},
year = {2019}
}
Downloads: 0
{"_id":"GeN2tXDdcFATLCPhM","bibbaseid":"hewitt-manning-astructuralprobeforfindingsyntaxinwordrepresentations-2019","authorIDs":[],"author_short":["Hewitt, J.","Manning, C. D."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","abstract":"Recent work has improved our ability to detect linguistic knowledge in word representations. However, current methods for detecting syntactic knowledge do not test whether syntax trees are represented in their entirety. In this work, we propose a structural probe, which evaluates whether syntax trees are embedded in a linear transformation of a neural network's word representation space. The probe identifies a linear transformation under which squared L2 distance encodes the distance between words in the parse tree, and one in which squared L2 norm encodes depth in the parse tree. Using our probe, we show that such transformations exist for both ELMo and BERT but not in baselines, providing evidence that entire syntax trees are embedded implicitly in deep models' vector geometry.","address":"Stroudsburg, PA, USA","author":[{"propositions":[],"lastnames":["Hewitt"],"firstnames":["John"],"suffixes":[]},{"propositions":[],"lastnames":["Manning"],"firstnames":["Christopher","D."],"suffixes":[]}],"booktitle":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)","doi":"10.18653/v1/N19-1419","file":":Users/shanest/Documents/Library/Hewitt, Manning/Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics Human Language Technologie./Hewitt, Manning - 2019 - A Structural Probe for Finding Syntax in Word Representations.pdf:pdf","keywords":"method: geometry,phenomenon: dependency parsing","pages":"4129–4138","publisher":"Association for Computational Linguistics","title":"A Structural Probe for Finding Syntax in Word Representations","url":"http://aclweb.org/anthology/N19-1419","year":"2019","bibtex":"@inproceedings{Hewitt2019,\nabstract = {Recent work has improved our ability to detect linguistic knowledge in word representations. However, current methods for detecting syntactic knowledge do not test whether syntax trees are represented in their entirety. In this work, we propose a structural probe, which evaluates whether syntax trees are embedded in a linear transformation of a neural network's word representation space. The probe identifies a linear transformation under which squared L2 distance encodes the distance between words in the parse tree, and one in which squared L2 norm encodes depth in the parse tree. Using our probe, we show that such transformations exist for both ELMo and BERT but not in baselines, providing evidence that entire syntax trees are embedded implicitly in deep models' vector geometry.},\naddress = {Stroudsburg, PA, USA},\nauthor = {Hewitt, John and Manning, Christopher D.},\nbooktitle = {Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},\ndoi = {10.18653/v1/N19-1419},\nfile = {:Users/shanest/Documents/Library/Hewitt, Manning/Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics Human Language Technologie./Hewitt, Manning - 2019 - A Structural Probe for Finding Syntax in Word Representations.pdf:pdf},\nkeywords = {method: geometry,phenomenon: dependency parsing},\npages = {4129--4138},\npublisher = {Association for Computational Linguistics},\ntitle = {{A Structural Probe for Finding Syntax in Word Representations}},\nurl = {http://aclweb.org/anthology/N19-1419},\nyear = {2019}\n}\n","author_short":["Hewitt, J.","Manning, C. D."],"key":"Hewitt2019","id":"Hewitt2019","bibbaseid":"hewitt-manning-astructuralprobeforfindingsyntaxinwordrepresentations-2019","role":"author","urls":{"Paper":"http://aclweb.org/anthology/N19-1419"},"keyword":["method: geometry","phenomenon: dependency parsing"],"metadata":{"authorlinks":{}},"downloads":0},"bibtype":"inproceedings","biburl":"https://www.shane.st/teaching/575/win20/MachineLearning-interpretability.bib","creationDate":"2020-01-05T04:04:02.894Z","downloads":0,"keywords":["method: geometry","phenomenon: dependency parsing"],"search_terms":["structural","probe","finding","syntax","word","representations","hewitt","manning"],"title":"A Structural Probe for Finding Syntax in Word Representations","year":2019,"dataSources":["okYcdTpf4JJ2zkj7A","znj7izS5PeehdLR3G"]}